diff --git a/src/ast/transpiler_cache.rs b/src/ast/transpiler_cache.rs index 9d7bab38d65f..e8fa6e70fb02 100644 --- a/src/ast/transpiler_cache.rs +++ b/src/ast/transpiler_cache.rs @@ -11,6 +11,7 @@ //! `bun_bundler::cache` and are stored here type-erased as `*mut ()`). use crate::{ExportsKind, Source}; +use bun_core::strings::EncodingNonAscii; use core::ptr::NonNull; pub struct RuntimeTranspilerCache { @@ -52,7 +53,7 @@ impl Default for RuntimeTranspilerCache { bun_dispatch::link_interface! { pub TranspilerCacheImpl[Jsc] { fn get(source: &Source, parser_options: NonNull<()>, used_jsx: bool) -> bool; - fn put(output_code: &[u8], sourcemap: &[u8], esm_record: &[u8]); + fn put(output_code: &[u8], output_encoding: EncodingNonAscii, sourcemap: &[u8], esm_record: &[u8]); fn is_disabled() -> bool; } } @@ -83,10 +84,19 @@ impl RuntimeTranspilerCache { } } + /// `output_code` is the printer's buffer in `output_encoding` (`Latin1`, or native-endian `Utf16`). #[inline] - pub fn put(&mut self, output_code: &[u8], sourcemap: &[u8], esm_record: &[u8]) { + pub fn put( + &mut self, + output_code: &[u8], + output_encoding: EncodingNonAscii, + sourcemap: &[u8], + esm_record: &[u8], + ) { match self.r#impl { - Some(k) => Self::handle(k, self).put(output_code, sourcemap, esm_record), + Some(k) => { + Self::handle(k, self).put(output_code, output_encoding, sourcemap, esm_record) + } None => { if self.input_hash.is_none() { return; diff --git a/src/bun_core/string/mod.rs b/src/bun_core/string/mod.rs index 522a11a2a279..2bd6d2253ded 100644 --- a/src/bun_core/string/mod.rs +++ b/src/bun_core/string/mod.rs @@ -265,6 +265,14 @@ impl String { } } } + /// [`clone_utf16`](Self::clone_utf16) without the narrowing scan: the string is 16-bit. + pub fn clone_utf16_wide(s: &[u16]) -> Self { + if s.is_empty() { + return Self::EMPTY; + } + // SAFETY: s.as_ptr()/len describe a valid u16 slice. + unsafe { BunString__fromUTF16(s.as_ptr(), s.len()) } + } pub fn create_atom(s: &[u8]) -> Self { // SAFETY: s.as_ptr()/len describe a valid byte slice. unsafe { BunString__createAtom(s.as_ptr(), s.len()) } @@ -306,6 +314,33 @@ impl String { is_latin1: bool, ctx: Ctx, callback: ExternalStringImplFreeFunction, + ) -> Self { + // The length is a character count. UTF-16 goes through `create_external_utf16`. + debug_assert!(is_latin1); + Self::create_external_impl(bytes.as_ptr(), bytes.len(), is_latin1, ctx, callback) + } + + /// [`create_external`](Self::create_external) for UTF-16. `callback` gets the length in code units. + pub fn create_external_utf16( + units: &[u16], + ctx: Ctx, + callback: ExternalStringImplFreeFunction, + ) -> Self { + Self::create_external_impl( + units.as_ptr().cast::(), + units.len(), + false, + ctx, + callback, + ) + } + + fn create_external_impl( + ptr: *const u8, + len: usize, + is_latin1: bool, + ctx: Ctx, + callback: ExternalStringImplFreeFunction, ) -> Self { use core::ffi::c_void; // `Ctx` must be a pointer-sized, pointer-aligned handle. @@ -320,9 +355,9 @@ impl String { }; } let () = AssertPtrSized::::OK; - debug_assert!(!bytes.is_empty()); - if bytes.len() > Self::max_length() { - callback(ctx, bytes.as_ptr().cast_mut().cast::(), bytes.len()); + debug_assert!(len != 0); + if len > Self::max_length() { + callback(ctx, ptr.cast_mut().cast::(), len); return Self::DEAD; } // The const-assert above only checks size/alignment, so an owning @@ -344,16 +379,8 @@ impl String { ExternalStringImplFreeFunction, extern "C" fn(*mut c_void, *mut c_void, usize), >(callback) }); - // SAFETY: bytes describes a valid slice; len <= max_length checked. - let s = unsafe { - BunString__createExternal( - bytes.as_ptr(), - bytes.len(), - is_latin1, - ctx_erased, - cb_erased, - ) - }; + // SAFETY: `ptr`/`len` describe a live slice of the matching width; len <= max_length checked. + let s = unsafe { BunString__createExternal(ptr, len, is_latin1, ctx_erased, cb_erased) }; debug_assert!(s.tag != Tag::WTFStringImpl || s.as_wtf().ref_count() == 1); s } diff --git a/src/codegen/bake-codegen.ts b/src/codegen/bake-codegen.ts index a96c9f8d5562..318a1940e4f6 100644 --- a/src/codegen/bake-codegen.ts +++ b/src/codegen/bake-codegen.ts @@ -1,7 +1,7 @@ import assert from "node:assert"; import { existsSync, readFileSync, rmSync } from "node:fs"; import { basename, join } from "node:path"; -import { argParse, writeIfNotChanged } from "./helpers"; +import { argParse, checkAscii, writeIfNotChanged } from "./helpers"; // arg parsing let { "codegen-root": codegenRoot, debug, ...rest } = argParse(["codegen-root", "debug"]); @@ -143,6 +143,8 @@ async function run() { code = `let ${outName("unloadedModuleRegistry")}={},${outName("config")}={separateSSRGraph:${outName("$separateSSRGraph")}},${outName("server_exports")};${code}`; code = debug ? `((${params}) => {${code}})\n` : `((${params})=>{${code}})\n`; + // The dev server loads this as a Latin-1 string. + checkAscii(code); } else { code = debug ? `(async (${names}) => {${code}})({\n` : `(async(${names})=>{${code}})({`; } diff --git a/src/js_printer/lib.rs b/src/js_printer/lib.rs index 3366c8906499..ab527f8c5c4e 100644 --- a/src/js_printer/lib.rs +++ b/src/js_printer/lib.rs @@ -3203,54 +3203,18 @@ pub(crate) mod __gated_printer { } } - fn print_raw_template_literal(&mut self, bytes: &[u8]) { - if IS_JSON || !ASCII_ONLY { - self.print(bytes); - return; - } - - // Translate any non-ASCII to unicode escape sequences - // Note that this does not correctly handle malformed template literal strings - // template literal strings can contain invalid unicode code points - // and pretty much anything else - // - // we use WTF-8 here, but that's still not good enough. - // - let mut ascii_start: usize = 0; - let mut is_ascii = false; - let iter = CodepointIterator::init(bytes); - let mut cursor = strings::Cursor::default(); - - while iter.next(&mut cursor) { - match cursor.c as u32 { - // unlike other versions, we only want to mutate > 0x7F - 0..=LAST_ASCII => { - if !is_ascii { - ascii_start = cursor.i as usize; - is_ascii = true; - } - } - _ => { - if is_ascii { - self.print(&bytes[ascii_start..(cursor.i as usize)]); - is_ascii = false; - } - - match cursor.c as u32 { - c @ 0..=0xFFFF => self.print(&bmp_escape(c)[..]), - _ => { - self.print(b"\\u{"); - let _ = self.fmt(format_args!("{:x}", cursor.c)); - self.print(b"}"); - } - } - } - } + /// Text the program can read back as written: `RegExp#source`, template `.raw`, `Function#toString`. + fn print_verbatim(&mut self, text: &[u8]) { + let start = self.writer.slice().len(); + let wide_line_terminator = self.writer.print_verbatim_utf8(text); + if GENERATE_SOURCE_MAP && wide_line_terminator { + self.source_map_builder + .update_generated_verbatim(self.writer.slice(), start, text); } + } - if is_ascii { - self.print(&bytes[ascii_start..]); - } + fn print_raw_template_literal(&mut self, bytes: &[u8]) { + self.print_verbatim(bytes); } pub(crate) fn check_stack_overflow(&self) -> crate::Result<()> { @@ -4542,7 +4506,7 @@ pub(crate) mod __gated_printer { self.print_expr(e.value, level, flags); if !self.options.minify_whitespace && !self.options.minify_identifiers { self.print(b" /* "); - self.print(&e.comment[..]); + self.print_verbatim(&e.comment[..]); self.print(b" */"); } } @@ -4703,41 +4667,7 @@ pub(crate) mod __gated_printer { self.print(b" "); } - if IS_BUN_PLATFORM { - // Translate any non-ASCII to unicode escape sequences - let mut ascii_start: usize = 0; - let mut is_ascii = false; - let iter = CodepointIterator::init(&e.value); - let mut cursor = strings::Cursor::default(); - while iter.next(&mut cursor) { - match cursor.c as u32 { - FIRST_ASCII..=LAST_ASCII => { - if !is_ascii { - ascii_start = cursor.i as usize; - is_ascii = true; - } - } - _ => { - if is_ascii { - self.print(&e.value[ascii_start..(cursor.i as usize)]); - is_ascii = false; - } - - match cursor.c as u32 { - c @ 0..=0xFFFF => self.print(&bmp_escape(c)[..]), - c => self.print(&surrogate_pair_escape(c)[..]), - } - } - } - } - - if is_ascii { - self.print(&e.value[ascii_start..]); - } - } else { - // UTF8 sequence is fine - self.print(&e.value[..]); - } + self.print_verbatim(&e.value[..]); // Need a space before the next identifier to avoid it turning into flags self.prev_reg_exp_end = self.writer.written(); @@ -6823,7 +6753,7 @@ pub(crate) mod __gated_printer { // TODO: rewrite this to handle if !strings::contains(comment, b"*/") { self.print(b" /* "); - self.print(comment); + self.print_verbatim(comment); self.print(b" */"); } } @@ -7001,22 +6931,22 @@ pub(crate) mod __gated_printer { let newline_index = newline_index as usize; // Skip over \r if it precedes \n if newline_index > 0 && text[newline_index - 1] == b'\r' { - self.print(&text[..newline_index - 1]); + self.print_verbatim(&text[..newline_index - 1]); self.print(b"\n"); } else { - self.print(&text[..newline_index + 1]); + self.print_verbatim(&text[..newline_index + 1]); } self.print_indent(); text = &text[newline_index + 1..]; } - self.print(text); + self.print_verbatim(text); self.print_newline(); } else { // Print a mandatory newline after single-line comments if !text.is_empty() && text[text.len() - 1] == b'\r' { text = &text[..text.len() - 1]; } - self.print(text); + self.print_verbatim(text); self.print(b"\n"); } } @@ -7284,6 +7214,11 @@ impl HasDefaultValue for js_ast::ArrayBinding { pub trait WriterContext { fn write_byte(&mut self, char: u8); fn write_all(&mut self, buf: &[u8]); + /// Returns true when an 8-bit context took U+2028 or U+2029, which [`WriterContext::slice`] cannot show. + fn write_verbatim_utf8(&mut self, text: &[u8]) -> bool; + fn output_encoding(&self) -> OutputEncoding; + /// Put the buffer into its final encoding. Nothing is printed after this. + fn resolve_encoding(&mut self); fn get_last_byte(&self) -> u8; fn get_last_last_byte(&self) -> u8; fn reserve_next(&mut self, count: u64) -> *mut u8; @@ -7300,6 +7235,11 @@ pub trait WriterTrait { fn prev_prev_char(&self) -> u8; fn print_byte(&mut self, b: u8); fn print_slice(&mut self, s: &[u8]); + /// See [`WriterContext::write_verbatim_utf8`]. + fn print_verbatim_utf8(&mut self, text: &[u8]) -> bool; + fn output_encoding(&self) -> OutputEncoding; + /// See [`WriterContext::resolve_encoding`]. + fn resolve_encoding(&mut self); fn reserve(&mut self, count: u64) -> *mut u8; fn advance(&mut self, count: u64); /// Reserve `bytes.len()`, memcpy `bytes` into the reserved region, then advance. @@ -7376,6 +7316,10 @@ impl Writer { self.ctx.write_all(s); } + pub(crate) fn print_verbatim_utf8(&mut self, text: &[u8]) -> bool { + self.ctx.write_verbatim_utf8(text) + } + pub(crate) fn done(&mut self) { self.ctx.done() } @@ -7404,6 +7348,18 @@ impl WriterTrait for Writer { self.print_slice(s) } #[inline] + fn print_verbatim_utf8(&mut self, text: &[u8]) -> bool { + self.print_verbatim_utf8(text) + } + #[inline] + fn output_encoding(&self) -> OutputEncoding { + self.ctx.output_encoding() + } + #[inline] + fn resolve_encoding(&mut self) { + self.ctx.resolve_encoding() + } + #[inline] fn reserve(&mut self, count: u64) -> *mut u8 { self.reserve(count) } @@ -7448,6 +7404,18 @@ impl WriterTrait for &mut W { (**self).print_slice(s) } #[inline] + fn print_verbatim_utf8(&mut self, text: &[u8]) -> bool { + (**self).print_verbatim_utf8(text) + } + #[inline] + fn output_encoding(&self) -> OutputEncoding { + (**self).output_encoding() + } + #[inline] + fn resolve_encoding(&mut self) { + (**self).resolve_encoding() + } + #[inline] fn reserve(&mut self, count: u64) -> *mut u8 { (**self).reserve(count) } @@ -7473,7 +7441,42 @@ impl WriterTrait for &mut W { // DirectWriter / BufferWriter // ─────────────────────────────────────────────────────────────────────────── +/// How the bytes in a [`BufferWriter`] are to be read. See [`WriterContext`]. +pub type OutputEncoding = strings::EncodingNonAscii; + +/// The native-endian code units of an [`OutputEncoding::Utf16`] buffer. Copied only when it is odd-aligned. +pub fn utf16_units(bytes: &[u8]) -> std::borrow::Cow<'_, [u16]> { + match bytemuck::try_cast_slice::(bytes) { + Ok(units) => std::borrow::Cow::Borrowed(units), + Err(_) => std::borrow::Cow::Owned( + bytes + .as_chunks::<2>() + .0 + .iter() + .map(|&pair| u16::from_ne_bytes(pair)) + .collect(), + ), + } +} + +/// A writer's output as a JSC string of the width it was printed in. +pub fn clone_as_string(bytes: &[u8], encoding: OutputEncoding) -> bun_core::String { + match encoding { + OutputEncoding::Utf8 => bun_core::String::clone_utf8(bytes), + OutputEncoding::Latin1 => bun_core::String::clone_latin1(bytes), + OutputEncoding::Utf16 => bun_core::String::clone_utf16_wide(&utf16_units(bytes)), + } +} + pub struct BufferWriter { + /// `Latin1` is one byte per code unit. [`Self::resolve_encoding`] can turn it into `Utf16` ([`utf16_units`]). + encoding: OutputEncoding, + /// What [`Self::reset`] returns `encoding` to. + initial_encoding: OutputEncoding, + /// The code units above U+00FF of a `Latin1` writer, in buffer order. The buffer has a placeholder byte for each. + wide_units: Vec, + /// `(offset of its first placeholder, length)` of each run in `wide_units`. + wide_runs: Vec<(u32, u32)>, pub buffer: MutableString, /// Watermark into `buffer.list` set by `done()`. Rust can't keep a /// self-borrowing slice in a field, so store the length and @@ -7498,11 +7501,12 @@ impl BufferWriter { } pub fn init() -> BufferWriter { - BufferWriter { - buffer: MutableString::init_empty(), - written_len: 0, - append_newline: false, - } + Self::with_capacity_and_encoding(0, OutputEncoding::Utf8) + } + + /// A writer whose output JSC takes as it is: Latin-1, or UTF-16 when the text needs it. + pub fn init_latin1() -> BufferWriter { + Self::with_capacity_and_encoding(0, OutputEncoding::Latin1) } /// Like [`init`], but pre-sizes the output buffer. The transpiled output is @@ -7511,23 +7515,193 @@ impl BufferWriter { /// otherwise do as the printer appends token-by-token. (`MutableString::init` /// is a no-op when `capacity == 0`.) pub(crate) fn with_capacity(capacity: usize) -> BufferWriter { + Self::with_capacity_and_encoding(capacity, OutputEncoding::Utf8) + } + + fn with_capacity_and_encoding(capacity: usize, encoding: OutputEncoding) -> BufferWriter { + debug_assert!(encoding != OutputEncoding::Utf16); BufferWriter { + encoding, + initial_encoding: encoding, + wide_units: Vec::new(), + wide_runs: Vec::new(), buffer: MutableString::init(capacity).unwrap_or_else(|_| MutableString::init_empty()), written_len: 0, append_newline: false, } } + pub fn output_encoding(&self) -> OutputEncoding { + self.encoding + } + + /// The finished text as a JSC string, in the width it was printed in. + pub fn clone_written_as_string(&self) -> bun_core::String { + debug_assert!(self.wide_runs.is_empty()); + clone_as_string(self.get_written(), self.encoding) + } + #[inline] pub(crate) fn write_byte(&mut self, byte: u8) { + debug_assert!( + self.encoding == OutputEncoding::Utf8 + || (self.encoding == OutputEncoding::Latin1 && byte < 0x80) + ); self.buffer.list.push(byte); } #[inline] pub(crate) fn write_all(&mut self, bytes: &[u8]) { + #[cfg(debug_assertions)] + if self.encoding != OutputEncoding::Utf8 { + self.assert_accepts_printed_bytes(bytes); + } self.buffer.list.extend_from_slice(bytes); } + /// Only `write_verbatim_utf8` stores non-ASCII in a Latin-1 buffer. Nothing is printed into a UTF-16 buffer. + #[cfg(debug_assertions)] + #[inline(never)] // Inlined, the message's temporaries take stack in each frame of the printer. + fn assert_accepts_printed_bytes(&self, bytes: &[u8]) { + assert!( + self.encoding == OutputEncoding::Latin1 && strings::is_all_ascii(bytes), + "printed into a {:?} buffer: {:?}", + self.encoding, + bstr::BStr::new(bytes) + ); + } + + /// Ill-formed bytes become the lexer's decode of them. A lone surrogate is an escape in UTF-8, a unit in Latin-1. + pub fn write_verbatim_utf8(&mut self, text: &[u8]) -> bool { + if self.encoding == OutputEncoding::Utf8 && strings::is_valid_utf8(text) { + self.buffer.list.extend_from_slice(text); + return false; + } + debug_assert!(self.encoding != OutputEncoding::Utf16); + let mut wide_line_terminator = false; + // Start of the pending run of ASCII bytes, copied in one go. + let mut run_start = 0usize; + let iter = CodepointIterator::init(text); + let mut cursor = strings::Cursor::default(); + while iter.next(&mut cursor) { + let c = cursor.c as u32; + if c < 0x80 { + continue; + } + let start = cursor.i as usize; + if start > run_start { + self.buffer.list.extend_from_slice(&text[run_start..start]); + } + run_start = start + cursor.width as usize; + if self.encoding == OutputEncoding::Utf8 { + if (FIRST_HIGH_SURROGATE..=LAST_LOW_SURROGATE).contains(&c) { + // After an odd number of backslashes, the last one escapes it and starts the `\u`. + let backslashes = text[..start].iter().rev().take_while(|&&b| b == b'\\'); + self.buffer + .list + .extend_from_slice(&bmp_escape(c)[backslashes.count() % 2..]); + } else { + let mut utf8 = [0u8; 4]; + let len = bun_core::encode_wtf8_rune(&mut utf8, c); + self.buffer.list.extend_from_slice(&utf8[..len]); + } + } else if c <= 0xFF { + self.buffer.list.push(c as u8); + } else if c <= 0xFFFF { + wide_line_terminator |= c == 0x2028 || c == 0x2029; + self.push_wide_unit(c as u16); + } else { + let [lead, trail] = strings::encode_surrogate_pair(c); + self.push_wide_unit(lead); + self.push_wide_unit(trail); + } + } + if text.len() > run_start { + self.buffer.list.extend_from_slice(&text[run_start..]); + } + wide_line_terminator + } + + /// `unit` is above U+00FF. The placeholder reads as a letter to [`Self::get_last_byte`]. + fn push_wide_unit(&mut self, unit: u16) { + let offset = self.buffer.list.len() as u32; + match self.wide_runs.last_mut() { + Some((start, len)) if *start + *len == offset => *len += 1, + _ => self.wide_runs.push((offset, 1)), + } + self.wide_units.push(unit); + self.buffer.list.push(0xFF); + } + + /// Call when the print is over. Only [`Self::append_ascii`] may add to the buffer after it. + #[inline] + pub fn resolve_encoding(&mut self) { + if !self.wide_runs.is_empty() { + self.widen_to_utf16(); + } + } + + /// In place, from the end: the bytes of unit `i` (`2i`, `2i + 1`) never overlap a byte `j < i` still to be read. + #[cold] + #[inline(never)] + fn widen_to_utf16(&mut self) { + debug_assert!(self.encoding == OutputEncoding::Latin1); + let list = &mut self.buffer.list; + let len = list.len(); + list.reserve(len); + let ptr = list.as_mut_ptr(); + for i in (0..len).rev() { + // SAFETY: `i < len` is initialized, `2i + 1 < 2 * len <= capacity`, and the write is unaligned. + unsafe { + let unit = u16::from(*ptr.add(i)); + ptr.add(2 * i).cast::().write_unaligned(unit); + } + } + let mut units = self.wide_units.iter(); + for &(start, run_len) in &self.wide_runs { + for (i, &unit) in (start as usize..).zip(units.by_ref().take(run_len as usize)) { + debug_assert!(i < len); + // SAFETY: each run is a range of placeholder offsets below `len`. + unsafe { ptr.add(2 * i).cast::().write_unaligned(unit) }; + } + } + // SAFETY: every byte in `0..2 * len` was written by the first loop. + unsafe { list.set_len(2 * len) }; + self.encoding = OutputEncoding::Utf16; + self.wide_units.clear(); + self.wide_runs.clear(); + } + + /// Add ASCII to a buffer whose encoding is final. + pub fn append_ascii(&mut self, bytes: &[u8]) { + debug_assert!(self.wide_runs.is_empty() && strings::is_all_ascii(bytes)); + if self.encoding != OutputEncoding::Utf16 { + self.buffer.list.extend_from_slice(bytes); + return; + } + self.buffer.list.reserve(2 * bytes.len()); + for &byte in bytes { + self.buffer + .list + .extend_from_slice(&u16::from(byte).to_ne_bytes()); + } + } + + /// [`Self::append_ascii`] for text that `fill` writes into `capacity` bytes. `fill` returns the count. + pub fn append_ascii_with(&mut self, capacity: usize, fill: impl FnOnce(&mut [u8]) -> usize) { + if self.encoding == OutputEncoding::Utf16 { + let mut bytes = vec![0u8; capacity]; + let wrote = fill(&mut bytes); + return self.append_ascii(&bytes[..wrote]); + } + debug_assert!(self.wide_runs.is_empty()); + let list = &mut self.buffer.list; + let len = list.len(); + list.resize(len + capacity, 0); + let wrote = fill(&mut list[len..]); + list.truncate(len + wrote); + } + #[inline] pub(crate) fn slice(&self) -> &[u8] { self.buffer.list.as_slice() @@ -7564,6 +7738,9 @@ impl BufferWriter { pub fn reset(&mut self) { self.buffer.reset(); self.written_len = 0; + self.encoding = self.initial_encoding; + self.wide_units.clear(); + self.wide_runs.clear(); } pub fn written_without_trailing_zero(&self) -> &[u8] { @@ -7575,9 +7752,10 @@ impl BufferWriter { } pub(crate) fn done(&mut self) { + self.resolve_encoding(); if self.append_newline { self.append_newline = false; - self.buffer.list.push(b'\n'); + self.append_ascii(b"\n"); } self.written_len = self.buffer.list.len(); } @@ -7592,6 +7770,17 @@ impl WriterContext for BufferWriter { fn write_all(&mut self, buf: &[u8]) { self.write_all(buf) } + fn write_verbatim_utf8(&mut self, text: &[u8]) -> bool { + self.write_verbatim_utf8(text) + } + #[inline] + fn output_encoding(&self) -> OutputEncoding { + self.output_encoding() + } + #[inline] + fn resolve_encoding(&mut self) { + self.resolve_encoding() + } #[inline] fn get_last_byte(&self) -> u8 { self.get_last_byte() @@ -7844,12 +8033,13 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR writer.reserve(source.contents().len() as u64); let mut opts = opts; - let source_map_builder = get_source_map_builder::( + let mut source_map_builder = get_source_map_builder::( GenerateSourceMap::lazy_if(GENERATE_SOURCE_MAP), &mut opts, source, tree, ); + source_map_builder.output_encoding = writer.output_encoding(); let mut printer = PrinterType::::init( writer, bump, @@ -7923,6 +8113,9 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR None }; + // After the source map builder, before the first consumer. + printer.writer.resolve_encoding(); + if let Some(cache) = printer.options.runtime_transpiler_cache { let mut srlz_res: Vec = Vec::new(); if have_module_info { @@ -7937,6 +8130,7 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR // SAFETY: caller guarantees the cache outlives the print call. unsafe { &mut *cache.as_ptr() }.put( printer.writer.slice(), + printer.writer.output_encoding(), source_maps_chunk .as_ref() .map(|c| c.buffer.list.as_slice()) @@ -8086,6 +8280,8 @@ pub(crate) fn print_with_writer_and_platform< Printer<'a, W, /*ASCII_ONLY=*/ B, B, false, G>; let module_type = opts.module_type; let mut opts = opts; + // The chunk is generated after `done()`, which a `Latin1` writer may widen in. + debug_assert!(writer.output_encoding() == OutputEncoding::Utf8); let source_map_builder = get_source_map_builder::( GenerateSourceMap::eager_if(GENERATE_SOURCE_MAPS), &mut opts, diff --git a/src/jsc/AsyncModule.rs b/src/jsc/AsyncModule.rs index 3943e1dc8703..eb47d92b8989 100644 --- a/src/jsc/AsyncModule.rs +++ b/src/jsc/AsyncModule.rs @@ -1088,11 +1088,11 @@ impl AsyncModule { // SAFETY: thread-local owns the leaked Box; only this thread touches it. let mut printer = core::mem::replace( unsafe { printer_ptr.as_mut() }, - bun_js_printer::BufferPrinter::init(bun_js_printer::BufferWriter::init()), + bun_js_printer::BufferPrinter::init(bun_js_printer::BufferWriter::init_latin1()), ); printer.ctx.reset(); // The writeback must fire at fn exit, - // *after* the `printer.ctx.get_written()` reads below. Declare the + // *after* the reads of `printer.ctx` below. Declare the // guard immediately after `printer` so it drops last (locals drop in // reverse declaration order) and the buffer is still populated when // read. @@ -1104,7 +1104,9 @@ impl AsyncModule { unsafe { *dst = core::mem::replace( &mut *src, - bun_js_printer::BufferPrinter::init(bun_js_printer::BufferWriter::init()), + bun_js_printer::BufferPrinter::init( + bun_js_printer::BufferWriter::init_latin1(), + ), ) }; }); @@ -1136,11 +1138,11 @@ impl AsyncModule { // `cfg(feature = "dump_source")` gate referenced a feature that doesn't // exist, which silently compiled this call out everywhere. if bun_core::env::DUMP_SOURCE { - crate::runtime_transpiler_store::dump_source_string( + crate::runtime_transpiler_store::dump_source( // SAFETY: `jsc_vm` is the live per-thread `VirtualMachine` (BACKREF, non-null). unsafe { core::ptr::NonNull::new_unchecked(jsc_vm) }, specifier, - printer.ctx.get_written(), + &printer, ); } @@ -1149,7 +1151,7 @@ impl AsyncModule { // the number recycled) by the transpile frame's fd guard. Ok(ResolvedSource { - source_code: BunString::clone_latin1(printer.ctx.get_written()), + source_code: printer.ctx.clone_written_as_string(), source_url: BunString::from_bytes(path.text), is_commonjs_module, is_static: self.is_static, diff --git a/src/jsc/NodeCompileCache.rs b/src/jsc/NodeCompileCache.rs index 40577936b363..74641beb5c36 100644 --- a/src/jsc/NodeCompileCache.rs +++ b/src/jsc/NodeCompileCache.rs @@ -8,6 +8,7 @@ use bstr::ByteSlice; use bun_boringssl::c as boring; use bun_collections::{HashMap, IdentityContext}; use bun_core::String as BunString; +use bun_core::strings::EncodingNonAscii; use bun_core::{Mutex, ZStr, env_var}; use bun_options_types::Format; use bun_paths::{MAX_PATH_BYTES, SEP}; @@ -53,6 +54,8 @@ struct Entry { /// Post-transpile text; `None` when the module never transpiled /// successfully (parse error) — mirrors Node's "not initialized" state. code: Option>, + /// Width of `code`. The bytecode generator builds the string that the loader built. + code_encoding: EncodingNonAscii, /// Deserialized bytecode blob handed to JSC (the cache was accepted). /// Kept alive for the process — `ZigSourceProvider` wraps it, no copy. blob: Option, @@ -553,11 +556,12 @@ pub fn get_dir() -> Option> { /// Module-fetch hook: register/refresh the entry for `filename`; returns the /// validated bytecode blob when the on-disk cache matches `code` (post- -/// transpile text). The pointer stays valid for the process (entry map owns it). +/// transpile text in the width `code_encoding`). The pointer stays valid for the process (entry map owns it). pub fn fetch( filename: &[u8], is_cjs: bool, code: &[u8], + code_encoding: EncodingNonAscii, ) -> Option { if !is_enabled() || filename.is_empty() || !bun_paths::is_absolute(filename) { return None; @@ -587,6 +591,7 @@ pub fn fetch( code_hash, code_size, code: None, + code_encoding, blob: None, persisted: false, }; @@ -640,6 +645,7 @@ pub fn note_parse_failure(filename: &[u8], is_cjs: bool) { code_hash: [0u8; HASH_SIZE], code_size: 0, code: None, + code_encoding: EncodingNonAscii::Latin1, blob: None, persisted: false, }; @@ -893,11 +899,17 @@ fn read_cache_file(state: &CacheState, key: u64, entry: &mut Entry, code: Option struct GenJob { format: Format, code: Box<[u8]>, + code_encoding: EncodingNonAscii, url: Box<[u8]>, resp: std::sync::mpsc::SyncSender>>, } -fn generate_bytecode(format: Format, code: &[u8], url: &[u8]) -> Option> { +fn generate_bytecode( + format: Format, + code: &[u8], + code_encoding: EncodingNonAscii, + url: &[u8], +) -> Option> { use std::sync::mpsc; static WORKER: Mutex>> = Mutex::new(None); @@ -913,8 +925,8 @@ fn generate_bytecode(format: Format, code: &[u8], url: &[u8]) -> Option Option, + code_encoding: EncodingNonAscii, filename: Box<[u8]>, is_cjs: bool, code_size: u32, @@ -998,6 +1012,7 @@ fn collect_persist_jobs(state: &mut CacheState) -> Vec { Format::Esm }, code, + code_encoding: entry.code_encoding, filename: entry.filename.clone(), is_cjs: entry.is_cjs, code_size: entry.code_size, @@ -1123,7 +1138,7 @@ fn persist_pass() { // Phase 2: generate bytecode, unlocked. let mut generated: Vec<(PersistJob, Option>)> = Vec::with_capacity(jobs.len()); for job in jobs { - let blob = generate_bytecode(job.format, &job.code, &job.filename); + let blob = generate_bytecode(job.format, &job.code, job.code_encoding, &job.filename); if blob.is_none() { cclog!( "[compile cache] generating cache for {} {} failed, skipping\n", diff --git a/src/jsc/RefString.rs b/src/jsc/RefString.rs index 3f63e3019f05..ee986a43f653 100644 --- a/src/jsc/RefString.rs +++ b/src/jsc/RefString.rs @@ -23,9 +23,16 @@ pub struct StringTooLong; pub type Callback = unsafe fn(ctx: *mut c_void, str: *mut RefString); +/// The interned text, owned, in the width the external string reads it in. +pub(crate) enum Buffer { + /// 8-bit characters; a `Box<[u8]>` of `len`. + Latin1 { ptr: *const u8, len: usize }, + /// Native-endian code units; a `Box<[u16]>` of `len`, so they are aligned. + Utf16 { ptr: *const u16, len: usize }, +} + pub struct RefString { - pub ptr: *const u8, - pub(crate) len: usize, + pub(crate) buffer: Buffer, pub(crate) hash: Hash, // `impl` is a Rust keyword — renamed to `impl_`. pub(crate) impl_: WTFStringImpl, @@ -38,8 +45,9 @@ pub struct RefString { } impl RefString { - pub(crate) fn compute_hash(input: &[u8]) -> u32 { - bun_hash::XxHash32::hash(0, input) + /// The map is keyed by this hash alone, so the width is in the seed. + pub(crate) fn compute_hash(input: &[u8], utf16: bool) -> u32 { + bun_hash::XxHash32::hash(u32::from(utf16), input) } /// Single audited deref of the set-once `impl_` backref so `ref_` / @@ -57,10 +65,17 @@ impl RefString { self.wtf_impl().r#ref(); } + /// The text as bytes. Its readers intern 8-bit strings only. pub fn leak(&self) -> &[u8] { - // SAFETY: `ptr` points to a live allocation of `len` bytes for the - // lifetime of `self` (freed only in `destroy`). - unsafe { bun_core::ffi::slice(self.ptr, self.len) } + // SAFETY: `buffer` is a live allocation of `len` elements until `destroy`. + unsafe { + match self.buffer { + Buffer::Latin1 { ptr, len } => bun_core::ffi::slice(ptr, len), + Buffer::Utf16 { ptr, len } => { + bun_core::slice_as_bytes(bun_core::ffi::slice(ptr, len)) + } + } + } } pub fn deref(&self) { @@ -79,9 +94,9 @@ impl RefString { /// call `this` is dangling. pub(crate) unsafe fn destroy(this: *mut RefString) { // SAFETY: caller contract — `this` is the unique live pointer to a - // `Box`-allocated value whose `ptr`/`len` describe a - // `Box<[u8]>`-allocated buffer. All raw derefs and `from_raw` calls - // below operate on those owned allocations. + // `Box`-allocated value whose `buffer` describes a boxed + // slice of the matching element type. All raw derefs and `from_raw` + // calls below operate on those owned allocations. unsafe { if let Some(on_before_deinit) = (*this).on_before_deinit { // Caller guarantees `ctx` is set @@ -94,10 +109,14 @@ impl RefString { // materialized — the WTF::StringImpl finalizer may still hold a // shared view at this instant, so forming `&mut [u8]` would assert // exclusivity we cannot prove). - drop(bun_core::heap::take(core::ptr::slice_from_raw_parts_mut( - (*this).ptr.cast_mut(), - (*this).len, - ))); + match (*this).buffer { + Buffer::Latin1 { ptr, len } => drop(bun_core::heap::take( + core::ptr::slice_from_raw_parts_mut(ptr.cast_mut(), len), + )), + Buffer::Utf16 { ptr, len } => drop(bun_core::heap::take( + core::ptr::slice_from_raw_parts_mut(ptr.cast_mut(), len), + )), + } // `allocator.destroy(this)` drop(bun_core::heap::take(this)); } diff --git a/src/jsc/RuntimeTranspilerCache.rs b/src/jsc/RuntimeTranspilerCache.rs index 8d40a1bcfd28..3df7b8018bff 100644 --- a/src/jsc/RuntimeTranspilerCache.rs +++ b/src/jsc/RuntimeTranspilerCache.rs @@ -5,6 +5,7 @@ use core::sync::atomic::{AtomicBool, Ordering}; use bun_ast::ExportsKind; use bun_ast::Source; +use bun_core::strings::EncodingNonAscii; use bun_core::{FeatureFlags, env_var}; use bun_core::{String as BunString, ZStr}; use bun_js_parser::ParserOptions; @@ -77,7 +78,8 @@ bun_core::declare_scope!(cache, visible); /// Version 42: The body of a class with lowered class decorators names the class by the /// binding that holds what the decorators return. /// Version 43: A `bun:` specifier that names no builtin is printed as it is written. -const EXPECTED_VERSION: u32 = 43; +/// Version 44: Regex and raw template text is printed as written. Output is LATIN1 or UTF16, never UTF8. +const EXPECTED_VERSION: u32 = 44; /// Source files smaller than this are not written to / read from the on-disk /// transpiler cache. Originally 50 KiB, which excluded almost every file in a @@ -115,7 +117,7 @@ pub struct Encoding(u8); impl Encoding { pub(crate) const NONE: Encoding = Encoding(0); - pub(crate) const UTF8: Encoding = Encoding(1); + // 1 was UTF8, before version 44. pub(crate) const UTF16: Encoding = Encoding(2); pub(crate) const LATIN1: Encoding = Encoding(3); } @@ -169,6 +171,16 @@ impl Default for Metadata { } impl Metadata { + /// The width of the entry's output. `decode` rejects every other tag. + pub fn output_width(&self) -> EncodingNonAscii { + if self.output_encoding == Encoding::UTF16 { + EncodingNonAscii::Utf16 + } else { + debug_assert!(self.output_encoding == Encoding::LATIN1); + EncodingNonAscii::Latin1 + } + } + // 1×u32 + 2×u8 (enum reprs) + 12×u64 = 4 + 2 + 96 = 102 pub(crate) const SIZE: usize = 4 + 1 + 1 + 12 * 8; @@ -239,7 +251,7 @@ impl Metadata { self.output_encoding = Encoding(output_encoding_raw); match self.output_encoding { - Encoding::UTF8 | Encoding::UTF16 | Encoding::LATIN1 => {} + Encoding::UTF16 | Encoding::LATIN1 => {} // Invalid encoding _ => return Err(crate::CrateError::UnknownEncoding), } @@ -251,6 +263,7 @@ impl Metadata { #[derive(Default)] pub struct Entry { pub metadata: Metadata, + /// 8-bit for a `LATIN1` entry, 16-bit for a `UTF16` entry. Empty until `load`. pub output_code: BunString, pub sourcemap: Box<[u8]>, pub esm_record: Box<[u8]>, @@ -265,7 +278,8 @@ impl Entry { features_hash: u64, sourcemap: &[u8], esm_record: &[u8], - output_code: &BunString, + output_bytes: &[u8], + output_encoding: Encoding, exports_kind: ExportsKind, ) -> crate::CrateResult<()> { let _tracer = bun_core::perf::trace("RuntimeTranspilerCache.save"); @@ -281,8 +295,6 @@ impl Entry { // it for the errdefer `unlinkat` below. let tmpfilename: &ZStr = &*tmpfilename; - let output_bytes = output_code.byte_slice(); - // First we open the tmpfile, to avoid any other work in the event of failure. let mut tmpfile = sys::Tmpfile::create(destination_dir, tmpfilename)?; let _close_guard = sys::CloseOnDrop::new(tmpfile.fd); @@ -301,13 +313,7 @@ impl Entry { ExportsKind::Cjs => ModuleType::Cjs, _ => ModuleType::Esm, }, - output_encoding: if output_code.is_utf16() { - Encoding::UTF16 - } else if output_code.is_utf8() { - Encoding::UTF8 - } else { - Encoding::LATIN1 - }, + output_encoding, sourcemap_byte_length: sourcemap.len() as u64, output_byte_offset: Metadata::SIZE as u64, output_byte_length: output_bytes.len() as u64, @@ -422,51 +428,6 @@ impl Entry { BunString::EMPTY } else { match self.metadata.output_encoding { - Encoding::UTF8 => { - // PERF: a per-call arena (`output_code_allocator`) here - // would let the ~1.2 MB scratch buffer - // be bump-freed with the parse arena. Instead this - // `pread_box`'s into a `Box<[u8]>` - // on the worker thread's mimalloc heap, which — even after - // the consumer's `String::clone_utf8` + drop — leaves the - // segment resident in that thread heap (build/create-vue - // bench regression). - // - // Instead, pread straight into a WTF-allocated Latin-1 - // buffer (`WTF::StringImpl::tryCreateUninitialized` → - // bmalloc, not the worker mimalloc heap). Transpiler - // output is overwhelmingly pure ASCII, in which case the - // buffer *is* the final `BunString` and we skip the - // 1.2 MB `clone_utf8` memcpy the consumer used to do at - // RuntimeTranspilerStore.rs / jsc_hooks.rs. Only if the - // bytes contain non-ASCII UTF-8 do we fall back to - // `clone_utf8` (transcode → UTF-16) and deref the scratch. - let len = self.metadata.output_byte_length as usize; - let (scratch, bytes) = BunString::create_uninitialized_latin1(len); - // `(dead, &mut [])` on WTF allocation failure; `len > 0` - // (handled above), so an empty slice means OOM. - if bytes.is_empty() { - return Err(crate::CrateError::Alloc(bun_alloc::AllocError)); - } - let read_bytes = file.pread_all(bytes, self.metadata.output_byte_offset)?; - if read_bytes as u64 != self.metadata.output_byte_length { - return Err(crate::CrateError::MissingData); - } - - if self.metadata.output_hash != 0 && hash(bytes) != self.metadata.output_hash { - return Err(crate::CrateError::InvalidHash); - } - - if bun_core::strings::is_all_ascii(bytes) { - // Fast path: ASCII ⊂ Latin-1, so `scratch` is already - // the correct `BunString`. - scratch - } else { - // Rare path: real multi-byte UTF-8. Transcode into a - // fresh WTF string; the Latin-1 scratch drops. - BunString::clone_utf8(bytes) - } - } Encoding::LATIN1 => { let len = self.metadata.output_byte_length as usize; let (latin1, bytes) = BunString::create_uninitialized_latin1(len); @@ -814,7 +775,8 @@ impl RuntimeTranspilerCache { features_hash: u64, sourcemap: &[u8], esm_record: &[u8], - source_code: &BunString, + output_bytes: &[u8], + output_encoding: Encoding, exports_kind: ExportsKind, ) -> crate::CrateResult<()> { let _tracer = bun_core::perf::trace("RuntimeTranspilerCache.toFile"); @@ -862,7 +824,8 @@ impl RuntimeTranspilerCache { features_hash, sourcemap, esm_record, - source_code, + output_bytes, + output_encoding, exports_kind, ) } @@ -992,7 +955,7 @@ bun_ast::link_impl_TranspilerCacheImpl! { } hit }, - put(output_code_bytes, sourcemap, esm_record) => { + put(output_code_bytes, output_encoding, sourcemap, esm_record) => { let this = &mut *this; if this.input_hash.is_none() || IS_DISABLED.load(Ordering::Relaxed) { return; @@ -1003,17 +966,24 @@ bun_ast::link_impl_TranspilerCacheImpl! { } debug_assert!(this.entry.is_none()); - // Borrowed Latin-1 view: `to_file` only reads `byte_slice()` + the encoding - // tag (unmarked 8-bit EncodedSlice -> Encoding::LATIN1, same as clone_latin1), - // and `output_code_bytes` outlives the synchronous `to_file` call. - let output_code = BunString::ascii(output_code_bytes); + // `Entry::load` reads the entry back in this width. + let tag = match output_encoding { + EncodingNonAscii::Latin1 => Encoding::LATIN1, + EncodingNonAscii::Utf16 => Encoding::UTF16, + // Only the runtime's Latin-1/UTF-16 printers carry a cache. + EncodingNonAscii::Utf8 => { + debug_assert!(false, "transpiler cache fed UTF-8 printer output"); + return; + } + }; let result = RuntimeTranspilerCache::to_file( this.input_byte_length.unwrap(), this.input_hash.unwrap(), this.features_hash.unwrap(), sourcemap, esm_record, - &output_code, + output_code_bytes, + tag, this.exports_kind, ); if let Err(err) = result { diff --git a/src/jsc/RuntimeTranspilerStore.rs b/src/jsc/RuntimeTranspilerStore.rs index aed8a4cce083..ac758e9da243 100644 --- a/src/jsc/RuntimeTranspilerStore.rs +++ b/src/jsc/RuntimeTranspilerStore.rs @@ -60,11 +60,20 @@ bun_core::declare_scope!(RuntimeTranspilerStore, hidden); // the caller's `&mut TranspilerJob` (which is stored inside // `vm.transpiler_store`). Only the `source_mappings` leaf field is touched, // under its own internal lock. -fn dump_source(vm: NonNull, specifier: &[u8], printer: &BufferPrinter) { - dump_source_string(vm, specifier, printer.ctx.get_written()); +pub(crate) fn dump_source(vm: NonNull, specifier: &[u8], printer: &BufferPrinter) { + dump_source_code(vm, specifier, &printer.ctx.clone_written_as_string()); } -pub(crate) fn dump_source_string(vm: NonNull, specifier: &[u8], written: &[u8]) { +/// Written as UTF-8, to be read. +pub(crate) fn dump_source_code( + vm: NonNull, + specifier: &[u8], + source_code: &String, +) { + dump_source_string(vm, specifier, source_code.to_utf8().slice()); +} + +fn dump_source_string(vm: NonNull, specifier: &[u8], written: &[u8]) { if let Err(e) = dump_source_string_failiable(vm, specifier, written) { bun_core::debug_warn!("Failed to dump source string: {}", e.name()); } @@ -976,7 +985,7 @@ impl TranspilerJob { if bun_core::env::DUMP_SOURCE { // SAFETY: `vm` is the live owning VM (BACKREF — see `vm` note above). let vm = unsafe { NonNull::new_unchecked(vm) }; - dump_source_string(vm, specifier, entry.output_code.byte_slice()); + dump_source_code(vm, specifier, &entry.output_code); } let module_info = if use_isolation_source_provider_cache @@ -1045,7 +1054,7 @@ impl TranspilerJob { } let source_code_printer = tls_get_or_leak(&SOURCE_CODE_PRINTER, || { - let writer = BufferWriter::init(); + let writer = BufferWriter::init_latin1(); Box::new(BufferPrinter::init(writer)) }); @@ -1053,7 +1062,7 @@ impl TranspilerJob { // _writeback guard (the thread-local's buffer is reused). let mut printer = core::mem::replace( source_code_printer, - BufferPrinter::init(BufferWriter::init()), + BufferPrinter::init(BufferWriter::init_latin1()), ); printer.ctx.reset(); @@ -1061,11 +1070,11 @@ impl TranspilerJob { const MAX_BUFFER_CAP: usize = 512 * 1024; if printer.ctx.buffer.list.capacity() > MAX_BUFFER_CAP { // printer.ctx.buffer.deinit() → Drop - let writer = BufferWriter::init(); + let writer = BufferWriter::init_latin1(); *source_code_printer = BufferPrinter::init(writer); printer = core::mem::replace( source_code_printer, - BufferPrinter::init(BufferWriter::init()), + BufferPrinter::init(BufferWriter::init_latin1()), ); } @@ -1111,8 +1120,10 @@ impl TranspilerJob { |(dst, src)| { // SAFETY: both pointees outlive this scope; no aliases at drop. unsafe { - *dst = - core::mem::replace(&mut *src, BufferPrinter::init(BufferWriter::init())) + *dst = core::mem::replace( + &mut *src, + BufferPrinter::init(BufferWriter::init_latin1()), + ) }; }, ); @@ -1146,12 +1157,16 @@ impl TranspilerJob { // `cache.output_code` (only the `r#impl == None` fallback does, // and `r#impl` is `Some(Jsc)` here), so it is always `None`. debug_assert!(cache.output_code.is_none()); - let result = String::clone_latin1(written); + // A copy in the printed width, not a transcode. + debug_assert!( + source_code_printer.ctx.output_encoding() != strings::EncodingNonAscii::Utf8 + ); + let result = source_code_printer.ctx.clone_written_as_string(); // SAFETY: leaf scalar field read on `*vm`; see `vm` note above. if written.len() > 1024 * 1024 * 2 || unsafe { (*vm).smol } { // printer.ctx.buffer.deinit() → Drop - let writer = BufferWriter::init(); + let writer = BufferWriter::init_latin1(); *source_code_printer = BufferPrinter::init(writer); } // else: writeback guard already restored `printer` into the thread-local. diff --git a/src/jsc/VirtualMachine.rs b/src/jsc/VirtualMachine.rs index 869f8eb4b873..0c83072c0c69 100644 --- a/src/jsc/VirtualMachine.rs +++ b/src/jsc/VirtualMachine.rs @@ -8,6 +8,7 @@ use core::ffi::{c_int, c_void}; use core::ptr::NonNull; use bun_bundler::Transpiler; +use bun_core::strings::EncodingNonAscii; use bun_io as Async; use bun_uws as uws; @@ -3957,11 +3958,6 @@ impl<'a> bun_js_printer::OnSourceMapChunk for SourceMapHandlerGetter<'a> { .map_err(|_| bun_js_printer::Error::WriteFailed)?; const SOURCE_MAP_URL_PREFIX_START: &[u8] = b"//# sourceMappingURL=data:application/json;base64,"; - // TODO: do we need to %-encode the path? - let source_url_len = source.path.text.len(); - const SOURCE_MAPPING_URL: &[u8] = b"\n//# sourceURL="; - let prefix_len = - SOURCE_MAP_URL_PREFIX_START.len() + SOURCE_MAPPING_URL.len() + source_url_len; self.vm_source_mappings_mut() .put_mappings(source, chunk.buffer) @@ -3979,39 +3975,16 @@ impl<'a> bun_js_printer::OnSourceMapChunk for SourceMapHandlerGetter<'a> { // `SourceMapHandlerGetter` doc for why `printer` is not stored as `&'a mut`. let printer = unsafe { &mut *self.printer }; - let encode_len = bun_base64::encode_len(temp_json_buffer.list.as_slice()); + // No `//# sourceURL=`: the module's URL is the provider's. + let source_map_json = temp_json_buffer.list.as_slice(); + printer.ctx.append_ascii(b"\n"); + printer.ctx.append_ascii(SOURCE_MAP_URL_PREFIX_START); printer .ctx - .buffer - .grow_if_needed(encode_len + prefix_len + 2)?; - printer.ctx.buffer.append_assume_capacity(b"\n"); - printer - .ctx - .buffer - .append_assume_capacity(SOURCE_MAP_URL_PREFIX_START); - { - // `MutableString::list` is a `Vec`; write into spare capacity, - // then commit the written length. - let buf = &mut printer.ctx.buffer.list; - // SAFETY: `grow_if_needed` reserved ≥encode_len spare; encode writes - // `wrote<=encode_len` bytes. - let wrote = unsafe { - bun_base64::encode( - &mut bun_core::vec::spare_bytes_mut(buf)[..encode_len], - temp_json_buffer.list.as_slice(), - ) - }; - // SAFETY: `wrote <= encode_len` bytes were just initialized in the - // spare capacity reserved by `grow_if_needed` above. - unsafe { bun_core::vec::commit_spare(buf, wrote) }; - } - printer - .ctx - .buffer - .append_assume_capacity(SOURCE_MAPPING_URL); - // TODO: do we need to %-encode the path? - printer.ctx.buffer.append_assume_capacity(source.path.text); - printer.ctx.buffer.append(b"\n")?; + .append_ascii_with(bun_base64::encode_len(source_map_json), |encoded| { + bun_base64::encode(encoded, source_map_json) + }); + printer.ctx.append_ascii(b"\n"); Ok(()) } } @@ -4151,7 +4124,7 @@ fn specifier_cache_resolver_buf() -> *mut bun_paths::PathBuffer { fn ensure_source_code_printer() { if SOURCE_CODE_PRINTER.get().is_none() { - let writer = bun_js_printer::BufferWriter::init(); + let writer = bun_js_printer::BufferWriter::init_latin1(); let printer = Box::new(bun_js_printer::BufferPrinter::init(writer)); SOURCE_CODE_PRINTER.set(NonNull::new(bun_core::heap::into_raw(printer))); } @@ -5024,9 +4997,11 @@ impl VirtualMachine { } /// Builds a `ResolvedSource` backed by a ref-counted copy of `code` interned in the VM's ref-string map. + /// `code` is the printer's buffer in `encoding`. The interned copy keeps that width. pub fn ref_counted_resolved_source( &mut self, code: &[u8], + encoding: EncodingNonAscii, specifier: &bun_core::String, source_url: &[u8], hash_: Option, @@ -5038,7 +5013,7 @@ impl VirtualMachine { ..Default::default() }); } - let source = self.ref_counted_string(code, hash_)?; + let source = self.ref_counted_string(code, encoding, hash_)?; // SAFETY: `ref_counted_string` returns a live `RefString` held in // `self.ref_strings` until JSC calls the external-string finalizer. let source_ref = unsafe { source.as_ref() }; @@ -5051,16 +5026,20 @@ impl VirtualMachine { } /// Interns `input_` in the VM's ref-string map and returns the ref-counted entry. + /// `input_` is `Latin1`, or `Utf16` as native-endian code units. pub fn ref_counted_string( &mut self, input_: &[u8], + encoding: EncodingNonAscii, hash_: Option, ) -> Result, crate::ref_string::StringTooLong> { - use crate::ref_string::{RefString, StringTooLong}; + use crate::ref_string::{Buffer, RefString, StringTooLong}; use bun_collections::zig_hash_map::MapEntry as Entry; jsc::mark_binding(); debug_assert!(!input_.is_empty()); - let hash = hash_.unwrap_or_else(|| RefString::compute_hash(input_)); + debug_assert!(encoding != EncodingNonAscii::Utf8); + let utf16 = encoding == EncodingNonAscii::Utf16; + let hash = hash_.unwrap_or_else(|| RefString::compute_hash(input_, utf16)); let _unlock = self.ref_strings_mutex.lock_guard(); // Note: reshaped for borrowck — capture the back-pointer before // `ref_strings.entry()` takes its unique borrow on `self`. @@ -5069,15 +5048,32 @@ impl VirtualMachine { match self.ref_strings.entry(hash) { Entry::Occupied(o) => Ok(*o.get()), Entry::Vacant(v) => { - if input_.len() > bun_core::String::max_length() { + let unit_count = if utf16 { + input_.len() / 2 + } else { + input_.len() + }; + if unit_count > bun_core::String::max_length() { return Err(StringTooLong); } - let buf = Box::<[u8]>::from(input_); - let len = buf.len(); - let ptr = bun_core::heap::into_raw(buf).cast::().cast_const(); + let buffer = if utf16 { + // As `[u16]`, so that the units are aligned. + let units: Box<[u16]> = bun_js_printer::utf16_units(input_).into(); + let len = units.len(); + Buffer::Utf16 { + ptr: bun_core::heap::into_raw(units).cast::().cast_const(), + len, + } + } else { + let buf = Box::<[u8]>::from(input_); + let len = buf.len(); + Buffer::Latin1 { + ptr: bun_core::heap::into_raw(buf).cast::().cast_const(), + len, + } + }; let ref_ = bun_core::heap::into_raw_nn(Box::new(RefString { - ptr, - len, + buffer, hash, // Filled in just below — `create_external` needs the // `*mut RefString` ctx pointer first. @@ -5087,16 +5083,29 @@ impl VirtualMachine { on_before_deinit: None, })); // SAFETY: `ref_` is the unique live `*mut RefString` (just - // boxed); `(ptr, len)` is its owned latin-1 buffer. The + // boxed); `buffer` is its owned allocation, in either width. The // external-string finalizer (`free_ref_string`) is called by // WTF on the JS thread when the impl refcount hits zero, with // `ref_` as ctx. - let s = bun_core::String::create_external::<*mut RefString>( - unsafe { bun_core::ffi::slice(ptr, len) }, - true, - ref_.as_ptr(), - free_ref_string, - ); + let s = unsafe { + match (*ref_.as_ptr()).buffer { + Buffer::Utf16 { ptr, len } => { + bun_core::String::create_external_utf16::<*mut RefString>( + bun_core::ffi::slice(ptr, len), + ref_.as_ptr(), + free_ref_string, + ) + } + Buffer::Latin1 { ptr, len } => { + bun_core::String::create_external::<*mut RefString>( + bun_core::ffi::slice(ptr, len), + true, + ref_.as_ptr(), + free_ref_string, + ) + } + } + }; if s.tag() == bun_core::Tag::Dead { // `create_external` already freed `ref_`. return Err(StringTooLong); diff --git a/src/runtime/api/filesystem_router.rs b/src/runtime/api/filesystem_router.rs index b169d4b1890c..34f7e8c2c094 100644 --- a/src/runtime/api/filesystem_router.rs +++ b/src/runtime/api/filesystem_router.rs @@ -55,7 +55,11 @@ struct RefCountedString(BackRef); impl RefCountedString { fn new(vm: &mut VirtualMachine, bytes: &[u8]) -> Result { - let string = BackRef::from(vm.ref_counted_string(bytes, None)?); + let string = BackRef::from(vm.ref_counted_string( + bytes, + bun_core::strings::EncodingNonAscii::Latin1, + None, + )?); // The cache adds no ref on a hit. string.ref_(); Ok(Self(string)) diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs index c7b3174f2def..4357f1c5aef3 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs @@ -17,6 +17,7 @@ //! `__bun_http_sync_download_*` — low-tier extern impls. use bun_core::WTFStringImplExt as _; +use bun_core::strings::EncodingNonAscii; use bun_options_types::LoaderExt as _; use core::cell::Cell; use core::ffi::c_void; @@ -2923,17 +2924,16 @@ fn transpile_source_code_of_any_provenance( }; let is_commonjs_module = entry.metadata.module_type == CacheModuleType::Cjs; // Node compile cache hook (transpiler-cache-hit path); must - // read `output_code` before it is consumed below. UTF-16 - // output would hash differently than the print path — skip. + // read `output_code` before it is consumed below. let node_compile_cache_blob = if bun_jsc::node_compile_cache::is_enabled() && source.path.is_file() && loader.is_java_script_like() - && !entry.output_code.is_utf16() { bun_jsc::node_compile_cache::fetch( source.path.text, is_commonjs_module, entry.output_code.byte_slice(), + entry.metadata.output_width(), ) } else { None @@ -3166,11 +3166,17 @@ fn transpile_source_code_of_any_provenance( let printer: &mut bun_js_printer::BufferPrinter = unsafe { &mut *(*extra).source_code_printer }; let written = printer.ctx.get_written(); + let encoding = printer.ctx.output_encoding(); let node_compile_cache_blob = if bun_jsc::node_compile_cache::is_enabled() && path.is_file() && loader.is_java_script_like() { - bun_jsc::node_compile_cache::fetch(path.text, is_commonjs_module, written) + bun_jsc::node_compile_cache::fetch( + path.text, + is_commonjs_module, + written, + encoding, + ) } else { None }; @@ -3179,6 +3185,7 @@ fn transpile_source_code_of_any_provenance( let mut resolved_source = unsafe { (*jsc_vm).ref_counted_resolved_source( written, + encoding, input_specifier, path.text, None, @@ -3248,13 +3255,19 @@ fn transpile_source_code_of_any_provenance( let printer: &mut bun_js_printer::BufferPrinter = unsafe { &mut *(*extra).source_code_printer }; let written = printer.ctx.get_written(); + let encoding = printer.ctx.output_encoding(); // Node compile cache hook (sync transpile path). `fetch` copies // `written`; the printer may be replaced below. let node_compile_cache_blob = if bun_jsc::node_compile_cache::is_enabled() && path.is_file() && loader.is_java_script_like() { - bun_jsc::node_compile_cache::fetch(path.text, is_commonjs_module, written) + bun_jsc::node_compile_cache::fetch( + path.text, + is_commonjs_module, + written, + encoding, + ) } else { None }; @@ -3264,15 +3277,18 @@ fn transpile_source_code_of_any_provenance( // `None`. debug_assert!(cache.output_code.is_none()); let written_len = written.len(); - let source_code = bun_core::String::clone_latin1(written); + // A copy in the printed width, not a transcode. + debug_assert!(encoding != EncodingNonAscii::Utf8); + let source_code = printer.ctx.clone_written_as_string(); // `printer.ctx.buffer.deinit()`: release the // large/--smol print buffer now instead of holding it until the // next transpile. Replacing the printer drops the old buffer // (same shape as RuntimeTranspilerStore). // SAFETY: per fn contract — `jsc_vm` is the live per-thread VM. if written_len > 1024 * 1024 * 2 || unsafe { &*jsc_vm }.smol { - *printer = - bun_js_printer::BufferPrinter::init(bun_js_printer::BufferWriter::init()); + *printer = bun_js_printer::BufferPrinter::init( + bun_js_printer::BufferWriter::init_latin1(), + ); } // (fd close handled by `_fd_guard` registered above; spec @@ -4424,7 +4440,7 @@ pub(crate) unsafe extern "C" fn Bun__transpileFile( let printer_ptr: *mut bun_js_printer::BufferPrinter = TRANSPILE_PRINTER.with(|cell| { let mut p = cell.get(); if p.is_null() { - let writer = bun_js_printer::BufferWriter::init(); + let writer = bun_js_printer::BufferWriter::init_latin1(); let bp = Box::new(bun_js_printer::BufferPrinter::init(writer)); p = bun_core::heap::into_raw(bp); cell.set(p); @@ -4574,7 +4590,7 @@ pub(crate) extern "C" fn Bun__transpileVirtualModule( let printer_ptr: *mut bun_js_printer::BufferPrinter = TRANSPILE_PRINTER.with(|cell| { let mut p = cell.get(); if p.is_null() { - let writer = bun_js_printer::BufferWriter::init(); + let writer = bun_js_printer::BufferWriter::init_latin1(); let bp = Box::new(bun_js_printer::BufferPrinter::init(writer)); p = bun_core::heap::into_raw(bp); cell.set(p); diff --git a/src/sourcemap/Chunk.rs b/src/sourcemap/Chunk.rs index 792d1cac74fd..863fb6a6d956 100644 --- a/src/sourcemap/Chunk.rs +++ b/src/sourcemap/Chunk.rs @@ -1,4 +1,5 @@ use bun_ast::{Loc, Source}; +use bun_core::strings::EncodingNonAscii; use bun_core::{MutableString, strings}; use bun_paths::fs::FileSystem; use bun_ptr::RawSlice; @@ -382,6 +383,9 @@ pub struct NewBuilder<'a, T: SourceMapFormatCtx> { /// When generating sourcemappings for bun, we store a count of how many mappings there were pub prepend_count: bool, + + /// How `output` is encoded during the print: `Utf8`, or `Latin1` for one byte per UTF-16 code unit. + pub output_encoding: EncodingNonAscii, } impl Default for NewBuilder<'_, T> { @@ -401,6 +405,7 @@ impl Default for NewBuilder<'_, T> { line_starts_with_mapping: false, cover_lines_without_mappings: false, prepend_count: false, + output_encoding: EncodingNonAscii::Utf8, } } } @@ -508,32 +513,47 @@ impl NewBuilder<'_, VLQSourceMap> { self.last_generated_update = output.len() as u32; return; } - self.update_generated_line_and_column_slow(output); + self.update_generated_line_and_column_slow(output, false); } + /// `as_source_text`: `output` is source text, to be read as the lexer reads it. #[inline(never)] #[cold] - fn update_generated_line_and_column_slow(&mut self, output: &[u8]) { + fn update_generated_line_and_column_slow(&mut self, output: &[u8], as_source_text: bool) { let slice = &output[self.last_generated_update as usize..]; let mut needs_mapping = self.cover_lines_without_mappings && !self.line_starts_with_mapping && self.has_prev_state; + let latin1 = !as_source_text && self.output_encoding == EncodingNonAscii::Latin1; let mut i: usize = 0; let n: usize = slice.len(); let mut c: i32; while i < n { - let len = strings::wtf8_byte_sequence_length_with_invalid(slice[i]); - let mut cp_bytes = [0u8; 4]; - let take = (len as usize).min(n - i); - cp_bytes[..take].copy_from_slice(&slice[i..i + take]); - c = strings::decode_wtf8_rune_t::( - cp_bytes, - len, - strings::UNICODE_REPLACEMENT as i32, - ); - i += len as usize; + if latin1 { + c = slice[i] as i32; + i += 1; + } else if as_source_text { + let mut cursor = strings::Cursor { + i: i as u32, + ..Default::default() + }; + strings::CodepointIterator::init(slice).next(&mut cursor); + c = cursor.c as i32; + i += cursor.width as usize; + } else { + let len = strings::wtf8_byte_sequence_length_with_invalid(slice[i]); + let mut cp_bytes = [0u8; 4]; + let take = (len as usize).min(n - i); + cp_bytes[..take].copy_from_slice(&slice[i..i + take]); + c = strings::decode_wtf8_rune_t::( + cp_bytes, + len, + strings::UNICODE_REPLACEMENT as i32, + ); + i += len as usize; + } match c { 14..=127 => { @@ -559,7 +579,8 @@ impl NewBuilder<'_, VLQSourceMap> { 0x0D | 0x0A | 0x2028 | 0x2029 => { // windows newline if c == 0x0D { - let newline_check = self.last_generated_update as usize + i + 1; + // `i` is already past the '\r'. + let newline_check = self.last_generated_update as usize + i; if newline_check < output.len() && output[newline_check] == b'\n' { continue; } @@ -602,6 +623,16 @@ impl NewBuilder<'_, VLQSourceMap> { self.last_generated_update = output.len() as u32; } + /// A `Latin1` writer stores U+2028 and U+2029 as placeholder bytes, so `output[start..]` is counted from `text`. + #[cold] + pub fn update_generated_verbatim(&mut self, output: &[u8], start: usize, text: &[u8]) { + debug_assert!(self.output_encoding == EncodingNonAscii::Latin1); + self.update_generated_line_and_column(&output[..start]); + self.last_generated_update = 0; + self.update_generated_line_and_column_slow(text, true); + self.last_generated_update = output.len() as u32; + } + #[inline(always)] pub(crate) fn append_mapping(&mut self, current_state: SourceMapState) { self.append_mapping_without_remapping(current_state); diff --git a/test/bake/dev/production.test.ts b/test/bake/dev/production.test.ts index 5cc643e65e1b..febf35e0299b 100644 --- a/test/bake/dev/production.test.ts +++ b/test/bake/dev/production.test.ts @@ -166,6 +166,33 @@ export default function IndexPage() { expect(indexHtml).toContain(`
© café-中-🐰|11
`); }); + // Same handoff, for the text the printer used to escape: the server chunks + // are printed for the Bun target, so this pins that regex and raw template + // text reach the prerendered page as written. + test("non-ASCII regex and tagged template text survives prerendering", async () => { + const dir = await tempDirWithBakeDeps("bake-production-non-ascii-verbatim", { + "src/index.tsx": `export default { app: { framework: "react" } };`, + "pages/index.tsx": ` +const re = /café-中-🐰/u; +const raw = String.raw\`é中🐰\\n\`; + +export default function IndexPage() { + return
{[re.source, re.source.length, raw, raw.length].join("|")}
; +} +`, + }); + + const build = await Bun.$`${bunExe()} build --app ./src/index.tsx --outdir ./dist` + .cwd(dir) + .env(bunEnv) + .throws(false); + expect(build.stderr.toString()).not.toContain("error"); + expect(build.exitCode).toBe(0); + + const indexHtml = await Bun.file(path.join(dir, "dist", "index.html")).text(); + expect(indexHtml).toContain(`
café-中-🐰|9|é中🐰\\n|6
`); + }); + test("import.meta properties are inlined in catch-all routes during production build", async () => { const dir = await tempDirWithBakeDeps("bake-production-catch-all", { "src/index.tsx": `export default { diff --git a/test/bundler/bundler_bun.test.ts b/test/bundler/bundler_bun.test.ts index 1156f2720276..ba25716ed0fc 100644 --- a/test/bundler/bundler_bun.test.ts +++ b/test/bundler/bundler_bun.test.ts @@ -104,6 +104,32 @@ describe("bundler", () => { }, }, }); + // Regex literals and tagged template raw text are printed verbatim (UTF-8), so + // the .jsc sidecar's SourceCodeKey has to be computed from the same decoding + // the loader uses; otherwise JSC silently ignores the bytecode. (ESM bytecode + // only exists with --compile; see bundler_compile.test.ts for both formats.) + itBundled("bun/BytecodeSidecarNonAsciiRegexAndRawTemplate", { + target: "bun", + format: "cjs", + bytecode: true, + outdir: "/out", + files: { + "/entry.ts": /* js */ ` + const r = /café-中-🐰/u; + const raw = String.raw\`é中🐰\`; + console.log(JSON.stringify([r.source, r.source.length, raw, raw.length])); + `, + }, + run: { + stdout: JSON.stringify(["café-中-🐰", "café-中-🐰".length, "é中🐰", "é中🐰".length]), + env: { + BUN_JSC_verboseDiskCache: "1", + }, + validate({ stderr }) { + expect(stderr).toContain("[Disk Cache] Cache hit for sourceCode"); + }, + }, + }); itBundled("bun/embedded-sqlite-file", { target: "bun", outfile: "", @@ -194,6 +220,71 @@ error: Hello World`, }, }, }); + // The regex and raw template text make the bundle non-ASCII, so the runtime + // holds it as a 16-bit string: the error preview is rendered from that + // string and, with --sourcemap=inline, the sourceMappingURL comment is + // located in it. Apart from the text itself, the output below is identical + // to what the ASCII version of this file produces. + const nonAsciiThrowFixture = /* js */ ` + // this file has comments and weird whitespace, intentionally + // to make it obvious if sourcemaps were generated and mapped properly + const marker = /café-中-🐰/u; + if (marker.test("café-中-🐰")) code(); + function code() { + // hello world + throw new class Boom extends Error { + constructor() { + super(String.raw\`Hello 🐰\`); + } + }(); + } + `; + itBundled("bun/TargetBunNoSourcemapNonAsciiPreview", { + target: "bun", + files: { "/entry.ts": nonAsciiThrowFixture }, + run: { + exitCode: 1, + validate({ stderr }) { + // The regex text is shown as written while the string literal next to + // it is still escaped. Context-line labels are not asserted: they are + // off by one for bundled modules today, independently of this change. + expect(stderr).toInclude("| var marker = /café-中-🐰/u;\n"); + expect(stderr).toInclude('| if (marker.test("caf\\xE9-\\u4E2D-\\uD83D\\uDC30"))\n'); + expect(stderr).toInclude( + `7 | throw new class Boom extends Error { + ^ +error: Hello 🐰 +`, + ); + expect(stderr).toMatch(/at code \(.*out\.js:7:9\)\n/); + expect(stderr).toInclude("\nnote: missing sourcemaps for "); + }, + }, + }); + itBundled("bun/TargetBunSourcemapInlineNonAscii", { + target: "bun", + files: { "/entry.ts": nonAsciiThrowFixture }, + sourceMap: "inline", + run: { + exitCode: 1, + validate({ stderr }) { + expect(stderr).toStartWith( + `2 | // to make it obvious if sourcemaps were generated and mapped properly +3 | const marker = /café-中-🐰/u; +4 | if (marker.test("café-中-🐰")) code(); +5 | function code() { +6 | // hello world +7 | throw new class Boom extends Error { + ^ +error: Hello 🐰 +`, + ); + expect(stderr).toInclude("entry.ts:7:9"); + expect(stderr).toInclude("entry.ts:4:41"); + expect(stderr).not.toInclude("missing sourcemaps"); + }, + }, + }); itBundled("bun/unicode comment", { target: "bun", files: { diff --git a/test/bundler/bundler_bytecode_portable.test.ts b/test/bundler/bundler_bytecode_portable.test.ts index dbb8ef7ff722..713969002390 100644 --- a/test/bundler/bundler_bytecode_portable.test.ts +++ b/test/bundler/bundler_bytecode_portable.test.ts @@ -454,24 +454,24 @@ describe("bytecode cache portability", () => { "sha256": "2a5d62fb4ca9d107e3a5bb2abe6a73f3c859a6f1a91c6c3d77653cb8c2053361", }, "bun build --bytecode --minify all.js": { - "js": "50b3e5192dd86a205c73583d585884c1b414467b14db54d03c77d3026bf236ff", + "js": "64dc71f15378f793815556b5199d909e535008f5246bcdffd39d1420d86bd874", "jsc": { - "bytes": 1730952, - "sha256": "38da7a1b809b26521552ef82fa0bee88d50c4205cf5b28717543481c03f5bf95", + "bytes": 1730856, + "sha256": "7ff6e82cbd4478d9ce32603dad311ed444673ff64462122a14d997026932fb6a", }, }, "bun build --bytecode --minify features.js": { - "js": "d30a5febed53e316cc2dd2b076502079e809bb0c201ef1671e9a190ecdcf093d", + "js": "3e0c71e700601afeef9130620b9270e85adb2e2d36c1cc242c9888116bbb30f9", "jsc": { - "bytes": 41608, - "sha256": "c81d6367c81f41f3b9749c921346070bdbb39433767dd128f398fb3b624494e3", + "bytes": 41584, + "sha256": "d79c2adb95be43aefd301e12d0d3e71703c67830d8f6532238ad8670f14639da", }, }, "bun build --bytecode --minify records.js": { - "js": "889cbb2c9525ff69a2676a6e81d97bb87760bdee65178b836c9c1d6808ac7c6e", + "js": "4545c6bbab726a0997a22995efe03c3b1baa1baea91b06f564f5ddb9deb3eb4f", "jsc": { - "bytes": 80112, - "sha256": "49786e42e67826975e0f97d8eac6e47c89f5d8501f6b9104b41979cd285cda31", + "bytes": 80016, + "sha256": "355ea7a4a098119c3272654ed2762fcaf20938d394fbcf8c23c3a6654ae8b69c", }, }, "bun build --bytecode acorn/dist/acorn.mjs": { @@ -482,10 +482,10 @@ describe("bytecode cache portability", () => { }, }, "bun build --bytecode all.js": { - "js": "ce4cf9db35e0aa3257f982fb756363a5686fb52a3b7cc63ac0c66bdcaf13a849", + "js": "9bf9637296542d089e6b93d505caa31694ab473d25a00337ffb99fcb7be6dc01", "jsc": { - "bytes": 1875248, - "sha256": "be31afcb449d24fd1b491c2931aa680ddf3030a3fe6a5063e458cac76b4055f1", + "bytes": 1875168, + "sha256": "3a2c216f818f7684b955cd78639879142077b97401be3e062cc042a90b732dad", }, }, "bun build --bytecode big.js": { @@ -496,10 +496,10 @@ describe("bytecode cache portability", () => { }, }, "bun build --bytecode features.js": { - "js": "2ee211924620db96d6e99e9490bfe0ee60a3bc6b003f37940c5631e6eabc2c73", + "js": "36684929c4ffe436d33976eb35c74f1d832ea561c6a4989e27e1e70c3c948943", "jsc": { - "bytes": 43384, - "sha256": "ddea629eab359252825e28aed849fa528604b9f005247909584dc423f55b3499", + "bytes": 43368, + "sha256": "6a081b338ee25b758abceee88584ef492bbbd49f54216a77b8981e3e77fdddb5", }, }, "bun build --bytecode happy-dom/lib/index.js": { @@ -517,10 +517,10 @@ describe("bytecode cache portability", () => { }, }, "bun build --bytecode libraries.js": { - "js": "7b8ce5d1861461086be174e60797d52794328828458a2086f4d49b39a3f8b28a", + "js": "f1b94de94f479b4aa6edb470c08da02e1dacba3001f6375b7a5547e51cc76bf1", "jsc": { - "bytes": 21178528, - "sha256": "bc660ddcd08d23219275a712d3fc6b651f29ea965d5fc7d7d7e7c2974e8fb456", + "bytes": 21175136, + "sha256": "679b672a0705574492a8e31e2c33df4481eb4176df43f79be8ea35865dc542df", }, }, "bun build --bytecode lodash/lodash.js": { @@ -538,10 +538,10 @@ describe("bytecode cache portability", () => { }, }, "bun build --bytecode records.js": { - "js": "c87ea35df4ad6b2063402c9901ef4775a82f12594f280db76f50695f4b0eba13", + "js": "7cc352643a740ea9ce181c991bd67463cc2f62a25556d983c13a6f54c47950a0", "jsc": { - "bytes": 82832, - "sha256": "a981521d10ecbda14ea3398a5d7d4d34ab17470f5f186ea0ad56355cafd17a3f", + "bytes": 82760, + "sha256": "6a1a0b32c52aea41faed0a1e6d671680193653860a2706e70efc2b68fd48a2e6", }, }, "bun build --bytecode shapes.js": { diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index 2aa50608d532..cb7690a5e481 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -152,6 +152,33 @@ describe("bundler", () => { }, }); + // Non-ASCII regex and tagged template text is printed as written, so the + // bytecode has to be generated from the same decoding of the output the + // loader uses, or JSC silently ignores it. + for (const format of ["cjs", "esm"] as const) { + itBundled(`compile/BytecodeNonAsciiRegexAndRawTemplate+${format}`, { + compile: true, + bytecode: true, + format, + files: { + "/entry.ts": /* js */ ` + const r = /café-中-🐰/u; + const raw = String.raw\`é中🐰\`; + console.log(JSON.stringify([r.source, r.source.length, raw, raw.length])); + `, + }, + run: { + stdout: JSON.stringify(["café-中-🐰", "café-中-🐰".length, "é中🐰", "é中🐰".length]), + env: { + BUN_JSC_verboseDiskCache: "1", + }, + validate({ stderr }) { + expect(stderr).toContain("[Disk Cache] Cache hit for sourceCode"); + }, + }, + }); + } + // `import defer * as ns from "..."` must not break bytecode generation. // The bundler inlines the deferred module into the entry chunk (documented // out-of-scope limitation — same as esbuild), so the defer semantics are diff --git a/test/bundler/transpiler/template-literal.test.ts b/test/bundler/transpiler/template-literal.test.ts index aa17d4839547..ae5165756a26 100644 --- a/test/bundler/transpiler/template-literal.test.ts +++ b/test/bundler/transpiler/template-literal.test.ts @@ -1,10 +1,9 @@ -import { expect, test } from "bun:test"; -import { bunEnv, bunExe } from "harness"; +import { describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, tempDir } from "harness"; +import { readdirSync, readFileSync, statSync } from "node:fs"; import { join } from "path"; -// When targeting Bun's runtime, -// We must escape latin1 characters in raw template literals -// This is somewhat brittle +// Tagged template cooked strings round-trip through the runtime transpiler. test("template literal", () => { const { stdout, exitCode } = Bun.spawnSync({ cmd: [bunExe(), "run", join(import.meta.dir, "template-literal-fixture-test.js")], @@ -20,3 +19,667 @@ test("template literal", () => { "8J+QsDEyMzEyM/CfkLDwn5Cw8J+QsPCfkLDwn5Cw8J+QsDEyM/CfkLAxMjPwn5CwMTIzMTIz8J+QsDEyM/CfkLAxMjPwn5CwLPCfkLB0cnVl", ); }); + +// The runtime transpiler must not rewrite non-ASCII characters inside tagged +// template raw contents or regex literal patterns: both are observable at +// runtime via `.raw` / `.source`. +// https://github.com/oven-sh/bun/issues/8745 +// https://github.com/oven-sh/bun/issues/18115 +// https://github.com/oven-sh/bun/issues/15492 +// https://github.com/oven-sh/bun/issues/16763 +// https://github.com/oven-sh/bun/issues/8207 +// https://github.com/oven-sh/bun/issues/13853 +// https://github.com/oven-sh/bun/issues/33930 +async function run(code: string) { + await using proc = Bun.spawn({ + cmd: [bunExe(), "-e", code], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + return { stdout, stderr, exitCode }; +} + +describe.concurrent("RegExp literal .source preserves non-ASCII", () => { + test.each([ + ["latin1 range", "/café/u", "café"], + ["BMP", "/中文/u", "中文"], + ["BOM", "/a\uFEFFb/u", "a\uFEFFb"], + ["astral", "/a🐰b/u", "a🐰b"], + ])("%s", async (_name, literal, source) => { + const { stdout, stderr, exitCode } = await run( + `const r = ${literal}; process.stdout.write(JSON.stringify([r.source, r.source.length, String(r)]));`, + ); + expect({ stdout, stderr }).toEqual({ + stdout: JSON.stringify([source, source.length, `/${source}/u`]), + stderr: "", + }); + expect(exitCode).toBe(0); + }); + + // https://github.com/oven-sh/bun/issues/13853 + // parsel-js widens a regex via re.source.replace("(?¶*)", ...); when + // the transpiler escapes ¶ to \u00B6 the replace never matches and puppeteer's + // ::-p-* selector arguments come back empty. + test("parsel-js .source.replace on literal ¶ (#13853)", async () => { + const { stdout, stderr, exitCode } = await run( + `const re = /::(?[-\\w\\P{ASCII}]+)(?:\\((?¶*)\\))?/gu;` + + `const widened = re.source.replace("(?¶*)", "(?.*)");` + + `process.stdout.write(JSON.stringify([re.source.includes("(?\\xB6*)"), widened.includes(".*"), re.source]));`, + ); + expect({ stdout, stderr }).toEqual({ + stdout: JSON.stringify([true, true, "::(?[-\\w\\P{ASCII}]+)(?:\\((?¶*)\\))?"]), + stderr: "", + }); + expect(exitCode).toBe(0); + }); + + // https://github.com/oven-sh/bun/issues/33930 + // Raw control bytes (0x00-0x1F) in a regex literal must come through as one + // code point in .source, not the six-character \uNNNN escape text. Uses a + // file because argv is NUL-terminated. + test("raw NUL and control bytes (#33930)", async () => { + using dir = tempDir("regex-source-control", { + "entry.js": + `const nul = /` + + "\x00" + + `HFMASK(\\d+)` + + "\x00" + + `/g;\n` + + `const ctrl = /a` + + "\x01\x1f" + + `b/;\n` + + `process.stdout.write(JSON.stringify({\n` + + ` nul: [...nul.source].map(c => c.codePointAt(0)),\n` + + ` nulLen: nul.source.length,\n` + + ` match: nul.test("\\x00HFMASK42\\x00"),\n` + + ` ctrl: [...ctrl.source].map(c => c.codePointAt(0)),\n` + + ` ctrlLen: ctrl.source.length,\n` + + `}));\n`, + }); + await using proc = Bun.spawn({ + cmd: [bunExe(), join(String(dir), "entry.js")], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout, stderr }).toEqual({ + stdout: JSON.stringify({ + nul: [0, 72, 70, 77, 65, 83, 75, 40, 92, 100, 43, 41, 0], + nulLen: 13, + match: true, + ctrl: [97, 1, 31, 98], + ctrlLen: 4, + }), + stderr: "", + }); + expect(exitCode).toBe(0); + }); +}); + +describe.concurrent("tagged template .raw preserves non-ASCII", () => { + test.each([ + ["latin1 range", "Redémarrage"], + ["BMP", "before中after"], + ["astral", "æ™弟気👋"], + ])("%s", async (_name, value) => { + const { stdout, stderr, exitCode } = await run( + `process.stdout.write(JSON.stringify([String.raw\`${value}\`, String.raw\`${value}\`.length]));`, + ); + expect({ stdout, stderr }).toEqual({ stdout: JSON.stringify([value, value.length]), stderr: "" }); + expect(exitCode).toBe(0); + }); + + test("String.raw iterates code points (#18115)", async () => { + const { stdout, stderr, exitCode } = await run( + `const text = String.raw\`a中\`; const chars = []; for (const c of text) chars.push(c); ` + + `process.stdout.write(JSON.stringify(chars));`, + ); + expect({ stdout, stderr }).toEqual({ stdout: JSON.stringify(["a", "中"]), stderr: "" }); + expect(exitCode).toBe(0); + }); + + test("substitutions between non-ASCII", async () => { + const { stdout, stderr, exitCode } = await run(`process.stdout.write(String.raw\`中\${"x"}弟\${"y"}気\`);`); + expect({ stdout, stderr }).toEqual({ stdout: "中x弟y気", stderr: "" }); + expect(exitCode).toBe(0); + }); + + test(".raw matches source, cooked is unchanged", async () => { + const { stdout, stderr, exitCode } = await run( + `const tag = (s) => JSON.stringify({ raw: s.raw[0], cooked: s[0] }); process.stdout.write(tag\`é\\n中\`);`, + ); + expect({ stdout, stderr }).toEqual({ stdout: JSON.stringify({ raw: "é\\n中", cooked: "é\n中" }), stderr: "" }); + expect(exitCode).toBe(0); + }); +}); + +// Ill-formed UTF-8 in verbatim text is decoded by the same routine the lexer +// decodes string literals with, so whatever policy the lexer has for a byte +// sequence (#38262 is making it U+FFFD per maximal subpart, like node), the +// regex source and the raw text built from the same bytes must come out equal +// to a string literal built from them. This pins the two paths to each other +// rather than to a particular policy. A file written by `bun build` stores the +// same decode as UTF-8, so the bundle has to agree too. +describe.concurrent("ill-formed bytes in verbatim text decode like a string literal", () => { + const wrap = (bad: number[], before: string, after: string) => + Buffer.concat([Buffer.from(before), Buffer.from(bad), Buffer.from(after)]); + async function bundle(dir: string) { + await using build = Bun.spawn({ + cmd: [bunExe(), "build", "--target=bun", "--outfile", join(dir, "out.js"), "./entry.js"], + cwd: dir, + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [, buildErr, buildExit] = await Promise.all([build.stdout.text(), build.stderr.text(), build.exited]); + expect(buildErr).not.toContain("error"); + expect(buildExit).toBe(0); + } + + const sequences: [string, number[]][] = [ + ["stray continuation byte", [0xa9]], + ["truncated sequence followed by ASCII", [0xe4, 0xb8, 0x78]], + ["overlong encoding", [0xc0, 0x80]], + ["encoded surrogate", [0xed, 0xa0, 0x80]], + ["byte that cannot start a sequence", [0xff]], + ]; + test.each( + sequences.flatMap(([name, bytes]): [string, "run" | "bundled", number[]][] => + name === "encoded surrogate" + ? [[name, "run", bytes]] + : [ + [name, "run", bytes], + [name, "bundled", bytes], + ], + ), + )("%s (%s)", async (_name, mode, bytes) => { + using dir = tempDir("ill-formed-verbatim", { + "entry.js": Buffer.concat([ + wrap(bytes, "const literal = 'a", "b';\n"), + wrap(bytes, "const source = /a", "b/.source;\n"), + wrap(bytes, "const raw = ((s) => s.raw[0])`a", "b`;\n"), + Buffer.from( + "const units = (s) => [...s].map(c => c.codePointAt(0));\n" + + "process.stdout.write(JSON.stringify([units(literal), source === literal, raw === literal]));\n", + ), + ]), + }); + if (mode === "bundled") await bundle(String(dir)); + const { stdout, stderr, exitCode } = await runIn(String(dir), [mode === "bundled" ? "out.js" : "entry.js"]); + expect(stderr).toBe(""); + const [literal, sourceMatches, rawMatches] = JSON.parse(stdout); + // The literal itself is the lexer's reading of the bytes; only its shape + // is asserted, the policy belongs to the lexer. + expect(literal[0]).toBe(0x61); + expect(literal.at(-1)).toBe(0x62); + expect({ sourceMatches, rawMatches, literal }).toEqual({ sourceMatches: true, rawMatches: true, literal }); + expect(exitCode).toBe(0); + }); + + // A lone surrogate has no UTF-8 form. A file written by `bun build` has it + // as an escape in the regex and in the raw text, as in the string literal: + // the regex still matches the literal and the cooked text still equals it. + // After a backslash (an identity escape) that backslash starts the escape. + test("encoded surrogate (bundled) is written as an escape", async () => { + const bytes = [0xed, 0xa0, 0x80]; + using dir = tempDir("lone-surrogate-bundled", { + "entry.js": Buffer.concat([ + wrap(bytes, "const literal = 'a", "b';\n"), + wrap(bytes, "const re = /^a", "b$/;\n"), + wrap(bytes, "const [cooked, raw] = ((s) => [s[0], s.raw[0]])`a", "b`;\n"), + wrap(bytes, "const afterBackslash = /^a\\", "b$/;\n"), + wrap(bytes, "const cookedAfterBackslash = ((s) => s[0])`a\\", "b`;\n"), + Buffer.from( + "const units = (s) => [...s].map(c => c.codePointAt(0));\n" + + "process.stdout.write(JSON.stringify([units(literal), re.test(literal), re.source, cooked === literal, raw, " + + "afterBackslash.test(literal), cookedAfterBackslash === literal]));\n", + ), + ]), + }); + await bundle(String(dir)); + const { stdout, stderr, exitCode } = await runIn(String(dir), ["out.js"]); + expect({ stdout, stderr }).toEqual({ + stdout: JSON.stringify([[0x61, 0xd800, 0x62], true, "^a\\uD800b$", true, "a\\uD800b", true, true]), + stderr: "", + }); + expect(exitCode).toBe(0); + }); +}); + +test("bun build --target=bun preserves non-ASCII in regex/raw templates", async () => { + using dir = tempDir("nonascii-regex-template", { + "index.ts": [ + `const r = /café-中-🐰/u;`, + `const raw = String.raw\`é中🐰\`;`, + `process.stdout.write(JSON.stringify([r.source, r.source.length, raw, raw.length]));`, + ].join("\n"), + }); + + await using build = Bun.spawn({ + cmd: [bunExe(), "build", "--target=bun", "--outfile", String(dir) + "/out.js", String(dir) + "/index.ts"], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [, buildErr, buildExit] = await Promise.all([build.stdout.text(), build.stderr.text(), build.exited]); + expect(buildErr).not.toContain("error"); + expect(buildExit).toBe(0); + + const out = await Bun.file(String(dir) + "/out.js").text(); + expect(out).not.toMatch(/\\u00E9/i); + expect(out).not.toMatch(/\\u4E2D/i); + expect(out).toContain("café-中-🐰"); + + await using proc = Bun.spawn({ + cmd: [bunExe(), String(dir) + "/out.js"], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout, stderr }).toEqual({ + stdout: JSON.stringify(["café-中-🐰", "café-中-🐰".length, "é中🐰", "é中🐰".length]), + stderr: "", + }); + expect(exitCode).toBe(0); +}); + +// When printing for the runtime the printer writes 8-bit (Latin-1) output and +// widens the whole buffer to UTF-16 when it had a code point above U+00FF, so +// every path that hands transpiler output to JSC has an 8-bit arm and a 16-bit +// arm; `// @bun` files skip the printer and are decoded from UTF-8. The same +// module goes through each of those paths here with text of each class; a path +// that reads the wrong width shows up as mojibake or garbage. `body` must +// define `values`. +// +// The `padded-*` modules are large enough (MINIMUM_CACHE_SIZE is 4 KiB) to be +// stored in the on-disk transpiler cache. +const padding = "// " + Buffer.alloc(8 * 1024, "x").toString() + "\n"; +function probeFiles(body: string) { + const print = `process.stdout.write(JSON.stringify(values));\nprocess.exit(0);\n`; + return { + "main.js": body + print, + "padded-main.js": padding + body + print, + "pragma-main.js": "// @bun\n" + body + print, + "probe.mjs": body + "export default values;\n", + "padded-probe.mjs": padding + body + "export default values;\n", + "pragma-probe.mjs": "// @bun\n" + body + "export default values;\n", + "probe.cjs": body + "module.exports = values;\n", + "import.mjs": `import values from "./probe.mjs";\n` + print, + "import-pragma.mjs": `import values from "./pragma-probe.mjs";\n` + print, + "require.cjs": `const values = require("./probe.cjs");\n` + print, + "empty.js": "", + "import-empty.mjs": `import "./empty.js";\nimport values from "./probe.mjs";\n` + print, + "import-padded.mjs": `import "./empty.js";\nimport values from "./padded-probe.mjs";\n` + print, + }; +} + +function probe(text: string) { + const body = ` +const re = /${text}/u; +const raw = String.raw\`${text}\\n\`; +const values = [re.source, re.source.length, String(re), raw, raw.length]; +`; + return { + text, + body, + expected: JSON.stringify([text, text.length, `/${text}/u`, `${text}\\n`, text.length + "\\n".length]), + files: probeFiles(body), + }; +} + +// Legal comments were already printed verbatim before regex and template text +// was (and came back as mojibake, e.g. "©", on every path), so this variant +// pins the module loading half independently of the regex/template change. +function legalCommentProbe(text: string) { + const body = ` +function tagged() { + /*! ${text} */ + return 1; +} +const values = [tagged.toString().split("/*! ")[1].split(" */")[0]]; +`; + return { text, body, expected: JSON.stringify([text]), files: probeFiles(body) }; +} + +type Probe = ReturnType; + +// "café-©-ÿ" stays within Latin-1, so the module's output is still 8-bit; +// "中" and "🐰" widen it to UTF-16 (the rabbit also checks surrogate pairs). +const latin1 = probe("café-©-ÿ"); +const nonAscii = probe("café-中-🐰"); +const asciiTwin = probe("cafe-x-y"); +const latin1Comment = legalCommentProbe("© café ÿ"); +const nonAsciiComment = legalCommentProbe("© café-中-🐰"); +const asciiCommentTwin = legalCommentProbe("(c) cafe-x-y"); +const kinds: [string, Probe][] = [ + ["8-bit regex and raw template text", latin1], + ["UTF-16 regex and raw template text", nonAscii], + ["8-bit legal comment text (module loading fix alone)", latin1Comment], + ["UTF-16 legal comment text (module loading fix alone)", nonAsciiComment], +]; + +async function runIn(cwd: string, args: string[], env: Record = {}) { + await using proc = Bun.spawn({ + cmd: [bunExe(), ...args], + cwd, + env: { ...bunEnv, ...env }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + return { stdout, stderr, exitCode }; +} + +function transpilerCacheEnv(dir: string) { + return { + BUN_RUNTIME_TRANSPILER_CACHE_PATH: join(dir, "transpiler-cache"), + // Debug builds only read the cache back with this set. + BUN_DEBUG_ENABLE_RESTORE_FROM_TRANSPILER_CACHE: "1", + }; +} + +// Identity of every entry in the transpiler cache directory. An entry that +// fails to load is silently deleted and written again by the re-transpile, so +// a second run that really hit the cache leaves these unchanged. +function cacheEntries(dir: string) { + const cacheDir = join(dir, "transpiler-cache"); + return readdirSync(cacheDir).map(name => { + const { ino, mtimeMs, size } = statSync(join(cacheDir, name)); + return { name, ino, mtimeMs, size }; + }); +} + +// Entry header, see `Metadata::encode` in src/jsc/RuntimeTranspilerCache.rs: +// u32 version, u8 module type, u8 encoding tag, then u64 fields starting with +// features_hash, input_byte_length, input_hash, output_byte_offset and +// output_byte_length. UTF-16 output is stored in native byte order; CI is +// little-endian everywhere. +const CACHE_TAG_UTF16 = 2; +const CACHE_TAG_LATIN1 = 3; +function cacheEntryOutputs(dir: string) { + const cacheDir = join(dir, "transpiler-cache"); + return readdirSync(cacheDir) + .map(name => { + const entry = readFileSync(join(cacheDir, name)); + const tag = entry[5]; + const offset = Number(entry.readBigUInt64LE(30)); + const length = Number(entry.readBigUInt64LE(38)); + const output = entry.subarray(offset, offset + length); + return { tag, output: output.toString(tag === CACHE_TAG_UTF16 ? "utf16le" : "latin1") }; + }) + .sort((a, b) => a.tag - b.tag); +} + +describe.concurrent("every module loading path preserves non-ASCII regex/raw text", () => { + describe.each(kinds)("%s", (_kind, { files, expected }) => { + test.each([ + ["entry point", ["main.js"]], + ["entry point with --hot (watcher keeps a ref-counted copy of the source)", ["--hot", "main.js"]], + ["entry point with the // @bun pragma", ["pragma-main.js"]], + ["import (async transpiler)", ["import.mjs"]], + ["import of a // @bun module (async transpiler)", ["import-pragma.mjs"]], + ["require (sync transpiler)", ["require.cjs"]], + ])("%s", async (_name, args) => { + using dir = tempDir("nonascii-load-path", files); + const { stdout, stderr, exitCode } = await runIn(String(dir), args); + expect({ stdout, stderr }).toEqual({ stdout: expected, stderr: "" }); + expect(exitCode).toBe(0); + }); + }); + + // The on-disk transpiler cache stores the output in the width the printer + // ended up with, and the entry's tag is the one place that width can be + // observed from outside: 8-bit text must leave the module 8-bit, and text + // above U+00FF must have widened it. The first run writes the entry and the + // second run must serve it: an entry read back in the wrong width is + // garbage, and an entry that fails to load is rewritten. + test.each([ + ["8-bit", "entry point (sync transpiler)", "padded-main.js", latin1, CACHE_TAG_LATIN1], + ["8-bit", "import (async transpiler)", "import-padded.mjs", latin1, CACHE_TAG_LATIN1], + ["UTF-16", "entry point (sync transpiler)", "padded-main.js", nonAscii, CACHE_TAG_UTF16], + ["UTF-16", "import (async transpiler)", "import-padded.mjs", nonAscii, CACHE_TAG_UTF16], + ])("transpiler cache round trip of %s output through the %s", async (_width, _name, entry, probe, tag) => { + const { files, expected, text } = probe; + using dir = tempDir("nonascii-transpiler-cache", files); + const env = transpilerCacheEnv(String(dir)); + + const first = await runIn(String(dir), [entry], env); + expect({ stdout: first.stdout, stderr: first.stderr }).toEqual({ stdout: expected, stderr: "" }); + expect(first.exitCode).toBe(0); + // Only the padded module is large enough to be cached. + const written = cacheEntries(String(dir)); + expect(written).toHaveLength(1); + const [stored] = cacheEntryOutputs(String(dir)); + expect(stored.tag).toBe(tag); + expect(stored.output).toContain(`/${text}/u`); + + const second = await runIn(String(dir), [entry], env); + expect({ stdout: second.stdout, stderr: second.stderr }).toEqual({ stdout: expected, stderr: "" }); + expect(second.exitCode).toBe(0); + expect(cacheEntries(String(dir))).toEqual(written); + }); + + // `require()` reuses one printer for every module of the process, so after + // a module has widened it, the next module must start out 8-bit again. + // Nothing else in this file loads a widening module before an 8-bit one in + // the same process. + test("a module printed after a widened one is 8-bit again", async () => { + using dir = tempDir("nonascii-printer-reset", { + "wide.cjs": padding + nonAscii.body + "module.exports = values;\n", + "narrow.cjs": padding + latin1.body + "module.exports = values;\n", + "main.cjs": `process.stdout.write(JSON.stringify([require("./wide.cjs"), require("./narrow.cjs")]));\n`, + }); + const { stdout, stderr, exitCode } = await runIn(String(dir), ["main.cjs"], transpilerCacheEnv(String(dir))); + expect({ stdout, stderr }).toEqual({ stdout: `[${nonAscii.expected},${latin1.expected}]`, stderr: "" }); + expect(exitCode).toBe(0); + const [wide, narrow] = cacheEntryOutputs(String(dir)); + expect([wide.tag, narrow.tag]).toEqual([CACHE_TAG_UTF16, CACHE_TAG_LATIN1]); + expect(wide.output).toContain(`/${nonAscii.text}/u`); + expect(narrow.output).toContain(`/${latin1.text}/u`); + }); +}); + +// NODE_COMPILE_CACHE persists JSC bytecode generated from the transpiler's +// output, in whichever width it has, and keyed by a hash of those bytes. On +// later runs the loader must hand it the same bytes in the same width, whether +// it transpiled the module again or served it from the transpiler cache, and +// the bytecode must have been generated from a string of that width too; any +// mismatch is a silent miss. The ASCII twin gives the expected number of hits +// (entry point, probe module and the empty module, whose output has zero +// bytes). Before this change a cached module with a non-ASCII legal comment +// still reached the compile cache, so that combination must keep working. +// +// The modes differ in which path populates the compile cache: "transpiled +// again" and "served from the transpiler cache" generate the bytecode from the +// print path on the first run, while "warmed" writes the transpiler cache +// entry first (as a machine that has run the program before would have) so +// that the bytecode is generated from the entry's bytes and stored width, and +// the last run's acceptance is the match between the two. +// +// Each mode runs three probes through two or three rounds of processes, the +// first of which also generates and persists the bytecode, so these run on +// their own after the concurrent groups instead of competing with them. +type CompileCacheMode = "transpiled again" | "served from the transpiler cache" | "warmed"; +describe("NODE_COMPILE_CACHE bytecode is accepted", () => { + test.each<[string, CompileCacheMode, string, Probe, Probe[]]>([ + [ + "regex and raw template text, module transpiled again", + "transpiled again", + "import-empty.mjs", + asciiTwin, + [latin1, nonAscii], + ], + [ + "regex and raw template text, module served from the transpiler cache", + "served from the transpiler cache", + "import-padded.mjs", + asciiTwin, + [latin1, nonAscii], + ], + [ + "regex and raw template text, bytecode generated from a warm transpiler cache", + "warmed", + "import-padded.mjs", + asciiTwin, + [latin1, nonAscii], + ], + [ + "legal comment text, module served from the transpiler cache", + "served from the transpiler cache", + "import-padded.mjs", + asciiCommentTwin, + [latin1Comment, nonAsciiComment], + ], + ])("%s", async (_name, mode, entry, twin, probes) => { + async function cacheHits({ files, expected }: Probe) { + using dir = tempDir("nonascii-compile-cache", files); + const compileCache = { NODE_COMPILE_CACHE: join(String(dir), "compile-cache") }; + const transpilerCache = transpilerCacheEnv(String(dir)); + const runs = + mode === "transpiled again" + ? [compileCache, compileCache] + : mode === "served from the transpiler cache" + ? [ + { ...compileCache, ...transpilerCache }, + { ...compileCache, ...transpilerCache }, + ] + : [transpilerCache, { ...compileCache, ...transpilerCache }, { ...compileCache, ...transpilerCache }]; + + let stderr = ""; + for (const [i, env] of runs.entries()) { + const last = i === runs.length - 1; + const result = await runIn(String(dir), [entry], last ? { ...env, BUN_JSC_verboseDiskCache: "1" } : env); + if (!last) expect({ stdout: result.stdout, stderr: result.stderr }).toEqual({ stdout: expected, stderr: "" }); + else expect(result.stdout).toBe(expected); + expect(result.exitCode).toBe(0); + if (i === 0 && mode !== "transpiled again") expect(cacheEntries(String(dir))).toHaveLength(1); + stderr = result.stderr; + } + return stderr.split("[Disk Cache] Cache hit for sourceCode").length - 1; + } + + const [twinHits, ...hits] = await Promise.all([twin, ...probes].map(cacheHits)); + expect(twinHits).toBeGreaterThanOrEqual(3); + // One entry per probe: 8-bit output, then output widened to UTF-16. + expect(hits).toEqual([twinHits, twinHits]); + }); +}); + +// Error positions must still map back to the original source when a module +// has text that makes its buffer UTF-16. These layouts pin the line accounting +// before and after that text (the runtime printer puts each statement on its +// own generated line, so they say nothing about columns within a line; the +// columns are pinned by test/cli/inspect/inspect-inline-sourcemap.test.ts). +// Columns below are 1-based UTF-16 offsets of the callee in the original +// line, which `indexOf` gives directly. +describe.concurrent("stack positions are remapped around non-ASCII regex/raw text", () => { + const layouts = { + "text before the error sites": (text: string) => [ + `const re = /${text}/u;`, + `const raw = String.raw\`${text}\`; const a = new Error("a");`, + `const b = new Error("b");`, + `print(a, b);`, + ], + "text between the error sites": (text: string) => [ + `const a = new Error("a");`, + `const re = /${text}/u; const b = new Error("b");`, + `const raw = String.raw\`${text}\`;`, + `const c = new Error("c");`, + `print(a, b, c);`, + ], + }; + const print = [ + `function print(...errors) {`, + ` process.stdout.write(JSON.stringify(errors.map(e => e.stack.split("\\n")[1].match(/:(\\d+:\\d+)$/)[1])));`, + `}`, + ]; + + function expectedPositions(lines: string[]) { + return lines.flatMap((line, row) => { + const column = line.indexOf("Error("); + return column === -1 ? [] : [`${row + 1}:${column + 1}`]; + }); + } + + const texts: [string, string][] = [ + ["ASCII", "xyz"], + ["8-bit", "é©"], + ["UTF-16", "中🐰"], + ]; + test.each(texts.flatMap(([width, text]) => Object.keys(layouts).map(layout => [width, layout, text] as const)))( + "%s %s", + async (_width, layout, text) => { + const lines = layouts[layout as keyof typeof layouts](text); + using dir = tempDir("nonascii-stack-positions", { "main.js": [...lines, ...print].join("\n") }); + const { stdout, stderr, exitCode } = await runIn(String(dir), ["main.js"]); + expect({ stdout, stderr }).toEqual({ stdout: JSON.stringify(expectedPositions(lines)), stderr: "" }); + expect(exitCode).toBe(0); + }, + ); + + // U+2028 is a line terminator inside raw template text too, so the lines + // after it move by one in the original source and in the printed module. + test("UTF-16 text with a line separator in it", async () => { + const lines = [ + `const raw = String.raw\`中\u2028x\`; const a = new Error("a");`, + `const b = new Error("b");`, + `print(a, b);`, + ]; + using dir = tempDir("nonascii-stack-positions", { "main.js": [...lines, ...print].join("\n") }); + const { stdout, stderr, exitCode } = await runIn(String(dir), ["main.js"]); + expect({ stdout, stderr }).toEqual({ + stdout: JSON.stringify(expectedPositions(lines.join("\n").split(/[\n\u2028]/))), + stderr: "", + }); + expect(exitCode).toBe(0); + }); + + // The text after the separator is on line 2. Its ill-formed bytes count as + // the code units they are decoded to, whatever the decode is: the program + // reports that number itself. + test("a line separator and an ill-formed sequence in one text", async () => { + using dir = tempDir("nonascii-stack-positions", { + "main.js": Buffer.concat([ + Buffer.from("const raw = String.raw`\u2028"), + Buffer.from([0xe4, 0xb8]), + Buffer.from( + 'x`; const a = new Error("a");\n' + + "process.stdout.write(JSON.stringify([a.stack.split('\\n')[1].match(/:(\\d+:\\d+)$/)[1], raw.length]));\n", + ), + ]), + }); + const { stdout, stderr, exitCode } = await runIn(String(dir), ["main.js"]); + expect(stderr).toBe(""); + const [position, rawLength] = JSON.parse(stdout); + // The line is the raw text after the separator, then `; const a = new Error(`. + expect(position).toBe(`2:${rawLength - 1 + "`; const a = new ".length + 1}`); + expect(exitCode).toBe(0); + }); + + // A CR with no LF right after it is a line terminator. + test.each([ + ["ASCII", "X"], + ["UTF-16", "中"], + ])("a lone CR in a preserved comment, one %s character before the LF", async (_width, text) => { + const lines = [ + `function k() {`, + ` /*! a\r${text}`, + ` b */`, + ` return 1;`, + `}`, + `const a = new Error("a");`, + `print(a);`, + ]; + using dir = tempDir("nonascii-stack-positions", { "main.js": [...lines, ...print].join("\n") }); + const { stdout, stderr, exitCode } = await runIn(String(dir), ["main.js"]); + expect({ stdout, stderr }).toEqual({ + stdout: JSON.stringify(expectedPositions(lines.join("\n").split(/[\n\r]/))), + stderr: "", + }); + expect(exitCode).toBe(0); + }); +}); diff --git a/test/bundler/transpiler/transpiler.test.js b/test/bundler/transpiler/transpiler.test.js index 2d68d35449d2..240c2893e454 100644 --- a/test/bundler/transpiler/transpiler.test.js +++ b/test/bundler/transpiler/transpiler.test.js @@ -5283,6 +5283,22 @@ console.log(foo, array); const result = await transpiler.transform(input); expect(result).toBe(`let list = [\"\\u2022\", \"-\", \"\\u25E6\", \"\\u25AA\", \"\\u25AB\"];\n`); }); + + // The bun target still escapes string literals (that is invisible to the + // program), but regex literals and tagged template raw text are observable + // through RegExp#source and .raw, so they are printed verbatim. + // https://github.com/oven-sh/bun/issues/18115 + it("bun target keeps regex literals and raw template text verbatim", async () => { + const transpiler = new Bun.Transpiler({ + loader: "js", + target: "bun", + }); + + const input = 'const re = /café-中-🐰/u;\nconst raw = String.raw`é中🐰\\n`;\nconst str = "é中🐰";\n'; + const expected = 'const re = /café-中-🐰/u, raw = String.raw`é中🐰\\n`, str = "\\xE9\\u4E2D\\uD83D\\uDC30";\n'; + expect(transpiler.transformSync(input)).toBe(expected); + expect(await transpiler.transform(input)).toBe(expected); + }); }); describe("edge cases", () => { diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index f8c6887f5d9c..3de704fe4004 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -1197,10 +1197,20 @@ it.skipIf(!isASAN)("stops the file watcher thread before the VM is freed at exit expect(await Promise.all([run(), run(), run(), run()])).toEqual([0, 0, 0, 0]); }); -// --hot and --watch intern a module's transpiled text in the VM. -it("throws ERR_STRING_TOO_LONG for a module whose transpiled text exceeds the string limit", async () => { +// --hot and --watch intern a module's transpiled text in the VM, in the width it was printed in. The limit counts +// code units: the UTF-16 text of the last case is about 80,000 bytes. The two UTF-16 cases guard that path. They +// also pass when the text is 8-bit. +it.each([ + ["throws ERR_STRING_TOO_LONG for a module whose transpiled text exceeds the string limit", "", 200 * 1024], + [ + "throws ERR_STRING_TOO_LONG for a module whose UTF-16 transpiled text exceeds the string limit", + "const re = /中/;\n", + 200 * 1024, + ], + ["loads a module whose UTF-16 transpiled text is below the string limit", "const re = /中/;\n", 40_000], +])("%s", async (title, prefix, length) => { using dir = tempDir("hot-string-limit", { - "big.cjs": `const s = ${JSON.stringify(Buffer.alloc(200 * 1024, "x").toString())};\nmodule.exports = { len: s.length };\n`, + "big.cjs": `${prefix}const s = ${JSON.stringify(Buffer.alloc(length, "x").toString())};\nmodule.exports = { len: s.length };\n`, "small.cjs": `module.exports = { ok: 2 };\n`, "entry.cjs": ` let big; @@ -1222,7 +1232,9 @@ it("throws ERR_STRING_TOO_LONG for a module whose transpiled text exceeds the st }); const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); expect({ stdout, stderr, exitCode }).toEqual({ - stdout: JSON.stringify({ big: "caught:ERR_STRING_TOO_LONG", small: 2 }) + "\n", + stdout: + JSON.stringify({ big: title.startsWith("loads") ? `loaded:${length}` : "caught:ERR_STRING_TOO_LONG", small: 2 }) + + "\n", stderr: "", exitCode: 0, }); diff --git a/test/cli/inspect/inspect-inline-sourcemap.test.ts b/test/cli/inspect/inspect-inline-sourcemap.test.ts index 9d16f40145cd..6bff4a8a9375 100644 --- a/test/cli/inspect/inspect-inline-sourcemap.test.ts +++ b/test/cli/inspect/inspect-inline-sourcemap.test.ts @@ -1,21 +1,20 @@ import { spawn } from "bun"; -import { expect, test } from "bun:test"; +import { describe, expect, test } from "bun:test"; import fs from "fs"; import { bunEnv, bunExe, tempDir } from "harness"; import { join } from "node:path"; +import { SourceMapConsumer } from "source-map"; import { WebSocket } from "ws"; -test("--inspect inline sourcemap sources[0] is a valid path under cwd", async () => { - // VT (0x0B) / BEL (0x07) in the source exercise quote_for_json's RFC 8259 - // escape handling for sourcesContent. - const source = "// comment [\x0b\x07]\nsetInterval(() => {}, 200);\n"; - using dir = tempDir("inspect-sourcemap", { - "sub/target.mjs": source, - }); +// Starts `entry` under --inspect-wait, connects, and returns its +// Debugger.scriptParsed event together with the inline source map the runtime +// transpiler attached to it. +async function runtimeSourceMap(files: Record, entry: string) { + using dir = tempDir("inspect-sourcemap", files); const cwd = fs.realpathSync(String(dir)); await using proc = spawn({ - cmd: [bunExe(), "--inspect-wait=127.0.0.1:0", join("sub", "target.mjs")], + cmd: [bunExe(), "--inspect-wait=127.0.0.1:0", entry], env: bunEnv, cwd, stdout: "ignore", @@ -51,6 +50,7 @@ test("--inspect inline sourcemap sources[0] is a valid path under cwd", async () await Promise.race([new Promise(resolve => ws.addEventListener("open", () => resolve())), failed]); + const entryFile = entry.split(/[\\/]/).pop()!; const pending = new Map void>(); const scriptParsed = new Promise(resolve => { ws.addEventListener("message", ({ data }) => { @@ -58,7 +58,7 @@ test("--inspect inline sourcemap sources[0] is a valid path under cwd", async () if (typeof msg.id === "number" && pending.has(msg.id)) { pending.get(msg.id)!(msg); pending.delete(msg.id); - } else if (msg.method === "Debugger.scriptParsed" && String(msg.params?.url ?? "").endsWith("target.mjs")) { + } else if (msg.method === "Debugger.scriptParsed" && String(msg.params?.url ?? "").endsWith(entryFile)) { resolve(msg.params); } }); @@ -81,12 +81,79 @@ test("--inspect inline sourcemap sources[0] is a valid path under cwd", async () const m = String(params.sourceMapURL ?? "").match(/base64,([A-Za-z0-9+/=]+)/); expect(m).not.toBeNull(); const raw = Buffer.from(m![1], "base64").toString(); - expect(raw).not.toMatch(/\\v|\\x[0-9A-Fa-f]{2}/); - const map = JSON.parse(raw); - - expect(map.sources[0]).toBe("/sub/target.mjs"); - expect(map.sourcesContent[0]).toBe(source); + return { cwd, params, raw, map: JSON.parse(raw) }; } finally { ws.close(); } +} + +type Mapping = { generatedLine: number; generatedColumn: number; originalLine: number; originalColumn: number }; + +async function mappingsOf(map: any): Promise { + const found: Mapping[] = []; + await SourceMapConsumer.with(map, null, consumer => { + consumer.eachMapping(({ generatedLine, generatedColumn, originalLine, originalColumn }) => { + found.push({ generatedLine, generatedColumn, originalLine, originalColumn }); + }); + }); + return found; +} + +test("--inspect inline sourcemap sources[0] is a valid path under cwd", async () => { + // VT (0x0B) / BEL (0x07) in the source exercise quote_for_json's RFC 8259 + // escape handling for sourcesContent. + const source = "// comment [\x0b\x07]\nsetInterval(() => {}, 200);\n"; + const { raw, map } = await runtimeSourceMap({ "sub/target.mjs": source }, join("sub", "target.mjs")); + expect(raw).not.toMatch(/\\v|\\x[0-9A-Fa-f]{2}/); + expect(map.sources[0]).toBe("/sub/target.mjs"); + expect(map.sourcesContent[0]).toBe(source); +}); + +test("--inspect inline sourcemap for a module in a non-ASCII directory", async () => { + // The regex makes the module's transpiled text UTF-16, so the source map + // comment is appended to, and has to be found in, a 16-bit string. The + // script's URL comes from the module loader. No sourceURL comment is added: + // it was the path's UTF-8 bytes in an 8-bit string. + const source = "const re = /中/u;\nsetInterval(() => {}, 200);\n"; + const { cwd, params, map } = await runtimeSourceMap( + { "sub-é-中/target.mjs": source }, + join("sub-é-中", "target.mjs"), + ); + expect({ url: params.url, sourceURL: params.sourceURL }).toEqual({ + url: join(cwd, "sub-é-中", "target.mjs"), + sourceURL: undefined, + }); + expect(map.sources[0]).toBe("/sub-é-中/target.mjs"); + expect(map.sourcesContent[0]).toBe(source); +}); + +// The runtime transpiler prints this line exactly as written, so every mapping +// on it must have equal generated and original columns, and since each +// variant's texts occupy the same number of UTF-16 code units, the variants +// must produce the same mappings as each other. The regex is the first +// non-ASCII text of its module: the 8-bit variant pins that a Latin-1 buffer is +// counted one column per byte after it, and the widening variant pins that +// text above U+00FF counts one column per code unit (astral text counting +// two), also for the raw template text after it. +describe.concurrent("--inspect inline sourcemap columns after verbatim text", () => { + test.each([ + ["ASCII", "xyz", "w"], + ["8-bit", "éÿ©", "ü"], + ["widening", "中🐰", "é"], + ])("%s", async (_name, regexText, rawText) => { + const line = `const x = [/${regexText}/u, new Error("b"), String.raw\`${rawText}\`, f()];`; + const source = `function f() {}\n${line}\nsetInterval(() => {}, 200);\n`; + const { map } = await runtimeSourceMap({ "target.mjs": source }, "target.mjs"); + + const onLine = (await mappingsOf(map)).filter(m => m.generatedLine === 2); + const columns = onLine.map(m => m.generatedColumn); + expect(onLine).toEqual( + columns.map(column => ({ generatedLine: 2, generatedColumn: column, originalLine: 2, originalColumn: column })), + ); + // Expressions after the regex (and, for `f()`, after the raw text) are + // mapped at their own columns. + expect(columns).toContain(line.indexOf("new Error")); + expect(columns).toContain(line.indexOf("String.raw")); + expect(columns).toContain(line.indexOf("f()")); + }); }); diff --git a/test/cli/test/coverage.test.ts b/test/cli/test/coverage.test.ts index 8f24a9ad1ebb..274c7dfc9bd8 100644 --- a/test/cli/test/coverage.test.ts +++ b/test/cli/test/coverage.test.ts @@ -1410,3 +1410,73 @@ test("reports an error for a coverage directory longer than the path buffer", as expect(stderr).toContain("ENAMETOOLONG: File name too long: Failed to write lcov.info"); expect(exitCode).toBe(1); }); + +// Regex and raw template text is transpiled as written. The module text that +// JSC holds is then 8-bit with characters above 0x7F (Latin-1 range text) or +// UTF-16 (text beyond it), and coverage offsets count its code units. Each +// variant has the same number of code units on every line as the ASCII twin +// (an astral character is two code units), so its lcov record must be +// identical to the twin's. +test("non-ASCII regex and raw template text does not shift coverage lines", async () => { + const units = 40; + // `units` is a multiple of the length of each piece. + const repeat = (piece: string) => Buffer.alloc((Buffer.byteLength(piece) * units) / piece.length, piece).toString(); + const variants = { ascii: repeat("x"), latin1: repeat("é"), utf16: repeat("é中🐰") }; + using dir = tempDir( + "cov-non-ascii", + Object.fromEntries( + Object.entries(variants).flatMap(([name, text]) => [ + [ + `${name}/demo.ts`, + `export const raw = String.raw\`${text}\`; +export const re = /${text}/u; +export function covered() { + return raw.length + re.source.length; +} +export function uncovered() { + return 1; +} +export function alsoUncovered() { + return 2; +} +`, + ], + [ + `${name}/demo.test.ts`, + ` +import { expect, test } from "bun:test"; +import { covered } from "./demo"; + +test("source text has the expected length", () => { + expect(covered()).toBe(${2 * units}); +}); +`, + ], + ]), + ), + ); + const [ascii, latin1, utf16] = await Promise.all( + Object.keys(variants).map(async variant => { + const cwd = path.join(String(dir), variant); + await using proc = Bun.spawn({ + cmd: [bunExe(), "test", "--coverage", "--coverage-reporter=lcov", "./demo.test.ts"], + cwd, + env: bunEnv, + stdout: "ignore", + stderr: "pipe", + }); + const [stderr, exitCode] = await Promise.all([proc.stderr.text(), proc.exited]); + expect(stderr).toContain(" 1 pass\n"); + expect(exitCode).toBe(0); + const record = readFileSync(path.join(cwd, "coverage", "lcov.info"), "utf8") + .split("end_of_record") + .find(record => record.includes("\nSF:demo.ts\n")); + expect(record).toBeDefined(); + return record!; + }), + ); + // `uncovered` and `alsoUncovered` start on lines 6 and 9. + expect(ascii).toContain("\nDA:6,0\n"); + expect(ascii).toContain("\nDA:9,0\n"); + expect({ latin1, utf16 }).toEqual({ latin1: ascii, utf16: ascii }); +}); diff --git a/test/js/bun/repl/repl.test.ts b/test/js/bun/repl/repl.test.ts index 6f3ecb50cd83..1d47d5c72b7a 100644 --- a/test/js/bun/repl/repl.test.ts +++ b/test/js/bun/repl/repl.test.ts @@ -431,6 +431,23 @@ describe.concurrent("Bun REPL", () => { expect(exitCode).toBe(0); }); + test(".load of a file with an ill-formed byte inside a regex evaluates it with U+FFFD", async () => { + // Regex text reaches the evaluator as written (it was escaped to the six + // characters \\uFFFD before), with the lexer's reading of the byte. + using dir = tempDir("repl-load-ill-formed-regex", { + "bad.js": Buffer.concat([Buffer.from("var badRegex = /a"), Buffer.from([0xe9]), Buffer.from("b/;\n")]), + }); + const filePath = path.join(String(dir), "bad.js"); + const { outputs, stderr, exitCode } = await runRepl([ + `.load ${filePath}`, + "JSON.stringify([badRegex.source.length, badRegex.source.codePointAt(1)])", + ".exit", + ]); + expect(outputs).toEqual([`Loading ${filePath}...\n/a\uFFFDb/`, '"[3,65533]"']); + expect(stderr).toBe(""); + expect(exitCode).toBe(0); + }); + test(".load with a nonexistent file shows the error and keeps running", async () => { // A relative path: Windows rejects a forward-slash absolute path with EINVAL. const { outputs, stderr, exitCode } = await runRepl([ @@ -887,6 +904,21 @@ describe.concurrent("Bun REPL", () => { expect(exitCode).toBe(0); }); + // The REPL transpiles its input like the runtime does; regex literals and + // tagged template raw text must reach the program unescaped. + // https://github.com/oven-sh/bun/issues/18115 + test("-e keeps non-ASCII regex source and raw template text", async () => { + const { stdout, stderr, exitCode } = await runReplWith([ + "-e", + "console.log(JSON.stringify([/é/.source, String.raw`中🐰`, String(/café/u)]))", + ]); + expect({ stdout, stderr }).toEqual({ + stdout: JSON.stringify(["é", "中🐰", "/café/u"]) + "\n", + stderr: "", + }); + expect(exitCode).toBe(0); + }); + test("-p prints undefined for void expressions", async () => { const { stdout, stderr, exitCode } = await runReplWith(["-p", "void 0"]); expect(stdout).toBe("undefined\n"); diff --git a/test/js/bun/shell/bunshell.test.ts b/test/js/bun/shell/bunshell.test.ts index 4093aeb1c6f8..3b203de072ff 100644 --- a/test/js/bun/shell/bunshell.test.ts +++ b/test/js/bun/shell/bunshell.test.ts @@ -422,10 +422,25 @@ describe("bunshell", () => { test("escape unicode", async () => { const { stdout } = await $`echo \\弟\\気`; - // TODO: Uncomment and replace after unicode in template tags is supported - // expect(stdout.toString("utf8")).toEqual(`\弟\気\n`); - // Set this here for now, because unicode in template tags while using .raw is broken, but should be fixed - expect(stdout.toString("utf8")).toEqual("\\u5F1F\\u6C17\n"); + expect(stdout.toString("utf8")).toEqual(`\\弟\\気\n`); + }); + + // https://github.com/oven-sh/bun/issues/15929 + test("literal non-ascii in template raw text", async () => { + const { stdout } = await $`echo 日本語 español "rêver" 🐰`; + expect(stdout.toString("utf8")).toEqual("日本語 español rêver 🐰\n"); + }); + + test("literal non-ascii path after interpolation", async () => { + using dir = tempDir("shell-nonascii", { + "日本語/a.txt": "jp\n", + "español/a.txt": "es\n", + }); + const base = String(dir); + expect((await $`cat ${base}/日本語/a.txt`).text()).toEqual("jp\n"); + expect((await $`cat ${base}/español/a.txt`).text()).toEqual("es\n"); + expect((await $`cat ${base}/${"日本語"}/a.txt`).text()).toEqual("jp\n"); + expect((await $`cat ${base}/${"español"}/a.txt`).text()).toEqual("es\n"); }); /** diff --git a/test/regression/issue/14976/14976.test.ts b/test/regression/issue/14976/14976.test.ts index 004599a26b15..943ff8490637 100644 --- a/test/regression/issue/14976/14976.test.ts +++ b/test/regression/issue/14976/14976.test.ts @@ -70,7 +70,7 @@ test("constant-folded equals doesn't lie", async () => { console.log("\"" === '"'); }); -test.skip("template literal raw property with unicode in an ascii-only build", async () => { +test("template literal raw property with unicode in an ascii-only build", async () => { expect(String.raw`你好𐃘\\`).toBe("你好𐃘\\\\"); expect((await $`echo 你好𐃘`.text()).trim()).toBe("你好𐃘"); });