diff --git a/src/bundler/transpiler.rs b/src/bundler/transpiler.rs index 2ae6a1d809f2..84ad0601dbc0 100644 --- a/src/bundler/transpiler.rs +++ b/src/bundler/transpiler.rs @@ -2265,7 +2265,7 @@ fn parse_unsupported_loader(loader: options::Loader, path: &bun_paths::fs::Path< // `format` is a runtime arg rather than a const generic — // `bun_js_printer::Format` doesn't derive `ConstParamTy` (and can't be added // from this crate). All callers pass a literal anyway; the inner -// `print_ast::<_, ASCII_ONLY, ENABLE_SOURCE_MAP>` keeps both const-generic +// `print_ast::<_, IS_BUN_PLATFORM, ENABLE_SOURCE_MAP>` keeps both const-generic // bools, so codegen monomorphizes the printer body identically. // PERF: outer `match format` is one extra branch — profile if hot. // ══════════════════════════════════════════════════════════════════════════ diff --git a/src/js_printer/lib.rs b/src/js_printer/lib.rs index ea71acc5c884..2628416b36d4 100644 --- a/src/js_printer/lib.rs +++ b/src/js_printer/lib.rs @@ -1009,12 +1009,12 @@ where } while i < n { - let width: u8 = match ENCODING { + let mut width: u8 = match ENCODING { Encoding::Latin1 | Encoding::Ascii => 1, Encoding::Utf8 => strings::wtf8_byte_sequence_length_with_invalid(text[i]), Encoding::Utf16 => 1, }; - let clamped_width = (width as usize).min(n.saturating_sub(i)); + let mut clamped_width = (width as usize).min(n.saturating_sub(i)); let c: i32 = match ENCODING { Encoding::Utf8 => { let bytes: [u8; 4] = match clamped_width { @@ -1032,10 +1032,19 @@ where } Encoding::Latin1 => text[i] as i32, Encoding::Utf16 => { - // TODO: if this is a part of a surrogate pair, we could parse the whole codepoint in order - // to emit it as a single \u{result} rather than two paired \uLOW\uHIGH. - // eg: "\u{10334}" will convert to "𐌴" without this. - code_unit_at!(i) + let unit = code_unit_at!(i); + if strings::u16_is_lead(unit as u16) && i + 1 < n { + let trail = code_unit_at!(i + 1); + if strings::u16_is_trail(trail as u16) { + width = 2; + clamped_width = 2; + strings::u16_get_supplementary(unit as u16, trail as u16) as i32 + } else { + unit + } + } else { + unit + } } }; @@ -4598,7 +4607,7 @@ pub mod __gated_printer { self.print(b" "); } - if IS_BUN_PLATFORM { + if ASCII_ONLY { // Translate any non-ASCII to unicode escape sequences let mut ascii_start: usize = 0; let mut is_ascii = false; @@ -7794,11 +7803,10 @@ pub type BufferPrinter = Writer; pub enum Format { Esm, Cjs, - // bun.js must escape non-latin1 identifiers in the output This is because - // we load JavaScript as a UTF-8 buffer instead of a UTF-16 buffer - // JavaScriptCore does not support UTF-8 identifiers when the source code - // string is loaded as const char* We don't want to double the size of code - // in memory... + // Runtime-transpiler path. The printer now emits UTF-8 for this variant and + // consumers call `String::clone_utf8`, so non-ASCII source text is preserved + // verbatim (observable via `Function.prototype.toString`, `RegExp#source`, + // tagged-template `.raw`). The name is kept for churn avoidance. EsmAscii, CjsAscii, } @@ -7903,7 +7911,12 @@ pub fn get_source_map_builder( // Top-level print entry points // ─────────────────────────────────────────────────────────────────────────── -pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOURCE_MAP: bool>( +pub fn print_ast< + 'a, + W: WriterTrait, + const IS_BUN_PLATFORM: bool, + const GENERATE_SOURCE_MAP: bool, +>( _writer: W, bump: &'a bun_alloc::Arena, tree: &'a Ast, @@ -8015,9 +8028,11 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR // defer: if minify_identifiers { renamer.deinit() } — Drop handles. - // `is_bun_platform = ascii_only` for printAst. - type PrinterType<'a, W, const A: bool, const G: bool> = - Printer<'a, W, A, false, /*IS_BUN_PLATFORM=*/ A, false, G>; + // Runtime-transpiler path: emit UTF-8 so `Function.prototype.toString`, + // `RegExp#source` and tagged-template `.raw` reflect the author's source + // text. Consumers pass the output through `String::clone_utf8`. + type PrinterType<'a, W, const B: bool, const G: bool> = + Printer<'a, W, /*ASCII_ONLY=*/ false, false, /*IS_BUN_PLATFORM=*/ B, false, G>; let mut writer = _writer; // Pre-size the output buffer ~proportional to the source. Transpiled output // is almost always within a small factor of the input, so reserving up front @@ -8026,13 +8041,13 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR let _ = writer.reserve(source.contents().len() as u64); let mut opts = opts; - let source_map_builder = get_source_map_builder::( + let source_map_builder = get_source_map_builder::( GenerateSourceMap::lazy_if(GENERATE_SOURCE_MAP), &mut opts, source, tree, ); - let mut printer = PrinterType::::init( + let mut printer = PrinterType::::init( writer, bump, tree.import_records.as_slice(), @@ -8050,7 +8065,7 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR // Borrowck: `opts` was moved into `Printer::init`; populate // `printer.module_info` by taking it back out of `printer.options` // (see `print_with_writer_and_platform`). - if PrinterType::::MAY_HAVE_MODULE_INFO { + if PrinterType::::MAY_HAVE_MODULE_INFO { printer.module_info = printer.options.module_info.take(); } printer.binary_expression_stack = Vec::new(); @@ -8070,7 +8085,7 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR // `require` must be an unbound variable. printer.print(b"var {require}=import.meta;"); - if PrinterType::::MAY_HAVE_MODULE_INFO { + if PrinterType::::MAY_HAVE_MODULE_INFO { if let Some(mi) = printer.module_info.as_deref_mut() { mi.flags.contains_import_meta = true; let s = mi.str(b"require"); @@ -8088,8 +8103,9 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR } printer.check_stack_overflow()?; - let have_module_info = PrinterType::::MAY_HAVE_MODULE_INFO - && printer.module_info.is_some(); + let have_module_info = + PrinterType::::MAY_HAVE_MODULE_INFO + && printer.module_info.is_some(); if have_module_info { printer .module_info diff --git a/src/jsc/AsyncModule.rs b/src/jsc/AsyncModule.rs index c2741524f6a5..0af6dd544d3d 100644 --- a/src/jsc/AsyncModule.rs +++ b/src/jsc/AsyncModule.rs @@ -1375,7 +1375,7 @@ impl AsyncModule { } Ok(ResolvedSource { - source_code: BunString::clone_latin1(printer.ctx.get_written()), + source_code: BunString::clone_utf8(printer.ctx.get_written()), specifier: BunString::init(specifier), source_url: BunString::init(path.text), is_commonjs_module, diff --git a/src/jsc/RuntimeTranspilerCache.rs b/src/jsc/RuntimeTranspilerCache.rs index 91b1426d2893..c783b6600837 100644 --- a/src/jsc/RuntimeTranspilerCache.rs +++ b/src/jsc/RuntimeTranspilerCache.rs @@ -41,7 +41,8 @@ bun_core::declare_scope!(cache, visible); /// Version 22: Serialize `has_tla` in the cached ESM record flags byte. Entries /// written before #30888 carried `has_tla=false` for every module; the cache-HIT /// path reinstates the bug for any previously-cached TLA module (#30887). -const EXPECTED_VERSION: u32 = 22; +/// Version 23: Runtime transpiler emits UTF-8 instead of ASCII-escaped output. +const EXPECTED_VERSION: u32 = 23; /// Source files smaller than this are not written to / read from the on-disk /// transpiler cache. Originally 50 KiB, which excluded almost every file in a @@ -1014,7 +1015,7 @@ impl RuntimeTranspilerCache { return; } debug_assert!(self.entry.is_none()); - let output_code = BunString::clone_latin1(output_code_bytes); + let output_code = BunString::clone_utf8(output_code_bytes); // Refcount stays at 1, sole owner. // BunString is Copy with no Drop, so an extra dupe_ref here would leak. self.output_code = Some(output_code); @@ -1033,7 +1034,7 @@ impl RuntimeTranspilerCache { } #[cfg(debug_assertions)] { - bun_core::scoped_log!(cache, "put() = {} bytes", output_code.latin1().len()); + bun_core::scoped_log!(cache, "put() = {} bytes", output_code_bytes.len()); } } } @@ -1083,10 +1084,9 @@ bun_ast::link_impl_TranspilerCacheImpl! { } debug_assert!(this.entry.is_none()); - // Borrowed Latin-1 view: `to_file` only reads `byte_slice()` + the encoding - // tag (unmarked 8-bit ZigString -> Encoding::LATIN1, same as clone_latin1), - // and `output_code_bytes` outlives the synchronous `to_file` call. - let output_code = BunString::ascii(output_code_bytes); + // UTF-8-tagged so `to_file` records Encoding::UTF8 (it then + // Box-copies into `OutputCode::Utf8`; see that fn's PERF note). + let output_code = BunString::borrow_utf8(output_code_bytes); let result = RuntimeTranspilerCache::to_file( this.input_byte_length.unwrap(), this.input_hash.unwrap(), diff --git a/src/jsc/RuntimeTranspilerStore.rs b/src/jsc/RuntimeTranspilerStore.rs index d9924856527d..ddff46ba7d80 100644 --- a/src/jsc/RuntimeTranspilerStore.rs +++ b/src/jsc/RuntimeTranspilerStore.rs @@ -1034,7 +1034,7 @@ impl TranspilerJob { _ => (ptr::null_mut(), 0), }; self.resolved_source = OwnedResolvedSource::from(ResolvedSource { - source_code: String::clone_latin1(&parse_result.source.contents), + source_code: String::clone_utf8(&parse_result.source.contents), already_bundled: true, bytecode_cache, bytecode_cache_size, @@ -1181,7 +1181,7 @@ impl TranspilerJob { // `cache.output_code` (only the `r#impl == None` fallback does, // and `r#impl` is `Some(Jsc)` here), so it is always `None`. debug_assert!(cache.output_code.is_none()); - let result = String::clone_latin1(written); + let result = String::clone_utf8(written); // SAFETY: leaf scalar field read on `*vm`; see `vm` note above. if written.len() > 1024 * 1024 * 2 || unsafe { (*vm).smol } { diff --git a/src/jsc/VirtualMachine.rs b/src/jsc/VirtualMachine.rs index 5af7f93e15e2..3f932701bc3a 100644 --- a/src/jsc/VirtualMachine.rs +++ b/src/jsc/VirtualMachine.rs @@ -3785,6 +3785,18 @@ impl VirtualMachine { ..Default::default() }; } + // `ref_counted_string` wraps the bytes as a Latin-1 external string. + // The runtime transpiler now emits UTF-8, so non-ASCII output would be + // mojibaked; transcode it instead of interning on that (rare) path. + if !bun_core::strings::is_all_ascii(code) { + return ResolvedSource { + source_code: bun_core::String::clone_utf8(code), + specifier, + source_url: create_if_different(&specifier, source_url), + source_code_needs_deref: true, + ..Default::default() + }; + } // Const-generic bool can't be `!ADD_DOUBLE_REF`, so branch. let source = if ADD_DOUBLE_REF { self.ref_counted_string::(code, hash_) diff --git a/src/runtime/cli/repl.rs b/src/runtime/cli/repl.rs index 989a93e7f005..768de3fe2a42 100644 --- a/src/runtime/cli/repl.rs +++ b/src/runtime/cli/repl.rs @@ -1862,7 +1862,7 @@ impl<'a> Repl<'a> { if bun_js_printer::print_ast::< _, - /* ASCII_ONLY */ true, + /* IS_BUN_PLATFORM */ true, /* GENERATE_SOURCE_MAP */ false, >( &mut buffer_printer, diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs index 7caf3e55e844..ac09ffac1a5b 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs @@ -2645,7 +2645,7 @@ fn transpile_source_code_inner( _ => (core::ptr::null_mut(), 0), }; return Ok(OwnedResolvedSource::from(ResolvedSource { - source_code: bun_core::String::clone_latin1(&source.contents), + source_code: bun_core::String::clone_utf8(&source.contents), specifier: input_specifier.dupe_ref(), source_url: create_if_different(input_specifier, path.text), already_bundled: true, @@ -3059,7 +3059,7 @@ fn transpile_source_code_inner( // `None`. debug_assert!(cache.output_code.is_none()); let written_len = written.len(); - let source_code = bun_core::String::clone_latin1(written); + let source_code = bun_core::String::clone_utf8(written); // `printer.ctx.buffer.deinit()`: release the // large/--smol print buffer now instead of holding it until the // next transpile. Replacing the printer drops the old buffer diff --git a/test/bundler/transpiler/transpiler.test.js b/test/bundler/transpiler/transpiler.test.js index bf2b3db6485c..7b2b9446a852 100644 --- a/test/bundler/transpiler/transpiler.test.js +++ b/test/bundler/transpiler/transpiler.test.js @@ -2308,13 +2308,16 @@ console.log(
);`), }); it("unicode surrogates", () => { - expectPrinted_(`console.log("𐌴")`, 'console.log("\\uD800\\uDF34")'); - expectPrinted_(`console.log("\\u{10334}")`, 'console.log("\\uD800\\uDF34")'); - expectPrinted_(`console.log("\\uD800\\uDF34")`, 'console.log("\\uD800\\uDF34")'); + expectPrinted_(`console.log("𐌴")`, 'console.log("𐌴")'); + expectPrinted_(`console.log("\\u{10334}")`, 'console.log("𐌴")'); + expectPrinted_(`console.log("\\uD800\\uDF34")`, 'console.log("𐌴")'); expectPrinted_(`console.log("\\u{10334}" === "\\uD800\\uDF34")`, "console.log(true)"); expectPrinted_(`console.log("\\u{10334}" === "\\uDF34\\uD800")`, "console.log(false)"); expectPrintedMin_(`console.log("abc" + "def")`, 'console.log("abcdef")'); + // Lone surrogates stay escaped; concatenation must not combine across operands. expectPrintedMin_(`console.log("\\uD800" + "\\uDF34")`, 'console.log("\\uD800" + "\\uDF34")'); + expectPrinted_(`console.log("\\uD800")`, 'console.log("\\uD800")'); + expectPrinted_(`console.log("\\uDF34")`, 'console.log("\\uDF34")'); }); it("fold string addition", () => { @@ -3945,7 +3948,7 @@ console.log(foo, array); const input = `let list = ["•", "-", "◦", "▪", "▫"];\n`; const result = await transpiler.transform(input); - expect(result).toBe(`let list = [\"\\u2022\", \"-\", \"\\u25E6\", \"\\u25AA\", \"\\u25AB\"];\n`); + expect(result).toBe(`let list = ["•", "-", "◦", "▪", "▫"];\n`); }); }); diff --git a/test/js/bun/shell/bunshell.test.ts b/test/js/bun/shell/bunshell.test.ts index 8082ef57b818..0c07d3c9af66 100644 --- a/test/js/bun/shell/bunshell.test.ts +++ b/test/js/bun/shell/bunshell.test.ts @@ -342,10 +342,7 @@ describe("bunshell", () => { test("escape unicode", async () => { const { stdout } = await $`echo \\弟\\気`; - // TODO: Uncomment and replace after unicode in template tags is supported - // expect(stdout.toString("utf8")).toEqual(`\弟\気\n`); - // Set this here for now, because unicode in template tags while using .raw is broken, but should be fixed - expect(stdout.toString("utf8")).toEqual("\\u5F1F\\u6C17\n"); + expect(stdout.toString("utf8")).toEqual("\\弟\\気\n"); }); /** diff --git a/test/js/bun/transpiler/transpiler-non-ascii-source-text.test.ts b/test/js/bun/transpiler/transpiler-non-ascii-source-text.test.ts new file mode 100644 index 000000000000..d83122538017 --- /dev/null +++ b/test/js/bun/transpiler/transpiler-non-ascii-source-text.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, test } from "bun:test"; +import { bunEnv, bunExe, tempDir } from "harness"; + +// ES2026 §20.2.3.5: Function.prototype.toString must return the source text +// the host has for the function. The runtime transpiler used to rewrite every +// non-ASCII codepoint to an escape sequence, which leaked into toString(), +// RegExp#source, and tagged-template .raw. + +const fixture = ` +function f() { return "café"; } +class Café { méth(x) { return \`é\${x}é\`; } } +const rx = () => /π+/u.test("ππ"); +const tag = (a) => a.raw[0]; +const raw = () => tag\`你好𐃘\\\\\`; +const dyn = new Function('return "café";'); + +console.log(JSON.stringify({ + value: f(), + fn: f.toString(), + cls: Café.toString(), + rx: rx.toString(), + raw: raw(), + dyn: dyn.toString(), + emoji: (() => "👍").toString(), +})); +process.exit(0); +`; + +async function run(cmd: string[], cwd: string) { + await using proc = Bun.spawn({ + cmd, + env: bunEnv, + cwd, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + return { stdout, stderr, exitCode }; +} + +describe.concurrent("runtime transpiler preserves non-ASCII source text", () => { + for (const [label, args] of [ + [".mjs", ["entry.mjs"]], + [".cjs", ["entry.cjs"]], + [".ts", ["entry.ts"]], + [".mjs --hot", ["--hot", "entry.mjs"]], + ] as const) { + test(label, async () => { + const ext = args[args.length - 1].split(".").pop()!; + using dir = tempDir("non-ascii-src", { [`entry.${ext}`]: fixture }); + const { stdout, exitCode } = await run([bunExe(), ...args], String(dir)); + const out = JSON.parse(stdout); + + // Evaluated value — must be the real string, not mojibake + expect(out.value).toBe("café"); + + // Function.prototype.toString — string literal preserved verbatim + expect(out.fn).toContain('"café"'); + expect(out.fn).not.toContain("\\xE9"); + expect(out.fn).not.toContain("\\u"); + + // Class source — identifier, method name, template literal all preserved + expect(out.cls).toContain("Café"); + expect(out.cls).toContain("méth"); + expect(out.cls).toContain("`é${x}é`"); + expect(out.cls).not.toContain("\\u{e9}"); + expect(out.cls).not.toContain("\\xE9"); + + // RegExp literal inside a function body + expect(out.rx).toContain("/π+/u"); + expect(out.rx).not.toContain("\\u03C0"); + + // Tagged template .raw — non-ASCII plus a raw backslash preserved + expect(out.raw).toBe("你好𐃘\\\\"); + + // Astral codepoint in a string literal + expect(out.emoji).toContain('"👍"'); + + // Control: new Function bodies never pass through the file transpiler + expect(out.dyn).toContain('"café"'); + + expect(exitCode).toBe(0); + }); + } + + test("RegExp.prototype.source", async () => { + using dir = tempDir("non-ascii-rx", { + "entry.mjs": `console.log(/π+/u.source);`, + }); + const { stdout, exitCode } = await run([bunExe(), "entry.mjs"], String(dir)); + expect({ stdout, exitCode }).toEqual({ stdout: "π+\n", exitCode: 0 }); + }); + + test("String.raw with non-ASCII", async () => { + using dir = tempDir("non-ascii-raw", { + "entry.mjs": `console.log(String.raw\`你好\\n𐃘\`);`, + }); + const { stdout, exitCode } = await run([bunExe(), "entry.mjs"], String(dir)); + expect({ stdout, exitCode }).toEqual({ stdout: "你好\\n𐃘\n", exitCode: 0 }); + }); + + test("// @bun pragma with non-ASCII source", async () => { + using dir = tempDir("non-ascii-pragma", { + "entry.mjs": `// @bun\nfunction f() { return "café"; }\nconsole.log(JSON.stringify({ value: f(), src: f.toString() }));\n`, + }); + const { stdout, exitCode } = await run([bunExe(), "entry.mjs"], String(dir)); + const out = JSON.parse(stdout); + expect(out.value).toBe("café"); + expect(out.src).toContain('"café"'); + expect(exitCode).toBe(0); + }); +}); diff --git a/test/regression/issue/14976/14976.test.ts b/test/regression/issue/14976/14976.test.ts index 1ea5e905af19..6d76a878a49c 100644 --- a/test/regression/issue/14976/14976.test.ts +++ b/test/regression/issue/14976/14976.test.ts @@ -70,7 +70,7 @@ test("constant-folded equals doesn't lie", async () => { console.log("\"" === '"'); }); -test.skip("template literal raw property with unicode in an ascii-only build", async () => { +test("template literal raw property with unicode", async () => { expect(String.raw`你好𐃘\\`).toBe("你好𐃘\\\\"); expect((await $`echo 你好𐃘`.text()).trim()).toBe("你好𐃘"); });