Preserve non-ASCII in regex .source and tagged template .raw

src/ast/transpiler_cache.rs

@@ -11,6 +11,7 @@
 //! `bun_bundler::cache` and are stored here type-erased as `*mut ()`).
 
 use crate::{ExportsKind, Source};
+use bun_core::strings::EncodingNonAscii;
 use core::ptr::NonNull;
 
 pub struct RuntimeTranspilerCache {
@@ -52,7 +53,7 @@ impl Default for RuntimeTranspilerCache {
 bun_dispatch::link_interface! {
     pub TranspilerCacheImpl[Jsc] {
         fn get(source: &Source, parser_options: NonNull<()>, used_jsx: bool) -> bool;
-        fn put(output_code: &[u8], sourcemap: &[u8], esm_record: &[u8]);
+        fn put(output_code: &[u8], output_encoding: EncodingNonAscii, sourcemap: &[u8], esm_record: &[u8]);
         fn is_disabled() -> bool;
     }
 }
@@ -83,10 +84,19 @@ impl RuntimeTranspilerCache {
         }
     }
 
+    /// `output_code` is the printer's buffer in `output_encoding` (`Latin1`, or native-endian `Utf16`).
     #[inline]
-    pub fn put(&mut self, output_code: &[u8], sourcemap: &[u8], esm_record: &[u8]) {
+    pub fn put(
+        &mut self,
+        output_code: &[u8],
+        output_encoding: EncodingNonAscii,
+        sourcemap: &[u8],
+        esm_record: &[u8],
+    ) {
         match self.r#impl {
-            Some(k) => Self::handle(k, self).put(output_code, sourcemap, esm_record),
+            Some(k) => {
+                Self::handle(k, self).put(output_code, output_encoding, sourcemap, esm_record)
+            }
             None => {
                 if self.input_hash.is_none() {
                     return;

src/bun_core/string/mod.rs

@@ -265,6 +265,14 @@ impl String {
             }
         }
     }
+    /// [`clone_utf16`](Self::clone_utf16) without the narrowing scan: the string is 16-bit.
+    pub fn clone_utf16_wide(s: &[u16]) -> Self {
+        if s.is_empty() {
+            return Self::EMPTY;
+        }
+        // SAFETY: s.as_ptr()/len describe a valid u16 slice.
+        unsafe { BunString__fromUTF16(s.as_ptr(), s.len()) }
+    }
     pub fn create_atom(s: &[u8]) -> Self {
         // SAFETY: s.as_ptr()/len describe a valid byte slice.
         unsafe { BunString__createAtom(s.as_ptr(), s.len()) }
@@ -306,6 +314,33 @@ impl String {
         is_latin1: bool,
         ctx: Ctx,
         callback: ExternalStringImplFreeFunction<Ctx>,
+    ) -> Self {
+        // The length is a character count. UTF-16 goes through `create_external_utf16`.
+        debug_assert!(is_latin1);
+        Self::create_external_impl(bytes.as_ptr(), bytes.len(), is_latin1, ctx, callback)
+    }
+
+    /// [`create_external`](Self::create_external) for UTF-16. `callback` gets the length in code units.
+    pub fn create_external_utf16<Ctx>(
+        units: &[u16],
+        ctx: Ctx,
+        callback: ExternalStringImplFreeFunction<Ctx>,
+    ) -> Self {
+        Self::create_external_impl(
+            units.as_ptr().cast::<u8>(),
+            units.len(),
+            false,
+            ctx,
+            callback,
+        )
+    }
+
+    fn create_external_impl<Ctx>(
+        ptr: *const u8,
+        len: usize,
+        is_latin1: bool,
+        ctx: Ctx,
+        callback: ExternalStringImplFreeFunction<Ctx>,
     ) -> Self {
         use core::ffi::c_void;
         // `Ctx` must be a pointer-sized, pointer-aligned handle.
@@ -320,9 +355,9 @@ impl String {
             };
         }
         let () = AssertPtrSized::<Ctx>::OK;
-        debug_assert!(!bytes.is_empty());
-        if bytes.len() > Self::max_length() {
-            callback(ctx, bytes.as_ptr().cast_mut().cast::<c_void>(), bytes.len());
+        debug_assert!(len != 0);
+        if len > Self::max_length() {
+            callback(ctx, ptr.cast_mut().cast::<c_void>(), len);
             return Self::DEAD;
         }
         // The const-assert above only checks size/alignment, so an owning
@@ -344,16 +379,8 @@ impl String {
                 ExternalStringImplFreeFunction<Ctx>,
                 extern "C" fn(*mut c_void, *mut c_void, usize),
             >(callback) });
-        // SAFETY: bytes describes a valid slice; len <= max_length checked.
-        let s = unsafe {
-            BunString__createExternal(
-                bytes.as_ptr(),
-                bytes.len(),
-                is_latin1,
-                ctx_erased,
-                cb_erased,
-            )
-        };
+        // SAFETY: `ptr`/`len` describe a live slice of the matching width; len <= max_length checked.
+        let s = unsafe { BunString__createExternal(ptr, len, is_latin1, ctx_erased, cb_erased) };
         debug_assert!(s.tag != Tag::WTFStringImpl || s.as_wtf().ref_count() == 1);
         s
     }

src/codegen/bake-codegen.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert";
 import { existsSync, readFileSync, rmSync } from "node:fs";
 import { basename, join } from "node:path";
-import { argParse, writeIfNotChanged } from "./helpers";
+import { argParse, checkAscii, writeIfNotChanged } from "./helpers";
 
 // arg parsing
 let { "codegen-root": codegenRoot, debug, ...rest } = argParse(["codegen-root", "debug"]);
@@ -143,6 +143,8 @@ async function run() {
           code = `let ${outName("unloadedModuleRegistry")}={},${outName("config")}={separateSSRGraph:${outName("$separateSSRGraph")}},${outName("server_exports")};${code}`;
 
           code = debug ? `((${params}) => {${code}})\n` : `((${params})=>{${code}})\n`;
+          // The dev server loads this as a Latin-1 string.
+          checkAscii(code);
         } else {
           code = debug ? `(async (${names}) => {${code}})({\n` : `(async(${names})=>{${code}})({`;
         }