diff --git a/src/bundler/defines.rs b/src/bundler/defines.rs index 8897c2649eb2..4c0caab7069e 100644 --- a/src/bundler/defines.rs +++ b/src/bundler/defines.rs @@ -167,6 +167,7 @@ impl DefineExt for Define { identifiers: StringHashMap::default(), dots: StringHashMap::default(), drop_debugger, + user_hash: None, }); define.dots.reserve(124); diff --git a/src/bundler/options.rs b/src/bundler/options.rs index 292026162119..25640d4581f4 100644 --- a/src/bundler/options.rs +++ b/src/bundler/options.rs @@ -946,12 +946,31 @@ pub(crate) fn defines_from_transform_options( let drop_debugger = drop.iter().any(|item| *item == b"debugger"); - Ok(defines::Define::init( + let user_hash = defines::Define::hash_user_inputs( + user_defines + .keys() + .iter() + .zip(user_defines.values().iter()) + .map(|(k, v)| (k.as_ref(), v.as_ref())), + environment_defines + .keys() + .iter() + .zip(environment_defines.values().iter()) + .filter_map(|(k, v)| match &v.value { + defines::DefineValue::EString(s) if s.is_utf8() => Some((k.as_ref(), s.slice8())), + _ => None, + }), + drop.iter().copied(), + ); + + let mut define = defines::Define::init( Some(resolved_defines), Some(environment_defines), drop_debugger, omit_unused_global_calls, - )?) + )?; + define.user_hash = user_hash; + Ok(define) } const DEFAULT_LOADER_EXT_BUN: &[&[u8]] = &[b".node", b".html"]; @@ -1414,6 +1433,7 @@ impl<'a> BundleOptions<'a> { identifiers: self.define.identifiers.clone(), dots: self.define.dots.clone(), drop_debugger: self.define.drop_debugger, + user_hash: self.define.user_hash, }), drop: self.drop.clone(), bundler_feature_flags: self @@ -1663,11 +1683,7 @@ impl<'a> BundleOptions<'a> { log, // `define` is filled by `load_defines` later; // initialize empty so the struct is well-formed before `load_defines` runs. - define: Box::new(defines::Define { - identifiers: Default::default(), - dots: Default::default(), - drop_debugger: false, - }), + define: Box::new(defines::Define::default()), loaders, output_dir: Box::from(transform.output_dir.as_deref().unwrap_or(b"out")), target, diff --git a/src/bundler/transpiler.rs b/src/bundler/transpiler.rs index 18c768c20b49..09dfbb2aee8a 100644 --- a/src/bundler/transpiler.rs +++ b/src/bundler/transpiler.rs @@ -1644,6 +1644,7 @@ impl<'a> Transpiler<'a> { .bundler_feature_flags .as_deref() .and_then(|s| s.clone().ok().map(Box::new)); + opts.features.define_hash = self.options.define.user_hash; opts.features.repl_mode = self.options.repl_mode; // we'll just always enable top-level await diff --git a/src/js_parser/lib.rs b/src/js_parser/lib.rs index b8f1f973138e..a6e38fea1919 100644 --- a/src/js_parser/lib.rs +++ b/src/js_parser/lib.rs @@ -493,9 +493,48 @@ pub mod defines { pub identifiers: StringHashMap, pub dots: StringHashMap>, pub drop_debugger: bool, + /// `hash_user_inputs` of this table's inputs, for the runtime transpiler cache key. + pub user_hash: Option, } impl Define { + /// Order-independent, length-prefixed hash of the three inputs. `None` when all are empty. + pub fn hash_user_inputs<'i>( + defines: impl IntoIterator, + env_defines: impl IntoIterator, + drop: impl IntoIterator, + ) -> Option { + let mut defines: Vec<(&[u8], &[u8])> = defines.into_iter().collect(); + let mut env_defines: Vec<(&[u8], &[u8])> = env_defines.into_iter().collect(); + let mut drop: Vec<&[u8]> = drop.into_iter().filter(|item| !item.is_empty()).collect(); + if defines.is_empty() && env_defines.is_empty() && drop.is_empty() { + return None; + } + defines.sort_unstable(); + env_defines.sort_unstable(); + drop.sort_unstable(); + drop.dedup(); + + let mut hasher = bun_wyhash::Wyhash::init(0); + let mut update = |bytes: &[u8]| { + hasher.update(&(bytes.len() as u64).to_le_bytes()); + hasher.update(bytes); + }; + // Separate sections: `init` lets a later env pair override a user pair. + for pairs in [defines, env_defines] { + update(&(pairs.len() as u64).to_le_bytes()); + for (key, value) in pairs { + update(key); + update(value); + } + } + update(&(drop.len() as u64).to_le_bytes()); + for item in drop { + update(item); + } + Some(hasher.final_()) + } + pub(crate) fn for_identifier(&self, name: &[u8]) -> Option<&IdentifierDefine> { if let Some(data) = self.identifiers.get(name) { return Some(data); diff --git a/src/js_parser/parse/parse_entry.rs b/src/js_parser/parse/parse_entry.rs index cdb4d149ceec..33e1b2a91bca 100644 --- a/src/js_parser/parse/parse_entry.rs +++ b/src/js_parser/parse/parse_entry.rs @@ -212,6 +212,7 @@ impl<'a> Options<'a> { runtime_transpiler_cache: None, lower_using: f.lower_using, bundler_feature_flags: None, + define_hash: f.define_hash, repl_mode: f.repl_mode, jsx_optimization_inline: f.jsx_optimization_inline, }, diff --git a/src/js_parser/parser.rs b/src/js_parser/parser.rs index 52d8fc0309ab..18daa38d32f1 100644 --- a/src/js_parser/parser.rs +++ b/src/js_parser/parser.rs @@ -257,6 +257,9 @@ pub mod Runtime { /// in watch/dev-server mode. pub bundler_feature_flags: Option>, + /// `Define::user_hash` of this parse's define table (runtime transpiler cache key). + pub define_hash: Option, + /// REPL mode: transforms code for interactive evaluation /// - Wraps lone object literals `{...}` in parentheses /// - Hoists variable declarations for REPL persistence @@ -304,6 +307,7 @@ pub mod Runtime { runtime_transpiler_cache: None, lower_using: true, bundler_feature_flags: None, + define_hash: None, repl_mode: false, jsx_optimization_inline: false, } @@ -391,6 +395,12 @@ pub mod Runtime { hasher.update(b"\x00"); } } + + // Define pairs and `--drop` entries. `None` adds nothing, like an empty flag set. + if let Some(define_hash) = self.define_hash { + hasher.update(b"define"); + hasher.update(&define_hash.to_le_bytes()); + } } pub(crate) fn should_unwrap_require(&self, package_name: &[u8]) -> bool { diff --git a/src/jsc/RuntimeTranspilerCache.rs b/src/jsc/RuntimeTranspilerCache.rs index 90b1b4a461f9..6ad49e143f1a 100644 --- a/src/jsc/RuntimeTranspilerCache.rs +++ b/src/jsc/RuntimeTranspilerCache.rs @@ -55,7 +55,8 @@ bun_core::declare_scope!(cache, visible); /// offsets picked by a header byte) plus a body of tagged records with /// u8/u16/u32 ids and implied slots dropped, instead of fixed u32 arrays. /// Version 27: ModuleInfo string table holds Latin-1 / UTF-16 bodies, not WTF-8. -const EXPECTED_VERSION: u32 = 27; +/// Version 28: the define table and `--drop` entries participate in the features hash. +const EXPECTED_VERSION: u32 = 28; /// Source files smaller than this are not written to / read from the on-disk /// transpiler cache. Originally 50 KiB, which excluded almost every file in a diff --git a/src/runtime/cli/Arguments.rs b/src/runtime/cli/Arguments.rs index 655873400c26..27f2e765721a 100644 --- a/src/runtime/cli/Arguments.rs +++ b/src/runtime/cli/Arguments.rs @@ -1569,13 +1569,6 @@ pub(crate) fn parse(cmd: CommandTag, ctx: Context<'_>) -> crate::Result { expect(newCacheCount()).toBe(0); // cache hit, order doesn't matter }); + // A define replaces an identifier at parse time, so the define table is part + // of the cache key. The cache is shared by every project of the user, and + // keyed by source bytes, so two projects with the same file must not see + // each other's values. + describe("defines are part of the cache key", () => { + const code = `console.log(typeof BVAL === "undefined" ? "undefined" : BVAL);`; + const filler = Buffer.alloc((50 * 1024 * 1.5) | 0, "/").toString(); + + const run = (cwd: string, args: string[]) => { + const result = Bun.spawnSync({ cmd: [bunExe(), ...args], cwd, env }); + if (!result.success) throw new Error(result.stderr.toString()); + return result.stdout.toString().trim(); + }; + + test("bunfig [define] invalidates cache", () => { + // `bun run ` loads bunfig.toml after the command line is parsed, + // so the define table only exists once the runtime is up. + const projectA = join(temp_dir, "a"); + const projectB = join(temp_dir, "b"); + for (const dir of [projectA, projectB]) { + mkdirSync(dir); + writeFileSync(join(dir, "a.js"), code + "\n//" + filler); + } + const setDefine = (dir: string, value: string | null) => { + if (value === null) rmSync(join(dir, "bunfig.toml"), { force: true }); + else writeFileSync(join(dir, "bunfig.toml"), `[define]\nBVAL = '${JSON.stringify(value)}'\n`); + }; + + setDefine(projectA, "one"); + expect(run(projectA, ["run", "./a.js"])).toBe("one"); + expect(newCacheCount()).toBe(1); + expect(run(projectA, ["run", "./a.js"])).toBe("one"); + expect(newCacheCount()).toBe(0); + + // A new value: features_hash differs -> old entry deleted, new entry written + setDefine(projectA, "two"); + expect(run(projectA, ["run", "./a.js"])).toBe("two"); + expect(newCacheCount()).toBe(0); + + setDefine(projectA, null); + expect(run(projectA, ["run", "./a.js"])).toBe("undefined"); + expect(newCacheCount()).toBe(0); + + // The same source bytes in another project, with its own define + setDefine(projectB, "mine"); + expect(run(projectB, ["run", "./a.js"])).toBe("mine"); + expect(newCacheCount()).toBe(0); + expect(run(projectB, ["./a.js"])).toBe("mine"); + expect(newCacheCount()).toBe(0); + }); + + test("--define invalidates cache", () => { + writeFileSync(join(temp_dir, "a.js"), code + "\n//" + filler); + + expect(run(temp_dir, ["--define", 'BVAL:"cli"', "a.js"])).toBe("cli"); + expect(existsSync(cache_dir)).toBeTrue(); + expect(newCacheCount()).toBe(1); + expect(run(temp_dir, ["--define", 'BVAL:"cli"', "a.js"])).toBe("cli"); + expect(newCacheCount()).toBe(0); + + expect(run(temp_dir, ["--define", 'BVAL:"other"', "a.js"])).toBe("other"); + expect(newCacheCount()).toBe(0); + + expect(run(temp_dir, ["a.js"])).toBe("undefined"); + expect(newCacheCount()).toBe(0); + + expect(run(temp_dir, ["run", "--define", 'BVAL:"cli"', "./a.js"])).toBe("cli"); + expect(newCacheCount()).toBe(0); + + // The key is built from the resolved map: flag order does not matter, + // and a key given twice keeps its last value, so `x` then `cli` is + // served the entry that `--define BVAL:"cli"` alone wrote above. + expect(run(temp_dir, ["--define", 'BVAL:"cli"', "--define", "OTHER:1", "a.js"])).toBe("cli"); + expect(newCacheCount()).toBe(0); + const entry = join(cache_dir, readdirSync(cache_dir)[0]); + const written = readFileSync(entry); + expect(run(temp_dir, ["--define", "OTHER:1", "--define", 'BVAL:"cli"', "a.js"])).toBe("cli"); + expect(readFileSync(entry).equals(written)).toBeTrue(); + + expect(run(temp_dir, ["--define", 'BVAL:"cli"', "a.js"])).toBe("cli"); + const alone = readFileSync(entry); + expect(alone.equals(written)).toBeFalse(); + expect(run(temp_dir, ["--define", 'BVAL:"x"', "--define", 'BVAL:"cli"', "a.js"])).toBe("cli"); + expect(readFileSync(entry).equals(alone)).toBeTrue(); + expect(newCacheCount()).toBe(0); + }); + + test("--define passed to bun test invalidates cache", () => { + // `bun test` builds its define table on its own boot path. The test file + // is below the minimum cache size; the module it imports is not, and + // `bun a.js` loads that same module. + writeFileSync(join(temp_dir, "a.js"), code + "\n//" + filler); + writeFileSync( + join(temp_dir, "a.test.js"), + `import "./a.js";\nimport { test } from "bun:test";\ntest("x", () => {});\n`, + ); + // `bun test` prints its version banner to stdout ahead of the module's output. + const lastLine = (args: string[]) => run(temp_dir, args).split("\n").at(-1); + + expect(lastLine(["test", "--define", 'BVAL:"cli"', "./a.test.js"])).toBe("cli"); + expect(newCacheCount()).toBe(1); + + expect(lastLine(["test", "./a.test.js"])).toBe("undefined"); + expect(newCacheCount()).toBe(0); + expect(run(temp_dir, ["a.js"])).toBe("undefined"); + expect(newCacheCount()).toBe(0); + + expect(lastLine(["test", "--define", 'BVAL:"cli"', "./a.test.js"])).toBe("cli"); + expect(newCacheCount()).toBe(0); + expect(run(temp_dir, ["a.js"])).toBe("undefined"); + expect(newCacheCount()).toBe(0); + }); + + test("--drop invalidates cache", () => { + writeFileSync( + join(temp_dir, "a.js"), + `console.log("logged");\nprocess.stdout.write("written\\n");` + "\n//" + filler, + ); + + expect(run(temp_dir, ["a.js"])).toBe("logged\nwritten"); + expect(newCacheCount()).toBe(1); + + expect(run(temp_dir, ["--drop=console", "a.js"])).toBe("written"); + expect(newCacheCount()).toBe(0); + + expect(run(temp_dir, ["--drop=console", "a.js"])).toBe("written"); + expect(newCacheCount()).toBe(0); + + expect(run(temp_dir, ["a.js"])).toBe("logged\nwritten"); + expect(newCacheCount()).toBe(0); + }); + }); + // Serving the entry point from the cache must not change how the modules it // loads are resolved. Both of these are gated on the `has_loaded` flag, which // used to be set only on the path that runs the printer.