diff --git a/src/sourcemap/Chunk.rs b/src/sourcemap/Chunk.rs index 899e5671c01f..5e59d4b2e5aa 100644 --- a/src/sourcemap/Chunk.rs +++ b/src/sourcemap/Chunk.rs @@ -534,17 +534,16 @@ impl NewBuilder<'_, VLQSourceMap> { let mut i: usize = 0; let n: usize = slice.len(); let mut c: i32; + // An ill-formed sequence is one U+FFFD one byte wide, so a '\n' after it is not skipped. + let iter = strings::CodepointIterator::init(slice); while i < n { - let len = strings::wtf8_byte_sequence_length_with_invalid(slice[i]); - let mut cp_bytes = [0u8; 4]; - let take = (len as usize).min(n - i); - cp_bytes[..take].copy_from_slice(&slice[i..i + take]); - c = strings::decode_wtf8_rune_t::( - cp_bytes, - len, - strings::UNICODE_REPLACEMENT as i32, - ); - i += len as usize; + let mut cursor = strings::Cursor { + i: i as u32, + ..Default::default() + }; + let _ = iter.next(&mut cursor); + c = cursor.c; + i += cursor.width as usize; match c { 14..=127 => { diff --git a/src/sourcemap/LineOffsetTable.rs b/src/sourcemap/LineOffsetTable.rs index de4051acf294..8fe3d8e7a227 100644 --- a/src/sourcemap/LineOffsetTable.rs +++ b/src/sourcemap/LineOffsetTable.rs @@ -186,27 +186,18 @@ impl LineOffsetTable { // Hoist the base pointer so per-iteration offset math is a single sub + truncate. let base = contents.as_ptr() as usize; + // Same decoder as `Chunk.rs`: an ill-formed sequence is one U+FFFD one byte wide. + let iter = strings::CodepointIterator::init(contents); let mut remaining = contents; while !remaining.is_empty() { - let b0 = remaining[0]; - let len_ = strings::wtf8_byte_sequence_length_with_invalid(b0); - // After the SIMD skip below lands, the loop head - // is overwhelmingly an ASCII '\r'/'\n' or a non-ASCII lead byte, so keep the - // 1-byte path branch-only and confine the zero+copy pad to the cold - // multibyte arm. - // `len_` is the lead byte's *declared* width; a source whose final bytes are - // a truncated multibyte sequence declares more bytes than remain, so every - // slice below (decode, SIMD-skip offset, advance) must use the clamped width. - let cp_len = (len_ as usize).min(remaining.len()); - let c: i32 = if len_ == 1 { - b0 as i32 - } else { - let mut cp_bytes = [0u8; 4]; - cp_bytes[..cp_len].copy_from_slice(&remaining[..cp_len]); - strings::decode_wtf8_rune_t::(cp_bytes, len_, 0) - }; - let offset = (remaining.as_ptr() as usize - base) as u32; + let mut cursor = strings::Cursor { + i: offset, + ..Default::default() + }; + let _ = iter.next(&mut cursor); + let c: i32 = cursor.c; + let cp_len = cursor.width as usize; if column == 0 { line_byte_offset = offset; diff --git a/test/js/bun/sourcemap/internal-sourcemap-roundtrip.test.ts b/test/js/bun/sourcemap/internal-sourcemap-roundtrip.test.ts index ba26e5112af1..5d814694f517 100644 --- a/test/js/bun/sourcemap/internal-sourcemap-roundtrip.test.ts +++ b/test/js/bun/sourcemap/internal-sourcemap-roundtrip.test.ts @@ -1,6 +1,7 @@ import { internalSourceMap } from "bun:internal-for-testing"; import { describe, expect, test } from "bun:test"; import { bunEnv, bunExe, tempDir } from "harness"; +import { readFileSync } from "node:fs"; import path from "node:path"; // --- Reference VLQ codec (mirrors the spec; used as the oracle) --- @@ -491,6 +492,90 @@ describe.concurrent("sourcemap of a source with a truncated trailing UTF-8 seque }); }); +// Both sides of a mapping come from walking raw bytes: LineOffsetTable::generate +// walks the source and the builder in Chunk.rs walks the printed output. A lead +// byte that is not followed by the continuation bytes it declares (a Latin-1 +// file, a sequence cut short) is one character, one byte wide. Stepping over the +// declared width instead swallows the line break after it, and every mapping +// from there on is a line off. +describe.concurrent("sourcemap of a source with an ill-formed UTF-8 sequence right before a line break", () => { + // File contents are given as latin1 text, so "\xe9" is the one byte 0xE9 on + // disk; the output is read back the same way. Lines are 0-based like the map's. + async function bundle(latin1Files: Record, entry: string) { + using dir = tempDir( + "sourcemap-ill-formed-utf8", + Object.fromEntries(Object.entries(latin1Files).map(([name, text]) => [name, Buffer.from(text, "latin1")])), + ); + await using proc = Bun.spawn({ + cmd: [bunExe(), "build", "--sourcemap=external", "--outdir=out", entry], + env: bunEnv, + cwd: String(dir), + stdout: "pipe", + stderr: "pipe", + }); + const [, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ exitCode, stderr }).toEqual({ exitCode: 0, stderr: "" }); + const outfile = path.join(String(dir), "out", entry); + const outputLines = readFileSync(outfile, "latin1").split("\n"); + const map = JSON.parse(readFileSync(outfile + ".map", "utf8")); + const segs = decodeMappings(map.mappings); + // For each statement: the mapping at the start of the output line it was + // printed on, i.e. what a stack trace through that line would be remapped to. + return (statements: string[]) => + statements.map(statement => { + const generatedLine = outputLines.findIndex(line => line.includes(statement)); + expect(generatedLine).toBeGreaterThanOrEqual(0); + const seg = referenceFind(segs, generatedLine, 0); + return { statement, mapped: seg && { source: map.sources[seg.srcIdx], line: seg.origLine } }; + }); + } + + const illFormed = [ + ["Latin-1 0xE9, a 3-byte lead with no continuation bytes", "\xe9"], + ["0xC3, a 2-byte lead with no continuation byte", "\xc3"], + ["0xF0 0x9F 0x92, three bytes of a 4-byte sequence", "\xf0\x9f\x92"], + ]; + + // Source lines 0-1 are the comment, `first` is on line 2 and `second` on line + // 5. The output drops the blank lines between them, so the two walks cannot + // each be a line off and still agree with each other by accident. + const rest = '\nb */\nconsole.log("first");\n\n\nconsole.log("second");\n'; + const statements = ['console.log("first")', 'console.log("second")']; + const expected = [ + { statement: 'console.log("first")', mapped: { source: "../in.js", line: 2 } }, + { statement: 'console.log("second")', mapped: { source: "../in.js", line: 5 } }, + ]; + + // A legal comment is copied into the output, so both walks see the bytes. + test.each(illFormed)("in a legal comment: %s", async (_name, bytes) => { + const mappingsOf = await bundle({ "in.js": "/*! caf" + bytes + rest }, "in.js"); + expect(mappingsOf(statements)).toEqual(expected); + }); + + // A plain comment is dropped from the output, so only the source walk sees them. + test.each(illFormed)("in a plain comment: %s", async (_name, bytes) => { + const mappingsOf = await bundle({ "in.js": "/* caf" + bytes + rest }, "in.js"); + expect(mappingsOf(statements)).toEqual(expected); + }); + + // In a bundle the files' mappings are chained: each file's start where the + // line count of the file printed before it left off, so a file that counts + // one of its own line breaks short also pulls every file after it up a line. + test("in a legal comment of a file printed ahead of the entry point", async () => { + const mappingsOf = await bundle( + { + "in.js": 'import "./dep.js";\nconsole.log("entry");\n', + "dep.js": '/*! caf\xe9\nb */\nconsole.log("dep");\n', + }, + "in.js", + ); + expect(mappingsOf(['console.log("dep")', 'console.log("entry")'])).toEqual([ + { statement: 'console.log("dep")', mapped: { source: "../dep.js", line: 2 } }, + { statement: 'console.log("entry")', mapped: { source: "../in.js", line: 1 } }, + ]); + }); +}); + // `sources` entries in the emitted map are URLs: the path from the chunk // directory to each source must use forward slashes on every platform, never // the host path separator. diff --git a/test/js/bun/sourcemap/internal-sourcemap.test.ts b/test/js/bun/sourcemap/internal-sourcemap.test.ts index b8efdae7770b..54dbde867927 100644 --- a/test/js/bun/sourcemap/internal-sourcemap.test.ts +++ b/test/js/bun/sourcemap/internal-sourcemap.test.ts @@ -55,7 +55,7 @@ function extractPositions(stack: string): string[] { .filter((s): s is string => s !== null); } -async function run(files: Record, env: Record = {}) { +async function run(files: Record, env: Record = {}) { using dir = tempDir("internal-sourcemap", files); await using proc = Bun.spawn({ cmd: [bunExe(), "index.ts"], @@ -227,4 +227,49 @@ describe("InternalSourceMap", () => { } expect(exited).toBe(0); }); + + // Both line tables behind a remapped frame are built by walking raw bytes: the + // source's by LineOffsetTable::generate, the transpiled output's by the builder. + // A lead byte without the continuation bytes it declares (the file is Latin-1, + // not UTF-8) is one character, one byte wide; stepping over the declared width + // instead swallows the line break after it and every frame below comes out a + // line early. The fixtures are written to disk as latin1, so "\xe9" is the one + // byte 0xE9. + describe.concurrent("an ill-formed UTF-8 byte right before a line break", () => { + async function framePositions(latin1Source: string) { + const { stdout, stderr, exited } = await run({ "index.ts": Buffer.from(latin1Source, "latin1") }); + expect(stderr).toBe(""); + expect(exited).toBe(0); + return [...stdout.matchAll(/index\.ts:(\d+:\d+)/g)].map(m => m[1]); + } + + test("in a comment that is dropped from the output", async () => { + // Lines: 1 "/* caf", 2 "b */", 3 the Error, 4 console.log. + const positions = await framePositions( + '/* caf\xe9\nb */\nconst err = new Error("x");\nconsole.log(err.stack);\n', + ); + expect(positions).toEqual(["3:17"]); + }); + + test("in a legal comment that is kept in the output", async () => { + // Lines: 1 "/*! caf", 2 "b */", 3 the Error, 6 console.log. The output + // has no blank lines, so the walks over the source and over the output + // cannot each be a line off and still produce the right answer together. + const positions = await framePositions( + '/*! caf\xe9\nb */\nconst err = new Error("x");\n\n\nconsole.log(err.stack);\n', + ); + expect(positions).toEqual(["3:17"]); + }); + + test("in a template literal, next to a U+FFFD the printer wrote out for another one", async () => { + // Line 1 breaks inside the template literal right after the ill-formed + // byte. The string on line 3 is printed with a real U+FFFD (three bytes, + // one column) ahead of the Error on the same line, which the walk over + // the output must not mistake for an ill-formed byte either. + const positions = await framePositions( + 'const s = `caf\xe9\n`;\nconst err = ["caf\xe9", new Error("x")][1];\nconsole.log(err.stack);\n', + ); + expect(positions).toEqual(["3:26"]); + }); + }); });