diff --git a/src/runtime/webcore/Blob.rs b/src/runtime/webcore/Blob.rs index 2710caf622ab..39b6adf824c8 100644 --- a/src/runtime/webcore/Blob.rs +++ b/src/runtime/webcore/Blob.rs @@ -3033,18 +3033,23 @@ impl BlobExt for Blob { if let Some(blob) = item.as_class_ref::() { could_have_non_ascii = could_have_non_ascii || blob.charset.get() != strings::AsciiStatus::AllAscii; + let view = blob.shared_view(); + if view.is_empty() { + if blob.needs_to_read_file() || blob.is_s3() { + push_file_part(blob, &mut joiner, global)?; + } + continue; + } // A later part may run user JS that drops the // last ref to this Blob's Store before `done()`. if parts_can_run_js { - joiner.push_cloned(blob.shared_view()); + joiner.push_cloned(view); } else { // SAFETY: the prescan above proved no // remaining part can run user JS, so this // Blob (rooted via `_keep`/`arg`) keeps its // Store alive until `joiner.done()` below. - joiner.push(unsafe { - bun_ptr::detach_lifetime(blob.shared_view()) - }); + joiner.push(unsafe { bun_ptr::detach_lifetime(view) }); } continue; } else { @@ -3072,7 +3077,12 @@ impl BlobExt for Blob { // This arm only handles entries deferred onto the walk // stack; other pending entries may still run user JS and // free this Blob's Store before `done()`, so always copy. - joiner.push_cloned(blob.shared_view()); + let view = blob.shared_view(); + if !view.is_empty() { + joiner.push_cloned(view); + } else if blob.needs_to_read_file() || blob.is_s3() { + push_file_part(blob, &mut joiner, global)?; + } } else { let utf8 = current.to_utf8(global)?; could_have_non_ascii = could_have_non_ascii || utf8.is_owned(); @@ -3266,6 +3276,159 @@ impl BlobExt for Blob { } } +/// The null device has no bytes, and a read of it cannot block. +fn is_null_device(stat: &bun_sys::Stat) -> bool { + if !bun_sys::S::ISCHR(stat.st_mode as _) { + return false; + } + // Linux fixes the null device at major 1, minor 3, so a sandbox with no /dev still finds it. + #[cfg(any(target_os = "linux", target_os = "android"))] + { + stat.st_rdev == libc::makedev(1, 3) + } + #[cfg(not(any(target_os = "linux", target_os = "android")))] + { + #[cfg(windows)] + let null_device = bun_core::zstr!("\\\\.\\NUL"); + #[cfg(not(windows))] + let null_device = bun_core::zstr!("/dev/null"); + bun_sys::stat(null_device).is_ok_and(|null| null.st_rdev == stat.st_rdev) + } +} + +/// Reads a file-backed part of a multi-part `new Blob([...])`: `shared_view()` is empty for it. +#[cold] +#[inline(never)] +fn push_file_part( + blob: &Blob, + joiner: &mut bun_core::string_joiner::StringJoiner, + global: &JSGlobalObject, +) -> JsResult<()> { + let Some(store) = blob.store.get() else { + return Ok(()); + }; + let file = match &store.data { + store::Data::Bytes(_) => return Ok(()), + store::Data::S3(_) => { + return Err(global.throw_invalid_arguments(format_args!( + "Blob parts backed by S3 cannot be read synchronously; await .bytes() or .arrayBuffer() first" + ))); + } + store::Data::File(file) => file, + }; + let throw = |err: bun_sys::Error| { + let err = match &file.pathlike { + PathOrFileDescriptor::Path(path) => err.with_path(path.slice()), + PathOrFileDescriptor::Fd(fd) => err.with_fd(*fd), + }; + global.throw_value(err.to_js(global)) + }; + + let no_bytes_or_refused = |stat: &bun_sys::Stat| -> JsResult<()> { + if is_null_device(stat) { + return Ok(()); + } + Err(global.throw_invalid_arguments(format_args!( + "Blob parts backed by a pipe, socket or device cannot be read synchronously; await .bytes() or .arrayBuffer() first" + ))) + }; + let is_file_or_dir = |mode| bun_sys::S::ISREG(mode) || bun_sys::S::ISDIR(mode); + + let (fd, _close_on_drop) = match &file.pathlike { + PathOrFileDescriptor::Fd(fd) => (*fd, None), + PathOrFileDescriptor::Path(path) => { + let mut path_buf = bun_paths::path_buffer_pool::get(); + let path = path.slice_z(&mut path_buf); + // open(2) of a FIFO wakes the writer that waits for a reader, so refuse it unopened. + match bun_sys::stat(path) { + Ok(stat) if !is_file_or_dir(stat.st_mode as _) => { + return no_bytes_or_refused(&stat); + } + _ => {} + } + // O_NONBLOCK: if the path is a FIFO by now, open(2) must not block the JS thread. + let flags = bun_sys::O::RDONLY + | bun_sys::O::NONBLOCK + | bun_sys::O::CLOEXEC + | bun_sys::O::NOCTTY; + match bun_sys::File::open(path, flags, 0) { + Ok(opened) => (opened.handle(), Some(opened)), + Err(err) => return Err(throw(err)), + } + } + }; + let stat = match bun_sys::fstat(fd) { + Ok(stat) => stat, + Err(err) => return Err(throw(err)), + }; + if bun_sys::S::ISDIR(stat.st_mode as _) { + return Err(throw(bun_sys::Error::from_code( + bun_sys::E::EISDIR, + bun_sys::Tag::read, + ))); + } + if !bun_sys::S::ISREG(stat.st_mode as _) { + return no_bytes_or_refused(&stat); + } + + let offset = blob.offset.get(); + let size = blob.size.get(); + let file_len = stat.st_size.max(0) as SizeType; + // st_size==0 may be a virtual file (procfs): grow-until-EOF instead of capping at 0. + let avail = if file_len > 0 { + file_len.saturating_sub(offset) as usize + } else { + usize::MAX + }; + let cap = if size != MAX_SIZE { + (size as usize).min(avail) + } else { + avail + }; + if cap == 0 { + return Ok(()); + } + // The join holds a part twice (this buffer, then the joined one), so half the RAM is the most that fits. + if file_len > 0 && cap >= bun_core::get_total_memory_size() / 2 { + return Err(throw(bun_sys::Error::oom())); + } + let initial = if file_len > 0 { cap } else { 8192.min(cap) }.min(8 << 30); + let mut buf: Vec = Vec::new(); + if buf.try_reserve_exact(initial).is_err() { + return Err(throw(bun_sys::Error::oom())); + } + buf.resize(initial, 0); + let mut total = 0usize; + loop { + if total == buf.len() { + if total == cap { + break; + } + let new_len = buf.len().saturating_mul(2).min(cap); + if buf.try_reserve(new_len - buf.len()).is_err() { + return Err(throw(bun_sys::Error::oom())); + } + buf.resize(new_len, 0); + } + // pread, not read: the cursor of a caller's fd must not decide what a repeated part reads. + let n = match bun_sys::pread( + fd, + &mut buf[total..], + (offset as i64).saturating_add(total as i64), + ) { + Ok(n) => n, + Err(err) => return Err(throw(err)), + }; + if n == 0 { + break; + } + total += n; + } + buf.truncate(total); + joiner.push_owned(buf.into_boxed_slice()); + Ok(()) +} + // ────────────────────────────────────────────────────────────────────────── // Basic accessors // ────────────────────────────────────────────────────────────────────────── diff --git a/test/js/web/fetch/blob.test.ts b/test/js/web/fetch/blob.test.ts index dc0c772681ef..0913d013457d 100644 --- a/test/js/web/fetch/blob.test.ts +++ b/test/js/web/fetch/blob.test.ts @@ -1,8 +1,11 @@ import { estimateShallowMemoryUsageOf } from "bun:jsc"; import { describe, expect, test } from "bun:test"; -import { bunEnv, bunExe, expectRssDeltaBelow, gcTick, isASAN, tempDir } from "harness"; +import { bunEnv, bunExe, expectRssDeltaBelow, gcTick, isASAN, isLinux, isWindows, tempDir } from "harness"; +import { mkfifo } from "mkfifo"; import type { BlobPropertyBag } from "node:buffer"; import type { BinaryLike } from "node:crypto"; +import fs from "node:fs"; +import { devNull, totalmem } from "node:os"; import path from "node:path"; test("blob: imports have sourcemapped stacktraces", async () => { @@ -1045,3 +1048,322 @@ test.each([ const blob = new Blob(["abc"], { type }); expect(blob.slice(0, 1).type).toBe(expected); }); + +describe("new Blob([...]) with a file-backed Blob part", () => { + async function check(blob: Blob, expected: string | Uint8Array) { + if (typeof expected === "string") { + const text = await blob.text(); + expect({ size: blob.size, text }).toEqual({ size: Buffer.byteLength(expected), text: expected }); + } else { + const bytes = await blob.bytes(); + expect({ size: blob.size, bytes }).toEqual({ size: expected.byteLength, bytes: expected }); + } + } + + test("contributes the file's bytes alongside other parts", async () => { + using dir = tempDir("blob-file-part", { "f.bin": "ABCDEFGHIJ" }); + const p = path.join(String(dir), "f.bin"); + const file = Bun.file(p); + + // single-part fast path (already worked; kept as baseline) + await check(new Blob([file]), "ABCDEFGHIJ"); + // file + string (after) + await check(new Blob([file, "-tail"]), "ABCDEFGHIJ-tail"); + // string + file (before) + await check(new Blob(["head-", file]), "head-ABCDEFGHIJ"); + // in-memory Blob + sliced file + await check(new Blob([new Blob(["M"]), file.slice(2, 6)]), "MCDEF"); + // File constructor + await check(new File([file, "!"], "n"), "ABCDEFGHIJ!"); + // file + typed array + file (same file twice) + await check(new Blob([file, new Uint8Array([0x2d]), file]), "ABCDEFGHIJ-ABCDEFGHIJ"); + // empty string sibling (must not take the single-part clone path) + await check(new Blob(["", file]), "ABCDEFGHIJ"); + // structuredClone of a Bun.file part + await check(new Blob(["<", structuredClone(file), ">"]), ""); + // Response.blob() over a Bun.file still has a file store + await check(new Blob(["<", await new Response(file).blob(), ">"]), ""); + }); + + test("Response, Request and Bun.write join the parts of an array the same way", async () => { + using dir = tempDir("blob-file-part-callers", { "f.bin": "ABCDEFGHIJ" }); + const file = Bun.file(path.join(String(dir), "f.bin")); + const out = path.join(String(dir), "out.bin"); + const body = (part: Blob) => ["x", part] as unknown as BodyInit; + expect({ + response: await new Response(body(file)).text(), + request: await new Request("http://localhost/", { method: "POST", body: body(file) }).text(), + written: await Bun.write(out, ["x", file]), + out: fs.readFileSync(out, "utf8"), + }).toEqual({ response: "xABCDEFGHIJ", request: "xABCDEFGHIJ", written: 11, out: "xABCDEFGHIJ" }); + + // The parts are joined before Bun.write makes its promise, so it throws and does not reject. + const missing = Bun.file(path.join(String(dir), "missing")); + const enoent = expect.objectContaining({ code: "ENOENT" }); + fs.rmSync(out); + expect(() => new Response(body(missing))).toThrow(enoent); + expect(() => new Request("http://localhost/", { method: "POST", body: body(missing) })).toThrow(enoent); + expect(() => Bun.write(out, ["x", missing])).toThrow(enoent); + expect(fs.existsSync(out)).toBe(false); + }); + + test("keeps every byte of the file", async () => { + const all = Uint8Array.from({ length: 256 }, (_, i) => i); + using dir = tempDir("blob-file-part-bytes", { "all.bin": Buffer.from(all), "utf8.txt": "héllo wörld ✓" }); + const bin = Bun.file(path.join(String(dir), "all.bin")); + await check(new Blob([bin, bin.slice(250), new Uint8Array([7])]), Uint8Array.of(...all, ...all.subarray(250), 7)); + await check(new Blob(["é", Bun.file(path.join(String(dir), "utf8.txt")), "✓"]), "éhéllo wörld ✓✓"); + }); + + // Two parts are made for each case: a read must not change what the other part gives. + test("a part gives the bytes that the same part gives when it is read alone", async () => { + using dir = tempDir("blob-file-part-alone", { "f.bin": "0123456789ABCDEFGHIJ" }); + const p = path.join(String(dir), "f.bin"); + const sized = () => { + const file = Bun.file(p); + void file.size; + return file; + }; + const windows: [number?, number?][] = [ + [], + [0], + [5], + [0, 5], + [5, 5], + [10, 5], + [0, 20], + [0, 1000], + [19], + [20], + [25, 30], + ]; + const changes = { + "unchanged": () => {}, + "grown": () => fs.appendFileSync(p, "zzzzz"), + "shrunk": () => fs.truncateSync(p, 8), + }; + for (const make of [() => Bun.file(p), sized]) { + for (const window of windows) { + for (const [change, apply] of Object.entries(changes)) { + fs.writeFileSync(p, "0123456789ABCDEFGHIJ"); + const [alone, part] = [make().slice(...window), make().slice(...window)]; + apply(); + const label = `${make === sized ? "sized " : ""}slice(${window}), file ${change}`; + expect({ label, bytes: await new Blob(["", part]).bytes() }).toEqual({ label, bytes: await alone.bytes() }); + } + } + } + }); + + test("fd-backed BunFile part", async () => { + using dir = tempDir("blob-fd-part", { "f.bin": "0123456789", "empty.bin": "" }); + const fd = fs.openSync(path.join(String(dir), "f.bin"), "r"); + const emptyFd = fs.openSync(path.join(String(dir), "empty.bin"), "r"); + try { + const f = Bun.file(fd); + await check(new Blob(["<", f, ">"]), "<0123456789>"); + // same fd used twice: each read must start at the Blob's offset, not + // wherever the previous part left the cursor + await check(new Blob([f, "-", f]), "0123456789-0123456789"); + await check(new Blob([f.slice(2, 6), "|", f.slice(7, 9)]), "2345|78"); + // slice end past EOF must clamp to the fd's actual size, not allocate to the declared end + await check(new Blob(["<", f.slice(0, 1e12), ">"]), "<0123456789>"); + // an empty fd with a huge slice (st_size==0 so avail is unbounded) must not over-allocate + await check(new Blob(["<", Bun.file(emptyFd).slice(0, 1e12), ">"]), "<>"); + if (!isWindows) { + // On POSIX pread(2) never touches the fd cursor, so the constructor + // must not mutate the caller's position. Windows ReadFile with an + // OVERLAPPED offset on a synchronous handle advances the pointer (a + // libuv/Node quirk), so this guarantee does not hold there. + const buf = Buffer.alloc(20); + const n = fs.readSync(fd, buf, 0, 20, null); + expect(buf.subarray(0, n).toString()).toBe("0123456789"); + } + } finally { + fs.closeSync(fd); + fs.closeSync(emptyFd); + } + }); + + test("throws when the file cannot be read", () => { + using dir = tempDir("blob-file-part-missing", {}); + const p = path.join(String(dir), "does-not-exist"); + const missing = expect.objectContaining({ code: "ENOENT", syscall: "open", path: p }); + expect(() => new Blob(["x", Bun.file(p)])).toThrow(missing); + // still throws after `.size` was accessed (which resolves to 0 for a + // nonexistent path) + const observed = Bun.file(p); + void observed.size; + expect(() => new Blob(["x", observed])).toThrow(missing); + // a slice with no bytes still names a file that must exist + expect(() => new Blob(["x", Bun.file(p).slice(3, 3)])).toThrow(missing); + // a directory + const directory = expect.objectContaining({ code: "EISDIR", path: String(dir) }); + expect(() => new Blob(["x", Bun.file(String(dir))])).toThrow(directory); + }); + + test("throws for an S3 part", () => { + const client = new Bun.S3Client({ + accessKeyId: "id", + secretAccessKey: "secret", + bucket: "bucket", + endpoint: "http://127.0.0.1:1", + }); + expect(() => new Blob(["x", client.file("key")])).toThrow( + "Blob parts backed by S3 cannot be read synchronously; await .bytes() or .arrayBuffer() first", + ); + }); + + test.skipIf(isWindows || process.getuid?.() === 0)("throws when the file is not readable", () => { + using dir = tempDir("blob-file-part-eacces", { "secret.bin": "secret" }); + const p = path.join(String(dir), "secret.bin"); + fs.chmodSync(p, 0o000); + expect(() => new Blob(["x", Bun.file(p)])).toThrow(expect.objectContaining({ code: "EACCES" })); + }); + + // The constructor reads on the JS thread, so it must never wait for a writer. + // A child process runs it: a blocked constructor would also block this file's timeout. + async function constructInChild(part: string, env: Record = {}) { + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + "-e", + ` + try { + console.log("built a Blob of", new Blob(["x", ${part}]).size, "bytes"); + } catch (e) { + console.log(e.message); + } + `, + ], + env: { ...bunEnv, ...env }, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + return { stdout: stdout.trim(), stderr, exitCode }; + } + const refused = { + stdout: + "Blob parts backed by a pipe, socket or device cannot be read synchronously; await .bytes() or .arrayBuffer() first", + stderr: "", + exitCode: 0, + }; + + test.concurrent("a pipe part throws instead of blocking", async () => { + expect(await constructInChild("Bun.stdin")).toEqual(refused); + }); + + // The null device has no bytes and a read of it cannot block. Node gives an empty part for it too. + test("a null device part is an empty part", async () => { + using dir = tempDir("blob-null-device-part", { "f.bin": "ABC" }); + const file = Bun.file(path.join(String(dir), "f.bin")); + const fd = fs.openSync(devNull, "r"); + try { + // The file part shows that the parts are read: a build that drops each file part gives "x" too. + await check(new Blob([Bun.file(devNull), file, "x"]), "ABCx"); + await check(new Blob(["x", file, Bun.file(fd)]), "xABC"); + await check(new File([file, Bun.file(devNull), "x"], "n"), "ABCx"); + await check(new Blob([Bun.file(devNull), "x"]), "x"); + await check(new Blob(["", Bun.file(devNull)]), ""); + } finally { + fs.closeSync(fd); + } + }); + + test.concurrent.skipIf(isWindows)("only the null device is an empty part", async () => { + using dir = tempDir("blob-null-device-link", { "f.bin": "ABC" }); + const link = path.join(String(dir), "null"); + fs.symlinkSync("/dev/null", link); + await check(new Blob([Bun.file(path.join(String(dir), "f.bin")), Bun.file(link), "x"]), "ABCx"); + // /dev/zero never ends, so a child process constructs the Blob. + expect(await constructInChild("Bun.file('/dev/zero')")).toEqual(refused); + }); + + test.concurrent.skipIf(isWindows)("a FIFO part throws and leaves its writer waiting", async () => { + using dir = tempDir("blob-fifo-part", {}); + const fifo = path.join(String(dir), "fifo"); + mkfifo(fifo); + // The writer says that it runs, then waits in open(2) until a reader opens the FIFO. + await using writer = Bun.spawn({ + cmd: ["sh", "-c", `echo ready; printf 'from the writer' > "$1"`, "sh", fifo], + stdout: "pipe", + stderr: "inherit", + }); + for await (const chunk of writer.stdout) { + expect(new TextDecoder().decode(chunk)).toBe("ready\n"); + break; + } + expect(await constructInChild("Bun.file(process.env.FIFO_PATH)", { FIFO_PATH: fifo })).toEqual(refused); + // An open of the FIFO by the constructor lets the writer run: it exits, or SIGPIPE kills it. + expect({ exitCode: writer.exitCode, signalCode: writer.signalCode }).toEqual({ exitCode: null, signalCode: null }); + // The next reader still gets the bytes of the writer. + await using reader = Bun.spawn({ cmd: ["cat", fifo], stdout: "pipe", stderr: "inherit" }); + expect(await reader.stdout.text()).toBe("from the writer"); + expect(await writer.exited).toBe(0); + }); + + // procfs gives a stat size of 0 for a file that has bytes, so the read must go on to the end of the file. + test.concurrent.skipIf(!isLinux)("reads a file whose stat size is 0 to its end", async () => { + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + "-e", + ` + const fs = require("fs"); + const environ = fs.readFileSync("/proc/self/environ"); + const blob = new Blob(["<", Bun.file("/proc/self/environ"), ">"]); + console.log(JSON.stringify({ + statSize: fs.statSync("/proc/self/environ").size, + moreThanOneChunk: environ.length > 8192, + size: blob.size - environ.length, + sameBytes: Buffer.from(await blob.bytes()).equals(Buffer.concat([Buffer.from("<"), environ, Buffer.from(">")])), + })); + `, + ], + env: { ...bunEnv, BLOB_PART_PADDING: Buffer.alloc(20_000, "x").toString() }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout: JSON.parse(stdout || "null"), stderr, exitCode }).toEqual({ + stdout: { statSize: 0, moreThanOneChunk: true, size: 2, sameBytes: true }, + stderr: "", + exitCode: 0, + }); + }); + + // The join holds a part twice: in the buffer of the read and in the joined buffer. + // The sparse file is 3/4 of the RAM, so one copy fits and two copies do not. + test.concurrent.skipIf(!isLinux)("a part too large for the RAM throws ENOMEM and reads nothing", async () => { + using dir = tempDir("blob-file-part-huge", { "sparse.bin": "x" }); + const sparse = path.join(String(dir), "sparse.bin"); + fs.truncateSync(sparse, Math.floor(totalmem() * 0.75)); + await using proc = Bun.spawn({ + cmd: [ + bunExe(), + "-e", + ` + let code; + try { + new Blob(["x", Bun.file(process.env.SPARSE_PATH)]); + } catch (e) { + code = e.code; + } + // maxRSS is in KiB. The first reserve of a read is 8 GiB. + console.log(JSON.stringify({ code, peakBelow4GiB: process.resourceUsage().maxRSS < 4 * 1024 * 1024 })); + `, + ], + env: { ...bunEnv, SPARSE_PATH: sparse }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect({ stdout: JSON.parse(stdout || "null"), stderr, exitCode }).toEqual({ + stdout: { code: "ENOMEM", peakBelow4GiB: true }, + stderr: "", + exitCode: 0, + }); + }); +});