Bun.password: report an argon2 memory cost that cannot be allocated as OutOfMemory instead of aborting

src/runtime/crypto/pwhash.rs

@@ -96,6 +96,19 @@ pub(crate) mod argon2 {
     #[derive(Copy, Clone, Default)]
     pub(crate) struct VerifyOptions;
 
+    /// rust-argon2 allocates its `m` KiB block matrix infallibly (`Memory::new`), so
+    /// reserve that much fallibly first and report `OutOfMemory` instead of aborting.
+    pub(crate) fn check_memory_is_allocatable(mem_cost_kib: u32) -> Result<(), Error> {
+        let bytes = (mem_cost_kib as usize)
+            .checked_mul(1024)
+            .ok_or(bun_alloc::AllocError)?;
+        let mut probe: Vec<u8> = Vec::new();
+        probe
+            .try_reserve_exact(bytes)
+            .map_err(|_| bun_alloc::AllocError)?;
+        Ok(())
+    }
+
     fn map_err(e: &vendor::Error) -> Error {
         use vendor::Error as E;
         match e {
@@ -152,6 +165,7 @@ pub(crate) mod argon2 {
             version: vendor::Version::Version13,
         };
 
+        check_memory_is_allocatable(config.mem_cost)?;
         let encoded = vendor::hash_encoded(password, &salt, &config).map_err(|e| map_err(&e))?;
         let bytes = encoded.as_bytes();
 
@@ -211,6 +225,7 @@ pub(crate) mod argon2 {
             }
         };
 
+        let mut memory_cost: Option<u32> = None;
         if let Some(after_dollar) = normalised.as_bytes().strip_prefix(b"$") {
             if let Some(sep) = strings::index_of_char_usize(after_dollar, b'$') {
                 let mut rest = &after_dollar[sep + 1..];
@@ -244,9 +259,16 @@ pub(crate) mod argon2 {
                     if value > limit {
                         return Err(crate::Error::WeakParameters);
                     }
+                    if key == b"m" {
+                        memory_cost = Some(value);
+                    }
                 }
             }
         }
+        // Strings without exactly one `m=` fail to decode below, before allocating.
+        if let Some(memory_cost) = memory_cost {
+            check_memory_is_allocatable(memory_cost)?;
+        }
 
         match vendor::verify_encoded(&normalised, password) {
             Ok(true) => Ok(()),

test/js/bun/util/password.test.ts

@@ -1,6 +1,6 @@
 import { password } from "bun";
 import { describe, expect, test } from "bun:test";
-import { bunEnv, bunExe, isASAN, isDebug } from "harness";
+import { bunEnv, bunExe, isASAN, isDebug, isLinux } from "harness";
 
 const placeholder = "hey";
 
@@ -533,6 +533,86 @@ test("verify rejects encoded argon2 hashes with cost parameters above the suppor
   await expect(password.verify("correct horse", junkMemory)).rejects.toThrow("InvalidEncoding");
 });
 
+// argon2 allocates memoryCost KiB up front. When the system refuses that
+// allocation (memoryCost can go up to 4 TiB, and verify takes it from the
+// encoded hash) the result has to be the same error every other argon2 failure
+// produces, not a process abort. Both variants below run the same script in a
+// child whose allocations are made to fail: under ASAN through its
+// per-allocation cap, and on Linux release builds through a real address-space
+// limit, which makes mimalloc itself return null.
+describe("argon2 memory that cannot be allocated", () => {
+  const script = (memoryCost: number) => /* js */ `
+    const { password } = Bun;
+    const describeError = e => ({ name: e.name, code: e.code, message: e.message });
+    const options = { algorithm: "argon2id", memoryCost: ${memoryCost}, timeCost: 1 };
+    const small = { algorithm: "argon2id", memoryCost: 8, timeCost: 1 };
+    // A well-formed argon2id hash whose m= claims the unallocatable cost.
+    const encoded = password.hashSync("hunter2", small).replace("$m=8,", "$m=${memoryCost},");
+    const results = {};
+    try {
+      password.hashSync("hunter2", options);
+      results.hashSync = "resolved";
+    } catch (e) {
+      results.hashSync = describeError(e);
+    }
+    results.hash = await password.hash("hunter2", options).then(() => "resolved", describeError);
+    try {
+      results.verifySync = password.verifySync("hunter2", encoded);
+    } catch (e) {
+      results.verifySync = describeError(e);
+    }
+    results.verify = await password.verify("hunter2", encoded).then(v => v, describeError);
+    // Costs that do fit still work in this process afterwards.
+    results.afterwards = await password.verify("hunter2", await password.hash("hunter2", small));
+    console.log(JSON.stringify(results));
+  `;
+  const outOfMemory = (verb: string) => ({
+    name: "Error",
+    code: "PASSWORD_OUT_OF_MEMORY",
+    message: `Password ${verb} failed with error "OutOfMemory"`,
+  });
+  const expected = {
+    hashSync: outOfMemory("hashing"),
+    hash: outOfMemory("hashing"),
+    verifySync: outOfMemory("verification"),
+    verify: outOfMemory("verification"),
+    afterwards: true,
+  };
+
+  async function runChild(cmd: string[], env: NodeJS.Dict<string>) {
+    await using proc = Bun.spawn({ cmd, env, stdout: "pipe", stderr: "pipe" });
+    const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
+    // stderr is only informational: ASAN warns about every refused allocation.
+    return { result: JSON.parse(stdout.trim() || JSON.stringify({ stdout, stderr, exitCode })), exitCode };
+  }
+
+  test.skipIf(!isASAN)("is reported as PASSWORD_OUT_OF_MEMORY (ASAN allocation cap)", async () => {
+    // 65536 KiB (64 MiB) is also the default memoryCost; the cap refuses it.
+    const { result, exitCode } = await runChild([bunExe(), "-e", script(65536)], {
+      ...bunEnv,
+      ASAN_OPTIONS: [bunEnv.ASAN_OPTIONS, "allocator_may_return_null=1", "max_allocation_size_mb=32"]
+        .filter(Boolean)
+        .join(":"),
+    });
+    expect(result).toEqual(expected);
+    expect(exitCode).toBe(0);
+  });
+
+  test.skipIf(isASAN || !isLinux)("is reported as PASSWORD_OUT_OF_MEMORY (address-space limit)", async () => {
+    // A 4 GiB block (the largest cost verify accepts) can never fit in a process
+    // whose whole address space is limited to 4 GiB, while bun itself starts fine
+    // with far less. The limit applies before any page is touched, so nothing is
+    // actually consumed; an ASAN build cannot start under it, hence the skip.
+    const gib = 1024 * 1024; // in KiB, the unit of both ulimit -v and memoryCost
+    const { result, exitCode } = await runChild(
+      ["/bin/sh", "-c", `ulimit -v ${4 * gib} && exec "$0" "$@"`, bunExe(), "-e", script(4 * gib)],
+      bunEnv,
+    );
+    expect(result).toEqual(expected);
+    expect(exitCode).toBe(0);
+  });
+});
+
 test("verifySync reads the password buffer only after every argument has been coerced", () => {
   const hashed = password.hashSync("correct horse", { algorithm: "argon2id", memoryCost: 8, timeCost: 1 });
   const passwordBytes = new TextEncoder().encode("correct horse");