fix: reject lone surrogates on both the parse and generate sides

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Parsing: reject surrogate code points in `\UXXXXXXXX` escapes. Only the 4-digit `\uXXXX` form was checked; the 8-digit form let a lone surrogate through, since `String.fromCodePoint` accepts one. ([#261])
+- Stringify/Patching: reject unpaired surrogates in string values and keys instead of emitting a document that is not valid UTF-8. Valid astral characters (surrogate *pairs*) are unaffected. ([#261])
+
 ## [3.0.0] - 2026-07-29
 
 ### Changed
@@ -327,4 +332,5 @@ This first forked version from [timhall/toml-patch](https://github.com/timhall/t
 [#253]: https://github.com/DecimalTurn/toml-patch/pull/253
 [#259]: https://github.com/DecimalTurn/toml-patch/pull/259
 [#260]: https://github.com/DecimalTurn/toml-patch/pull/260
-[0e66e68]: https://github.com/DecimalTurn/toml-patch/commit/0e66e68cbf42a07bc44445e46c3ea7bea97f95c1
+[#261]: https://github.com/DecimalTurn/toml-patch/pull/261
+[0e66e68]: https://github.com/DecimalTurn/toml-patch/commit/0e66e68cbf42a07bc44445e46c3ea7bea97f95c1

src/__tests__/parse-string.test.ts

@@ -62,3 +62,42 @@ test('should handle odd/even preceding backslashes for \\e', () => {
   // 3 backslashes then e => one literal backslash + ESC
   expect(parseString('"\\\\\\e"')).toBe('\\' + '\u001b');
 });
+
+// Surrogate code points are not Unicode scalar values, so they have no valid UTF-8 encoding
+// and cannot appear in a TOML document — in either escape form, paired or not. The 8-digit
+// form needs an explicit check: unlike an out-of-range code point, String.fromCodePoint
+// accepts a surrogate and simply yields an unpaired code unit.
+
+test('should reject a lone high surrogate in a 4-digit unicode escape', () => {
+  expect(() => parseString('"\\uD800"')).toThrow(/surrogates not allowed/);
+});
+
+test('should reject a lone low surrogate in a 4-digit unicode escape', () => {
+  expect(() => parseString('"\\uDFFF"')).toThrow(/surrogates not allowed/);
+});
+
+test('should reject a surrogate pair written as two 4-digit unicode escapes', () => {
+  // These two would combine into a valid astral character in UTF-16, but TOML forbids the
+  // escapes themselves — the 8-digit form is the correct spelling.
+  expect(() => parseString('"\\uD83D\\uDE00"')).toThrow(/surrogates not allowed/);
+});
+
+test('should reject a lone high surrogate in an 8-digit unicode escape', () => {
+  expect(() => parseString('"\\U0000D800"')).toThrow(/surrogates not allowed/);
+});
+
+test('should reject a lone low surrogate in an 8-digit unicode escape', () => {
+  expect(() => parseString('"\\U0000DFFF"')).toThrow(/surrogates not allowed/);
+});
+
+test('should reject a surrogate in an 8-digit unicode escape inside a multiline string', () => {
+  expect(() => parseString('"""\\U0000D800"""')).toThrow(/surrogates not allowed/);
+});
+
+test('should still accept a valid astral character via an 8-digit unicode escape', () => {
+  expect(parseString('"\\U0001F600"')).toBe('\u{1F600}');
+});
+
+test('should still reject an out-of-range 8-digit unicode escape', () => {
+  expect(() => parseString('"\\U00110000"')).toThrow();
+});

src/__tests__/patch.test.ts

@@ -5708,12 +5708,45 @@ describe('comment removal with section', () => {
 
 });
 
-describe('lone surrogate handling in stringify', () => {
+describe('lone surrogate handling in stringify and patch', () => {
 
-  test.skip('should reject lone surrogates instead of emitting invalid TOML', () => {
-    // Lone surrogates are not valid Unicode scalar values.
-    // stringify should throw rather than emit \\uD800.
-    expect(() => stringify({ s: '\ud800' })).toThrow();
+  // JS strings are UTF-16, so an astral character is legitimately stored as a surrogate
+  // *pair* — those must keep working. An *unpaired* surrogate is not a Unicode scalar value,
+  // has no valid UTF-8 encoding, and so cannot be represented in TOML at all. Rejected at the
+  // two generation choke points (generateString for values, generateKey for keys), which both
+  // stringify and patch funnel through.
+  const HIGH = '\ud800';
+  const LOW = '\udfff';
+
+  test('should reject lone surrogates instead of emitting invalid TOML', () => {
+    expect(() => stringify({ s: HIGH })).toThrow(/lone surrogate \(U\+D800\)/);
+  });
+
+  test('should reject a lone low surrogate', () => {
+    expect(() => stringify({ s: LOW })).toThrow(/lone surrogate \(U\+DFFF\)/);
+  });
+
+  test('should reject a lone surrogate nested in a table or array', () => {
+    expect(() => stringify({ a: { b: [HIGH] } })).toThrow(/lone surrogate/);
+  });
+
+  test('should reject a lone surrogate in a key', () => {
+    expect(() => stringify({ [HIGH]: 1 })).toThrow(/lone surrogate/);
+  });
+
+  test('should reject a lone surrogate when patching an existing value', () => {
+    expect(() => patch('s = "ok"\n', { s: HIGH })).toThrow(/lone surrogate/);
+  });
+
+  test('should reject a lone surrogate when patching in a new key', () => {
+    expect(() => patch('a = 1\n', { a: 1, s: HIGH })).toThrow(/lone surrogate/);
+    expect(() => patch('a = 1\n', { a: 1, [HIGH]: 2 })).toThrow(/lone surrogate/);
+  });
+
+  test('should still accept a valid astral character (surrogate pair)', () => {
+    expect(stringify({ s: '\u{1F600}' })).toBe('s = "\u{1F600}"\n');
+    expect(patch('s = "ok"\n', { s: '\u{1F600}' })).toBe('s = "\u{1F600}"\n');
+    expect(patch('s = "\u{1F600}"\n', parse('s = "\u{1F600}"\n'))).toBe('s = "\u{1F600}"\n');
   });
 
 });

src/__tests__/surrogate-validation.test.ts

@@ -0,0 +1,122 @@
+import { describe, test, expect, vi } from 'vitest';
+import { parse, stringify } from '../';
+import patch from '../patch';
+
+// A backslash built at runtime, so the escape sequences under test are unambiguous in source.
+const BS = String.fromCharCode(92);
+const GRIN = '\u{1F600}'; // U+1F600, stored as the surrogate pair D83D DE00
+
+describe('round-trip of characters that need surrogate pairs', () => {
+  // An astral character is a *pair* of surrogates in a UTF-16 JS string, which is well-formed
+  // and must keep working — the lone-surrogate rejection must not catch these.
+
+  test('should round-trip a raw astral character through parse and stringify', () => {
+    const doc = `s = "${GRIN}"\n`;
+    expect(parse(doc)).toEqual({ s: GRIN });
+    expect(stringify(parse(doc))).toBe(doc);
+    expect(patch(doc, parse(doc))).toBe(doc);
+  });
+
+  test('should preserve an 8-digit escape for an astral character when patching', () => {
+    // escape-preference keeps the document's existing spelling rather than expanding it to the
+    // raw character, so an identity patch is byte-identical.
+    const doc = `s = "${BS}U0001F600"\n`;
+    expect(parse(doc)).toEqual({ s: GRIN });
+    expect(patch(doc, parse(doc))).toBe(doc);
+    expect(patch(doc, { s: GRIN })).toBe(doc);
+  });
+
+  test('should keep the escaped spelling when the surrounding value changes', () => {
+    const doc = `s = "${BS}U0001F600"\n`;
+    expect(patch(doc, { s: `x${GRIN}` })).toBe(`s = "x${BS}U0001F600"\n`);
+  });
+
+  test('should emit the raw character from stringify, which has no source spelling to keep', () => {
+    // stringify gets no existing raw text to learn a preference from, so the shortest valid
+    // form wins. Contrast with patch above.
+    expect(stringify({ s: GRIN })).toBe(`s = "${GRIN}"\n`);
+  });
+});
+
+describe('a lone surrogate cannot round-trip in any spelling', () => {
+  const LONE = '\ud800';
+
+  // There is deliberately no "keep it escaped" behaviour here. TOML forbids surrogate escapes
+  // outright (see toml-test invalid/string/bad-uni-esc-06.toml, which is `\uD801`), so emitting
+  // s = "\uD800" would produce a document this library — and any conforming parser — rejects.
+  // A lone surrogate also has no UTF-8 encoding, so the raw form is not valid TOML either.
+  // With neither spelling available, the value simply cannot be represented.
+
+  test('should reject the escaped spelling on the way in', () => {
+    expect(() => parse(`s = "${BS}uD800"`)).toThrow(/surrogates not allowed/);
+    expect(() => parse(`s = "${BS}U0000D800"`)).toThrow(/surrogates not allowed/);
+  });
+
+  test('should refuse to emit a value holding a lone surrogate', () => {
+    expect(() => stringify({ s: LONE })).toThrow(/lone surrogate/);
+    expect(() => patch('s = "ok"\n', { s: LONE })).toThrow(/lone surrogate/);
+  });
+
+  test('should not emit an escaped surrogate that it would then reject on re-parse', () => {
+    // Guards the tempting "just escape it" fix: whatever we emit must be re-readable.
+    let emitted: string | undefined;
+    try {
+      emitted = stringify({ s: LONE });
+    } catch {
+      emitted = undefined;
+    }
+    if (emitted !== undefined) {
+      expect(() => parse(emitted!)).not.toThrow();
+    }
+  });
+});
+
+// Simulates Node 16-19, where String.prototype.isWellFormed does not exist, to confirm the
+// scan fallback still detects and reports lone surrogates identically.
+describe('fallback when isWellFormed is unavailable', () => {
+  test('behaves the same without the native method', async () => {
+    const original = String.prototype.isWellFormed;
+    const results: Record<string, string[]> = {};
+
+    for (const mode of ['native', 'fallback'] as const) {
+      if (mode === 'fallback') {
+        // @ts-expect-error — deliberately removing to emulate an older runtime
+        delete String.prototype.isWellFormed;
+      } else if (original) {
+        String.prototype.isWellFormed = original;
+      }
+
+      vi.resetModules();
+      const { assertNoLoneSurrogate } = await import('../utils');
+
+      const observed: string[] = [];
+      for (const [label, value] of [
+        ['clean ascii', 'hello'],
+        ['astral pair', '\u{1F600}'],
+        ['lone high at 0', '\ud800'],
+        ['lone low at 2', 'ab\udfff'],
+        ['lone high after pair', '\u{1F600}\ud800'],
+      ] as [string, string][]) {
+        try {
+          assertNoLoneSurrogate(value, 'String value');
+          observed.push(`${label}: ok`);
+        } catch (e: any) {
+          observed.push(`${label}: ${e.message.split('.')[0]}`);
+        }
+      }
+      results[mode] = observed;
+    }
+
+    if (original) String.prototype.isWellFormed = original;
+
+    // Both modes must agree exactly, including the reported index and code point.
+    expect(results.fallback).toEqual(results.native);
+    expect(results.native).toEqual([
+      'clean ascii: ok',
+      'astral pair: ok',
+      'lone high at 0: String value contains a lone surrogate (U+D800) at index 0',
+      'lone low at 2: String value contains a lone surrogate (U+DFFF) at index 2',
+      'lone high after pair: String value contains a lone surrogate (U+D800) at index 2',
+    ]);
+  });
+});

src/generate.ts

@@ -29,7 +29,7 @@ import { shiftNode } from './writer';
 import { rebuildLineContinuation } from './line-ending-backslash';
 import { IS_BARE_KEY } from './tokenizer';
 import { escapeStringContent } from './escape-preference';
-import {isBasicString, isMultilineBasicString, isLiteralString, isMultilineLiteralString, temporalToTomlString} from './utils';
+import {isBasicString, isMultilineBasicString, isLiteralString, isMultilineLiteralString, temporalToTomlString, assertNoLoneSurrogate} from './utils';
 
 /**
  * Generates a new TOML document node.
@@ -138,7 +138,11 @@ function quoteTomlString(value: string): string {
 }
 
 function keyValueToRaw(value: string[]): string {
-  return value.map(part => (IS_BARE_KEY.test(part) ? part : quoteTomlString(part))).join('.');
+  return value.map(part => {
+    // Keys are encoded too, so a lone surrogate is just as invalid here as in a value.
+    assertNoLoneSurrogate(part, `Key "${part}"`);
+    return IS_BARE_KEY.test(part) ? part : quoteTomlString(part);
+  }).join('.');
 }
 
 export function generateKey(value: string[]): Key {
@@ -160,6 +164,10 @@ export function generateKey(value: string[]): Key {
  * @returns A new String node.
  */
 export function generateString(value: string, existingRaw?: string): String {
+  // Single choke point for string values from both stringify and patch — reject unpaired
+  // surrogates here rather than emitting a document that isn't valid UTF-8.
+  assertNoLoneSurrogate(value, 'String value');
+
   if (!existingRaw) {
     return generateBasicString(value);
   }

src/parse-string.ts

@@ -100,6 +100,12 @@ function unescapeBasicString(value: string, multiline: boolean): string {
           }
         }
         const cp = parseInt(value.slice(i + 1, i + 9), 16);
+        // Same restriction as the \u form above. Unlike an out-of-range code point, a
+        // surrogate is accepted by String.fromCodePoint (it just yields an unpaired code
+        // unit), so it has to be rejected explicitly.
+        if (cp >= 0xD800 && cp <= 0xDFFF) {
+          throw new Error(`Invalid \\U${value.slice(i + 1, i + 9)}: surrogates not allowed`);
+        }
         parts.push(String.fromCodePoint(cp));
         i += 8;
         break;

src/utils.ts

@@ -199,4 +199,63 @@ export function merge<TValue>(target: TValue[], values: TValue[]) {
   }
 }
 
+/**
+ * `String.prototype.isWellFormed`, when the runtime has it (Node 20+; this package supports
+ * Node 16, so it has to be optional). Returns false exactly when a string contains an unpaired
+ * surrogate — the same predicate as `findLoneSurrogate`, but far cheaper.
+ * 
+ * It only yields a boolean, so locating the offending unit for the error message still needs
+ * `findLoneSurrogate` — but that only runs on the failing path.
+ */
+const nativeIsWellFormed: ((this: string) => boolean) | undefined =
+  typeof String.prototype.isWellFormed === 'function' ? String.prototype.isWellFormed : undefined;
+
+/**
+ * Finds the first unpaired surrogate code unit in `value`, if any.
+ *
+ * JS strings are UTF-16, so an astral character is legitimately stored as a high/low surrogate
+ * *pair* — those are fine. An unpaired one is not a Unicode scalar value, so it has no valid
+ * UTF-8 encoding and cannot be represented in TOML.
+ */
+function findLoneSurrogate(value: string): { index: number; code: number } | undefined {
+  for (let i = 0; i < value.length; i++) {
+    const code = value.charCodeAt(i);
+    if (code < 0xd800 || code > 0xdfff) continue;
+
+    // A high surrogate is valid only when immediately followed by a low one.
+    if (code <= 0xdbff) {
+      const next = value.charCodeAt(i + 1); // NaN past the end — fails the range test below
+      if (next >= 0xdc00 && next <= 0xdfff) {
+        i++; // consume the pair
+        continue;
+      }
+    }
+
+    // Either an unpaired high surrogate, or a low surrogate with no high before it (a paired
+    // low is always consumed by the branch above).
+    return { index: i, code };
+  }
+
+  return undefined;
+}
+
+/**
+ * Throws if `value` contains an unpaired surrogate. `describe` names what is being encoded
+ * (e.g. `'String value'`, `'Key "a"'`) so the message points at the offending input.
+ */
+export function assertNoLoneSurrogate(value: string, describe: string): void {
+  // Fast path for the overwhelmingly common case of a clean string. When the native check is
+  // missing we fall through and scan, which is the same work as before.
+  if (nativeIsWellFormed !== undefined && nativeIsWellFormed.call(value)) return;
+
+  const found = findLoneSurrogate(value);
+  if (!found) return;
+
+  const hex = found.code.toString(16).toUpperCase().padStart(4, '0');
+  throw new Error(
+    `${describe} contains a lone surrogate (U+${hex}) at index ${found.index}. ` +
+    `Unpaired surrogates are not Unicode scalar values and cannot be encoded as TOML.`
+  );
+}
+