sourcemap: decode a VLQ written with more than 8 digits

src/base64/lib.rs

@@ -314,55 +314,98 @@ pub mod vlq {
         bytes
     };
 
-    // Shared body for `decode` / `decode_assume_valid` (which differ only by
-    // two asserts); const-generic `ASSERT_VALID` is const-folded so codegen
-    // matches hand-duplicated bodies.
-    // PERF: loop is not unrolled — profile if hot.
+    // Shared body of `decode` and `decode_assume_valid`; `ASSERT_VALID` is const-folded.
     #[inline(always)]
     fn decode_impl<const ASSERT_VALID: bool>(encoded: &[u8], start: usize) -> VLQResult {
-        let mut shift: u8 = 0;
+        let digits = &encoded[start..];
         let mut vlq: u32 = 0;
+        // Keep one value: a literal at each `return` makes call sites re-test `start` per field.
+        let no_progress = VLQResult { start, value: 0 };
+
+        // Unrolled by hand: a loop is only unrolled where the optimizer chooses to.
+        macro_rules! digit {
+            ($i:literal) => {
+                if digits.len() <= $i {
+                    // Empty, or the input ends mid-VLQ. `start` unchanged makes callers fail.
+                    return no_progress;
+                }
+                if ASSERT_VALID {
+                    debug_assert!(digits[$i] < U7_MAX); // invalid base64 character
+                }
+                // mask to 7 bits
+                let index = BASE64_LUT[(digits[$i] & 0x7f) as usize] as u32;
+                if ASSERT_VALID {
+                    debug_assert!(index != U7_MAX as u32); // invalid base64 character
+                }
 
-        // hint to the compiler what the maximum value is
-        let encoded_ = &encoded[start..][0..(encoded.len() - start).min(VLQ_MAX_IN_BYTES + 1)];
+                // No overflow check: payload past bit 31 is dropped (7th digit) or wraps (8th).
+                vlq |= (index & 31) << ((5 * $i) & 31);
 
-        // inlining helps for the 1 or 2 byte case, hurts a little for larger
-        for i in 0..encoded_.len() {
-            if ASSERT_VALID {
-                debug_assert!(encoded_[i] < U7_MAX); // invalid base64 character
-            }
-            // mask to 7 bits
-            let index = BASE64_LUT[(encoded_[i] & 0x7f) as usize] as u32;
-            if ASSERT_VALID {
-                debug_assert!(index != U7_MAX as u32); // invalid base64 character
+                // Stop if there's no continuation bit
+                if (index & 32) == 0 {
+                    return VLQResult {
+                        start: start + $i + 1,
+                        value: from_sign_magnitude(vlq),
+                    };
+                }
+            };
+        }
+        digit!(0);
+        digit!(1);
+        digit!(2);
+        digit!(3);
+        digit!(4);
+        digit!(5);
+        digit!(6);
+        digit!(7);
+
+        debug_assert!(!ASSERT_VALID); // `VLQ::encode` writes at most 7 digits
+        if !ASSERT_VALID && digits.len() > VLQ_MAX_IN_BYTES + 1 {
+            return decode_long(encoded, start);
+        }
+        no_progress
+    }
+
+    /// A VLQ of more than 8 digits: any number of digits, but the value has to fit in 32 bits.
+    #[cold]
+    #[inline(never)]
+    fn decode_long(encoded: &[u8], start: usize) -> VLQResult {
+        let mut vlq: u64 = 0;
+        for (i, &byte) in encoded[start..].iter().enumerate() {
+            let index = BASE64_LUT[(byte & 0x7f) as usize] as u64;
+            // Not base64. Node reads it as a terminal digit: https://github.com/nodejs/node/blob/v26.3.0/lib/internal/source_map/source_map.js#L331-L357
+            if byte > U7_MAX || index == U7_MAX as u64 {
+                break;
             }
 
-            // decode a byte
-            vlq |= (index & 31) << (shift & 31);
-            shift += 5;
+            // From the 8th digit on, every payload bit is past bit 31.
+            vlq |= (index & 31) << (5 * i).min(32);
+            // Over 32 bits: https://tc39.es/ecma426/#sec-VLQUnsignedValue throws, Node wraps.
+            if vlq > u32::MAX as u64 {
+                break;
+            }
 
-            // Stop if there's no continuation bit
             if (index & 32) == 0 {
                 return VLQResult {
                     start: start + i + 1,
-                    value: if (vlq & 1) == 0 {
-                        (vlq >> 1) as i32
-                    } else {
-                        -((vlq >> 1) as i32)
-                    },
+                    value: from_sign_magnitude(vlq as u32),
                 };
             }
         }
-
-        // Reached when the input is empty or ends mid-VLQ (the last byte's
-        // continuation bit is set with no following byte, or all 8 bytes have
-        // it set — both malformed). No value was decoded; return `start`
-        // unchanged so callers' no-progress checks treat the truncated
-        // mapping as a parse failure instead of silently accepting `value: 0`.
         VLQResult { start, value: 0 }
     }
 
-    #[inline]
+    /// Minus zero is 0 as in esbuild, not -2^31 as in https://tc39.es/ecma426/#sec-VLQSignedValue
+    #[inline(always)]
+    fn from_sign_magnitude(vlq: u32) -> i32 {
+        if (vlq & 1) == 0 {
+            (vlq >> 1) as i32
+        } else {
+            -((vlq >> 1) as i32)
+        }
+    }
+
+    #[inline(always)]
     pub fn decode(encoded: &[u8], start: usize) -> VLQResult {
         decode_impl::<false>(encoded, start)
     }
@@ -394,6 +437,72 @@ pub mod vlq {
             let encoded = VLQ::encode(i32::MIN);
             assert_eq!(decode(encoded.slice(), 0).value, 0);
         }
+
+        /// `value` in `digits` digits: those of `VLQ::encode`, then `g` padding, then `A`.
+        fn padded(value: i32, digits: usize) -> Vec<u8> {
+            let mut out = VLQ::encode(value).slice().to_vec();
+            let last = out.last_mut().unwrap();
+            *last = BASE64[(BASE64_LUT[*last as usize] | 32) as usize];
+            out.resize(digits - 1, b'g');
+            out.push(b'A');
+            out
+        }
+
+        fn decoded(encoded: &[u8], start: usize) -> (i32, usize) {
+            let result = decode(encoded, start);
+            (result.value, result.start)
+        }
+
+        #[test]
+        fn decode_has_no_limit_on_the_digit_count() {
+            for value in [0, 1, -1, 255, -256, i32::MAX, i32::MIN + 1] {
+                for digits in [8, 9, 10, 64] {
+                    assert_eq!(
+                        decoded(&padded(value, digits), 0),
+                        (value, digits),
+                        "{value} in {digits} digits"
+                    );
+                }
+            }
+            // `mappings` of the tc39 test validMappingLargeVLQ.
+            assert_eq!(decoded(&padded(1, 1987), 0), (1, 1987));
+            // The 7th digit has bits 30 and 31.
+            assert_eq!(decoded(b"ggggggjggA", 0), (3 << 29, 10));
+
+            // The VLQ ends at its terminal digit, and `start` can be past 0.
+            let mut encoded = b"AC".to_vec();
+            encoded.extend(padded(-7, 12));
+            encoded.extend(b"C,AAAA");
+            assert_eq!(decoded(&encoded, 2), (-7, 14));
+        }
+
+        #[test]
+        fn decode_does_not_check_a_vlq_of_up_to_eight_digits() {
+            // Payload past bit 31 is dropped in the 7th digit and wraps in the 8th.
+            assert_eq!(decoded(b"ggggggE", 0), (0, 7));
+            assert_eq!(decoded(b"gggggggC", 0), (8, 8));
+            // A byte that is not base64 is a continuation digit with payload 31.
+            assert_eq!(decoded(b"!A", 0), (-15, 2));
+        }
+
+        #[test]
+        fn decode_makes_no_progress_on_a_long_vlq_without_a_value() {
+            for encoded in [
+                // The input ends before a terminal digit.
+                &[b'g'; 9][..],
+                &[b'g'; 100][..],
+                // A byte that is not a base64 digit comes before it.
+                b"gggggggggg,AAAA",
+                b"gggggggggg!A",
+                b"gggggggggg\xC1",
+                // The value does not fit in 32 bits.
+                b"ggggggggB",
+                b"ggggggggggggghA",
+                b"ggggggkggA",
+            ] {
+                assert_eq!(decoded(encoded, 0), (0, 0), "{}", encoded.escape_ascii());
+            }
+        }
     }
 }
 

src/jsc/bindings/highway_sourcemap.cpp

@@ -846,8 +846,7 @@ size_t ParseMappingsImpl(const uint8_t* HWY_RESTRICT bytes, size_t len,
             // The LAST sextet of a VLQ has cont==0, so a VLQ whose final
             // byte still has cont set is truncated by the delimiter.
             // And seg_len > 10 means at least one VLQ is > 2 chars for
-            // field_count 5 (or > 7 chars for field_count 1, which the
-            // scalar decoder rejects as no-progress); hand both to scalar.
+            // field_count 5 (or > 7 chars for field_count 1); hand both to scalar.
             if (HWY_UNLIKELY(((cont >> (seg_len - 1)) & 1) || seg_len > 10))
                 goto bail;
             const uint64_t seg_cont = cont & ((uint64_t { 1 } << seg_len) - 1);
@@ -900,12 +899,7 @@ size_t ParseMappingsImpl(const uint8_t* HWY_RESTRICT bytes, size_t len,
                     d_name = DecodeVlqSextets(sextets, cont_bits, q, p + seg_len);
             } else if (field_count == 1) {
                 // 1-field segments don't emit a row (scalar skips them),
-                // only gen-col moves. seg_len <= 10; a 1-field VLQ of >= 9
-                // sextets (>= 8 continuation bytes) makes the scalar
-                // decoder return no-progress, so bail to scalar for
-                // byte-identical error reporting. Bailing at > 7 is
-                // conservative for seg_len == 8 (scalar succeeds there)
-                // and keeps DecodeVlqSextets' shift <= 30.
+                // only gen-col moves. Scalar takes > 7 sextets: DecodeVlqSextets shifts <= 30.
                 if (HWY_UNLIKELY(seg_len > 7))
                     goto bail;
                 size_t q = p;