feat: Incremental CBOR reader adoption

buildtools/ZipArtifactGenerator/manifests/net472.txt

@@ -12,7 +12,6 @@ extensions\CborProtocol\System.Runtime.CompilerServices.Unsafe.dll
 extensions\CborProtocol\System.Text.Encodings.Web.dll
 extensions\CborProtocol\System.Text.Json.dll
 extensions\CborProtocol\System.Threading.Tasks.Extensions.dll
-extensions\CborProtocol\System.ValueTuple.dll
 extensions\CloudFrontSigners\AWSSDK.Extensions.CloudFront.Signers.dll
 extensions\CloudFrontSigners\BouncyCastle.Cryptography.dll
 extensions\CloudFrontSigners\Microsoft.Bcl.AsyncInterfaces.dll

buildtools/ZipArtifactGenerator/manifests/net8.0.txt

@@ -1,6 +1,7 @@
 AWSSDK.Core.dll
 System.IO.Hashing.dll
 extensions\CborProtocol\AWSSDK.Extensions.CborProtocol.dll
+extensions\CborProtocol\Microsoft.Bcl.HashCode.dll
 extensions\CborProtocol\System.Formats.Cbor.dll
 extensions\CborProtocol\System.IO.Hashing.dll
 extensions\CloudFrontSigners\AWSSDK.Extensions.CloudFront.Signers.dll

extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetFramework.csproj

@@ -36,7 +36,7 @@
   </Choose>
 
   <ItemGroup>
-    <PackageReference Include="System.Formats.Cbor" Version="9.0.5" />
+    <PackageReference Include="System.Formats.Cbor" Version="11.0.0-rc.2.26504.105" />
   </ItemGroup>
 
   <ItemGroup>

extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetStandard.csproj

@@ -41,8 +41,9 @@
     </Otherwise>
   </Choose>  	
 
+  <!-- System.Formats.Cbor 11.0.0 is netstandard2.0, so all targets use it and its incremental APIs. -->
   <ItemGroup>
-    <PackageReference Include="System.Formats.Cbor" Version="9.0.5" />
+    <PackageReference Include="System.Formats.Cbor" Version="11.0.0-rc.2.26504.105" />
   </ItemGroup>
 
   <ItemGroup>

extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.nuspec

@@ -14,23 +14,23 @@
     <dependencies>
       <group targetFramework="net472">
         <dependency id="AWSSDK.Core" version="4.0.102.14" />
-        <dependency id="System.Formats.Cbor" version="9.0.5" />
+        <dependency id="System.Formats.Cbor" version="11.0.0-rc.2.26504.105" />
       </group>
       <group targetFramework="netstandard2.0">
         <dependency id="AWSSDK.Core" version="4.0.102.14" />
-        <dependency id="System.Formats.Cbor" version="9.0.5" />
+        <dependency id="System.Formats.Cbor" version="11.0.0-rc.2.26504.105" />
       </group>
       <group targetFramework="net8.0">
         <dependency id="AWSSDK.Core" version="4.0.102.14" />
-        <dependency id="System.Formats.Cbor" version="9.0.5" />
+        <dependency id="System.Formats.Cbor" version="11.0.0-rc.2.26504.105" />
       </group>
       <group targetFramework="net10.0">
         <dependency id="AWSSDK.Core" version="4.0.102.14" />
-        <dependency id="System.Formats.Cbor" version="9.0.5" />
+        <dependency id="System.Formats.Cbor" version="11.0.0-rc.2.26504.105" />
       </group>
       <group targetFramework="net11.0">
         <dependency id="AWSSDK.Core" version="4.0.102.14" />
-        <dependency id="System.Formats.Cbor" version="9.0.5" />
+        <dependency id="System.Formats.Cbor" version="11.0.0-rc.2.26504.105" />
       </group>
     </dependencies>
   </metadata>

extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs

@@ -1280,12 +1280,93 @@ public void CorruptArrayHeader_DeclaringHugeLength_DoesNotGrowBufferToDeclaredLe
         var bytes = new byte[] { 0x9A, 0x05, 0xF5, 0xE1, 0x00, 0x01, 0x02, 0x03 };
         using var reader = new CborStreamReader(new MemoryStream(bytes));
 
-        Assert.Throws<CborContentException>(() => reader.ReadStartArray());
+        // The incremental reader validates element counts lazily: the array header is well-formed, so
+        // ReadStartArray returns the declared count and the content error surfaces once the body runs out.
+        reader.ReadStartArray();
+        Assert.Throws<CborContentException>(() =>
+        {
+            for (int i = 0; i < 100_000_000; i++)
+                reader.ReadInt32();
+        });
 
         // The buffer grows only as received bytes arrive, so a header can't force a ~100 MB allocation.
         var buffer = (byte[])typeof(CborStreamReader)
             .GetField("_buffer", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!
             .GetValue(reader)!;
         Assert.True(buffer.Length <= 1024 * 1024, $"Read buffer grew to {buffer.Length} bytes for an 8-byte stream.");
     }
+
+    private static byte[] GetBuffer(CborStreamReader reader) =>
+        (byte[])typeof(CborStreamReader)
+            .GetField("_buffer", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)!
+            .GetValue(reader)!;
+
+    [Fact]
+    public void MalformedByte_DoesNotBufferRestOfStream()
+    {
+        // [ 0x82 = array(2), 0x1C = reserved additional info (malformed) ] + 2 MB padding.
+        var bytes = new byte[2 * 1024 * 1024];
+        bytes[0] = 0x82;
+        bytes[1] = 0x1C;
+        var stream = new MemoryStream(bytes);
+        using var reader = new CborStreamReader(stream);
+
+        reader.ReadStartArray();
+        Assert.Throws<CborContentException>(() => reader.ReadInt32());
+
+        // Fail fast: a malformed token must not buffer the whole stream first.
+        Assert.True(stream.Position < 2 * AWSConfigs.CborReaderInitialBufferSize,
+            $"Stream advanced to {stream.Position} bytes on a malformed token; expected an early failure.");
+        Assert.True(GetBuffer(reader).Length < 2 * 1024 * 1024,
+            "Read buffer grew toward the full stream size on a malformed token.");
+    }
+
+    [Fact]
+    public void SkipValue_OverLargeSubtree_SpanningManyChunks_Succeeds()
+    {
+        // A valid array far larger than the buffer, so SkipValue spans many refills.
+        var writer = new CborWriter();
+        writer.WriteStartArray(500);
+        for (int i = 0; i < 500; i++)
+            writer.WriteTextString($"element-with-some-length-{i}");
+        writer.WriteEndArray();
+        var payload = writer.Encode();
+        Assert.True(payload.Length > 10 * AWSConfigs.CborReaderInitialBufferSize);
+
+        // Wrap so there is a value after the skipped one to confirm the position.
+        var outer = new CborWriter();
+        outer.WriteStartArray(2);
+        outer.WriteEncodedValue(payload);
+        outer.WriteInt32(42);
+        outer.WriteEndArray();
+
+        using var reader = new CborStreamReader(new MemoryStream(outer.Encode()));
+        reader.ReadStartArray();
+        reader.SkipValue();
+        Assert.Equal(42, reader.ReadInt32());
+        reader.ReadEndArray();
+    }
+
+    [Fact]
+    public void LargeValidPayload_StreamsWithoutBufferingWholePayload()
+    {
+        // A valid payload larger than the buffer should read correctly while the buffer stays bounded.
+        const int count = 2000;
+        var writer = new CborWriter();
+        writer.WriteStartArray(count);
+        for (int i = 0; i < count; i++)
+            writer.WriteInt32(i);
+        writer.WriteEndArray();
+        var payload = writer.Encode();
+        Assert.True(payload.Length > 20 * AWSConfigs.CborReaderInitialBufferSize);
+
+        using var reader = new CborStreamReader(new MemoryStream(payload));
+        Assert.Equal(count, reader.ReadStartArray());
+        for (int i = 0; i < count; i++)
+            Assert.Equal(i, reader.ReadInt32());
+        reader.ReadEndArray();
+
+        Assert.True(GetBuffer(reader).Length < payload.Length,
+            $"Buffer ({GetBuffer(reader).Length}) should stay below the full payload size ({payload.Length}).");
+    }
 }