From a768a4dab0b0e49a4a7c94dca71eb1ec5f05967a Mon Sep 17 00:00:00 2001 From: Sichan Yoo Date: Fri, 16 Oct 2026 10:37:21 -0700 Subject: [PATCH 1/7] Add a net11 incremental CBOR decode path using SlideData and NeedsMoreData --- ...Extensions.CborProtocol.NetStandard.csproj | 6 +- .../Internal/CborStreamReader.cs | 160 ++++++++++++++++++ .../CborStreamReaderTests.cs | 11 ++ 3 files changed, 176 insertions(+), 1 deletion(-) diff --git a/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetStandard.csproj b/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetStandard.csproj index af8ce6533bcb..f31208104e24 100644 --- a/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetStandard.csproj +++ b/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetStandard.csproj @@ -41,7 +41,11 @@ - + + + + + diff --git a/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs b/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs index 373ca12f3d2e..a0e3dd37a852 100644 --- a/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs +++ b/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs @@ -27,8 +27,167 @@ namespace Amazon.Extensions.CborProtocol.Internal /// the entire payload into memory at once. This class wraps /// to provide streaming capabilities while maintaining the same reading interface. /// + /// + /// On net11.0 and later this is backed by the incremental APIs + /// (SlideData plus the NeedsMoreData state), which let the reader signal when it needs + /// the next chunk and retain its parsing state across chunk boundaries. On earlier targets the + /// incremental APIs are unavailable, so the reader buffers data itself and recovers from mid-item + /// boundaries by catching the reader's exceptions, refilling, and retrying. + /// public class CborStreamReader : IDisposable { +#if NET11_0_OR_GREATER + private static readonly ILogger _logger = Logger.GetLogger(typeof(CborStreamReader)); + private readonly Stream _stream; + private byte[] _buffer; + private CborReader _internalCborReader; + + /// + /// Total bytes handed to the reader by the most recent SlideData call. The reader consumes + /// from the front of this window, so the unconsumed bytes are the tail of _buffer[0.._slidLength]. + /// + private int _slidLength; + + /// + /// Set once a read from the underlying stream returns no data, so later fills mark the final block. + /// + private bool _streamExhausted; + + private bool _hasReadFromStream; + + /// + /// Initializes a new instance of the class that reads CBOR data + /// from the specified stream. + /// + /// The input stream containing CBOR-formatted data. + /// The stream must be readable and remain open for the lifetime of the reader. + /// Thrown when the stream parameter is null. + public CborStreamReader(Stream stream) + { + _stream = stream ?? throw new ArgumentNullException(nameof(stream)); + _buffer = ArrayPool.Shared.Rent(AWSConfigs.CborReaderInitialBufferSize); + + // Incremental reading requires Lax conformance: stricter modes reject a non-final block. + // The stream is read lazily on the first CBOR read (see ExecuteRead); an event stream + // response hands the stream on undecoded, so reading here would consume its first frames. + _internalCborReader = new CborReader( + ReadOnlyMemory.Empty, + new CborReaderOptions { ConformanceMode = CborConformanceMode.Lax, AllowMultipleRootLevelValues = true }, + isFinalBlock: false); + } + + /// + /// Pulls the next chunk from the stream and hands it to the reader via SlideData, preserving + /// any bytes the reader has not yet consumed. Grows the buffer when the unconsumed bytes already fill it. + /// + private void Fill() + { + int unconsumed = _internalCborReader.BytesRemaining; + + // The reader consumes from the front of the last slid window, so the unconsumed bytes are the tail + // of _buffer[0.._slidLength]. Move them to the front, then read new bytes after them. SlideData then + // takes the leftover plus the new bytes as one contiguous block, so the reader never sees a split item. + if (unconsumed > 0) + { + Buffer.BlockCopy(_buffer, _slidLength - unconsumed, _buffer, 0, unconsumed); + } + + if (unconsumed == _buffer.Length) + { + var grown = ArrayPool.Shared.Rent(_buffer.Length * 2); + Buffer.BlockCopy(_buffer, 0, grown, 0, unconsumed); + ArrayPool.Shared.Return(_buffer); + _buffer = grown; + } + + int read = _stream.Read(_buffer, unconsumed, _buffer.Length - unconsumed); + if (read == 0) + _streamExhausted = true; + + _slidLength = unconsumed + read; + _internalCborReader.SlideData(new ReadOnlyMemory(_buffer, 0, _slidLength), isFinalBlock: _streamExhausted); + + _logger.DebugFormat("Buffer refilled: read {0} byte(s), total in buffer now: {1}.", read, _slidLength); + } + + /// + /// Executes a CBOR read operation, feeding the reader more data whenever it reports + /// and retrying once more data arrives if an atomic + /// operation (for example SkipValue over a subtree, or a long string) runs past the end of + /// the buffered chunk. Both the incremental state and buffer position are preserved across + /// SlideData, so a retry resumes rather than restarts. A content error raised after the + /// stream is exhausted is a genuine malformed/truncated payload and is allowed to propagate. + /// + /// The return type of the CBOR read operation. + /// A delegate representing the read operation to execute. It receives this + /// and the internal as arguments so the delegates + /// passed here don't capture state and are cached by the compiler instead of allocated per call. + private T ExecuteRead(Func readOperation) + { + if (!_hasReadFromStream) + { + _hasReadFromStream = true; + Fill(); + } + + while (true) + { + while (_internalCborReader.PeekState() == CborReaderState.NeedsMoreData && !_streamExhausted) + { + Fill(); + } + + try + { + return readOperation(this, _internalCborReader); + } + catch (CborContentException) when (!_streamExhausted) + { + // An atomic operation reached the end of the buffered chunk mid-item. The reader keeps + // its position, so pull the next chunk and retry from where it left off. + Fill(); + } + catch (InvalidOperationException ex) + { + // The incremental reader raises InvalidOperationException for structural misuse, such as + // ending the wrong container type or reading a value as the wrong major type on a malformed + // payload. Surface it as a content error so callers see the same exception as on other targets. + throw new CborContentException(ex.Message, ex); + } + } + } + + public int? ReadStartMap() => ExecuteRead(static (_, r) => r.ReadStartMap()); + public int? ReadStartArray() => ExecuteRead(static (_, r) => r.ReadStartArray()); + public void ReadEndMap() => ExecuteRead(static (_, r) => { r.ReadEndMap(); return true; }); + public void ReadEndArray() => ExecuteRead(static (_, r) => { r.ReadEndArray(); return true; }); + + public CborReaderState PeekState() => ExecuteRead(static (_, r) => r.PeekState()); + + public string ReadTextString() => ExecuteRead(static (_, r) => r.ReadTextString()); + public byte[] ReadByteString() => ExecuteRead(static (_, r) => r.ReadByteString()); + public int ReadInt32() => ExecuteRead(static (_, r) => r.ReadInt32()); + public long ReadInt64() => ExecuteRead(static (_, r) => r.ReadInt64()); + public ulong ReadUInt64() => ExecuteRead(static (_, r) => r.ReadUInt64()); + public decimal ReadDecimal() => ExecuteRead(static (_, r) => r.ReadDecimal()); + public double ReadDouble() => ExecuteRead(static (_, r) => r.ReadDouble()); + public bool ReadBoolean() => ExecuteRead(static (_, r) => r.ReadBoolean()); + public float ReadSingle() => ExecuteRead(static (_, r) => r.ReadSingle()); + public void ReadNull() => ExecuteRead(static (_, r) => { r.ReadNull(); return true; }); + public void SkipValue() => ExecuteRead(static (_, r) => { r.SkipValue(); return true; }); + public CborTag ReadTag() => ExecuteRead(static (_, r) => r.ReadTag()); + + public int CurrentDepth => _internalCborReader.CurrentDepth; + + public void Dispose() + { + if (_buffer != null) + { + ArrayPool.Shared.Return(_buffer); + _buffer = null; + } + } +#else /// /// Represents a single container frame (map or array) within the nesting stack /// of the . This is a struct kept in an array-backed @@ -638,5 +797,6 @@ public void Dispose() _buffer = null; } } +#endif } } diff --git a/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs b/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs index 2a8dc5ce2806..5175e9734eb5 100644 --- a/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs +++ b/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs @@ -1280,7 +1280,18 @@ public void CorruptArrayHeader_DeclaringHugeLength_DoesNotGrowBufferToDeclaredLe var bytes = new byte[] { 0x9A, 0x05, 0xF5, 0xE1, 0x00, 0x01, 0x02, 0x03 }; using var reader = new CborStreamReader(new MemoryStream(bytes)); +#if NET11_0_OR_GREATER + // The incremental reader validates element counts lazily: the array header is well-formed, so + // ReadStartArray returns the declared count and the content error surfaces once the body runs out. + reader.ReadStartArray(); + Assert.Throws(() => + { + for (int i = 0; i < 100_000_000; i++) + reader.ReadInt32(); + }); +#else Assert.Throws(() => reader.ReadStartArray()); +#endif // The buffer grows only as received bytes arrive, so a header can't force a ~100 MB allocation. var buffer = (byte[])typeof(CborStreamReader) From 3b7d1093f060fc1c6913cb9d7ab75063b0b34d74 Mon Sep 17 00:00:00 2001 From: Sichan Yoo Date: Fri, 16 Oct 2026 10:42:35 -0700 Subject: [PATCH 2/7] Target the serde benchmark runner at net11.0 to exercise the incremental CBOR path --- .../SerdeBenchmarksRunner/SerdeBenchmarksRunner.csproj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/test/Performance/SerdeBenchmarks/SerdeBenchmarksRunner/SerdeBenchmarksRunner.csproj b/sdk/test/Performance/SerdeBenchmarks/SerdeBenchmarksRunner/SerdeBenchmarksRunner.csproj index 86192dea212a..a75d24def91d 100644 --- a/sdk/test/Performance/SerdeBenchmarks/SerdeBenchmarksRunner/SerdeBenchmarksRunner.csproj +++ b/sdk/test/Performance/SerdeBenchmarks/SerdeBenchmarksRunner/SerdeBenchmarksRunner.csproj @@ -2,7 +2,9 @@ Exe - $(SdkPerformanceTargets) + + net11.0 enable enable From 3242f07b542dadcdc1c9b2b923725d66a5a84547 Mon Sep 17 00:00:00 2001 From: Sichan Yoo Date: Mon, 26 Oct 2026 09:54:25 -0700 Subject: [PATCH 3/7] Address reviews. --- ...Extensions.CborProtocol.NetStandard.csproj | 7 +- .../Internal/CborStreamReader.cs | 682 ++---------------- .../CborStreamReaderTests.cs | 78 +- sdk/Directory.Build.props | 2 +- .../SerdeBenchmarksRunner.csproj | 4 +- 5 files changed, 125 insertions(+), 648 deletions(-) diff --git a/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetStandard.csproj b/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetStandard.csproj index f31208104e24..d628e806933f 100644 --- a/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetStandard.csproj +++ b/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetStandard.csproj @@ -41,13 +41,10 @@ - - + + - - - diff --git a/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs b/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs index a0e3dd37a852..45f5409e7911 100644 --- a/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs +++ b/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs @@ -28,15 +28,12 @@ namespace Amazon.Extensions.CborProtocol.Internal /// to provide streaming capabilities while maintaining the same reading interface. /// /// - /// On net11.0 and later this is backed by the incremental APIs - /// (SlideData plus the NeedsMoreData state), which let the reader signal when it needs - /// the next chunk and retain its parsing state across chunk boundaries. On earlier targets the - /// incremental APIs are unavailable, so the reader buffers data itself and recovers from mid-item - /// boundaries by catching the reader's exceptions, refilling, and retrying. + /// Backed by the incremental APIs (SlideData and the + /// NeedsMoreData state), which let the reader request the next chunk and keep its parsing + /// state across chunk boundaries. /// public class CborStreamReader : IDisposable { -#if NET11_0_OR_GREATER private static readonly ILogger _logger = Logger.GetLogger(typeof(CborStreamReader)); private readonly Stream _stream; private byte[] _buffer; @@ -111,49 +108,46 @@ private void Fill() } /// - /// Executes a CBOR read operation, feeding the reader more data whenever it reports - /// and retrying once more data arrives if an atomic - /// operation (for example SkipValue over a subtree, or a long string) runs past the end of - /// the buffered chunk. Both the incremental state and buffer position are preserved across - /// SlideData, so a retry resumes rather than restarts. A content error raised after the - /// stream is exhausted is a genuine malformed/truncated payload and is allowed to propagate. + /// Executes a single-token CBOR read, refilling first while the reader reports + /// so the token is fully buffered. A + /// from the read is a malformed payload and propagates. /// - /// The return type of the CBOR read operation. - /// A delegate representing the read operation to execute. It receives this - /// and the internal as arguments so the delegates - /// passed here don't capture state and are cached by the compiler instead of allocated per call. private T ExecuteRead(Func readOperation) + { + EnsureStarted(); + FillWhileNeedsMoreData(); + + try + { + return readOperation(this, _internalCborReader); + } + catch (InvalidOperationException ex) + { + // The incremental reader raises InvalidOperationException for structural misuse, such as + // ending the wrong container type or reading a value as the wrong major type on a malformed + // payload. Surface it as a content error so callers see the same exception as on other targets. + throw new CborContentException(ex.Message, ex); + } + } + + private void EnsureStarted() { if (!_hasReadFromStream) { _hasReadFromStream = true; Fill(); } + } - while (true) + /// + /// Refills while the reader needs more bytes for its next token, stopping once the stream is + /// exhausted (a remaining then means a truncated payload). + /// + private void FillWhileNeedsMoreData() + { + while (_internalCborReader.PeekState() == CborReaderState.NeedsMoreData && !_streamExhausted) { - while (_internalCborReader.PeekState() == CborReaderState.NeedsMoreData && !_streamExhausted) - { - Fill(); - } - - try - { - return readOperation(this, _internalCborReader); - } - catch (CborContentException) when (!_streamExhausted) - { - // An atomic operation reached the end of the buffered chunk mid-item. The reader keeps - // its position, so pull the next chunk and retry from where it left off. - Fill(); - } - catch (InvalidOperationException ex) - { - // The incremental reader raises InvalidOperationException for structural misuse, such as - // ending the wrong container type or reading a value as the wrong major type on a malformed - // payload. Surface it as a content error so callers see the same exception as on other targets. - throw new CborContentException(ex.Message, ex); - } + Fill(); } } @@ -174,619 +168,38 @@ private T ExecuteRead(Func readOperation) public bool ReadBoolean() => ExecuteRead(static (_, r) => r.ReadBoolean()); public float ReadSingle() => ExecuteRead(static (_, r) => r.ReadSingle()); public void ReadNull() => ExecuteRead(static (_, r) => { r.ReadNull(); return true; }); - public void SkipValue() => ExecuteRead(static (_, r) => { r.SkipValue(); return true; }); - public CborTag ReadTag() => ExecuteRead(static (_, r) => r.ReadTag()); - - public int CurrentDepth => _internalCborReader.CurrentDepth; - - public void Dispose() - { - if (_buffer != null) - { - ArrayPool.Shared.Return(_buffer); - _buffer = null; - } - } -#else - /// - /// Represents a single container frame (map or array) within the nesting stack - /// of the . This is a struct kept in an array-backed - /// stack so that entering a container does not allocate. - /// - private struct ContainerFrame - { - /// - /// Gets the type of CBOR container (map or array). - /// - public CborContainerType Type { get; } - - /// - /// Gets the number of items left to be read in this container. - /// For indefinite-length containers, this value is null. - /// - public int? RemainingItems { get; private set; } - - /// - /// Initializes a new instance of the struct. - /// - /// The container type (map or array). - /// - /// The number of items in the container, or null for indefinite-length containers. - /// - public ContainerFrame(CborContainerType type, int? remainingItems) - { - Type = type; - RemainingItems = remainingItems; - if (RemainingItems.HasValue && type == CborContainerType.Map) - { - // Multiply RemainingItems by 2 for maps to account for key value pairs. - RemainingItems *= 2; - } - } - - /// - /// Decrements the count when an item is consumed. - /// For indefinite-length containers, this method does nothing. - /// - public void ConsumeItem() - { - RemainingItems = RemainingItems.HasValue ? RemainingItems.Value - 1 : (int?)null; - - if (RemainingItems < 0) - { - // This should never happen if container boundaries are respected. - // It indicates that more items were read than the container declared. - throw new CborContentException( - $"Too many items read from {Type} container. Expected {RemainingItems + 1} more item(s)." - ); - } - } - - /// - /// Gets a value indicating whether this container has been fully consumed. - /// - public bool IsComplete => RemainingItems == 0; - } - - /// - /// Enum to track the type of CBOR container (map or array) - /// for state management within the CborStreamReader. - /// - private enum CborContainerType - { - Map, - Array - } - - /// - /// A minimal stack of values. Unlike , - /// returns the top frame by reference so it can be updated in place. - /// - private sealed class ContainerFrameStack - { - private ContainerFrame[] _frames = new ContainerFrame[8]; - - public int Count { get; private set; } - - public void Push(ContainerFrame frame) - { - if (Count == _frames.Length) - { - Array.Resize(ref _frames, _frames.Length * 2); - } - _frames[Count++] = frame; - } - - public ref ContainerFrame Peek() - { - if (Count == 0) - throw new InvalidOperationException("Stack empty."); - return ref _frames[Count - 1]; - } - - public void Pop() - { - if (Count == 0) - throw new InvalidOperationException("Stack empty."); - Count--; - } - } - - private static readonly ILogger _logger = Logger.GetLogger(typeof(CborStreamReader)); - private readonly ContainerFrameStack _nestingStack = new ContainerFrameStack(); - private readonly Stream _stream; - private byte[] _buffer; - private CborReader _internalCborReader; - private int _currentChunkSize; - private bool _hasReadFromStream; - - /// - /// Set once a read from the underlying stream returns no data, so the reader stops - /// trying to refill ahead of reads at the end of the stream. - /// - private bool _streamExhausted; - - /// - /// The longest CBOR data item header: the initial byte followed by up to 8 bytes of argument. - /// - private const int MaxCborHeaderLength = 1 + 8; - - /// - /// Initializes a new instance of the class that reads CBOR data - /// from the specified stream. - /// - /// The input stream containing CBOR-formatted data. - /// The stream must be readable and remain open for the lifetime of the reader. - /// Thrown when the stream parameter is null. - /// - /// The reader uses a configurable initial buffer size (from ) - /// and will automatically resize the buffer if needed to handle larger CBOR items. - /// - public CborStreamReader(Stream stream) - { - _stream = stream ?? throw new ArgumentNullException(nameof(stream)); - _buffer = ArrayPool.Shared.Rent(AWSConfigs.CborReaderInitialBufferSize); - - // The stream is first read on the first CBOR read, not here: the unmarshaller context constructs this - // reader before the response unmarshaller runs, and an event stream response hands the stream on - // undecoded, so reading here would consume its first frames. - // We must allow multiple root values because when refilling the new chunk is just a fragment of the whole stream. - _internalCborReader = new CborReader(ReadOnlyMemory.Empty, allowMultipleRootLevelValues: true); - } - - /// - /// This method is called when a read operation fails because it needs more - /// data than is currently available in the buffer. It handles stitching leftover - /// data with a new chunk from the stream and, if necessary, resizing the buffer. - /// - /// Number of bytes to skip before reading new data (e.g., 1 to skip CBOR break byte 0xFF) - private void RefillBuffer(int bytesToSkip = 0) - { - int leftoverBytesCount = _internalCborReader.BytesRemaining; - - // Determine where the leftover bytes start - int leftoverStartIndex = _currentChunkSize - leftoverBytesCount; - - // Read into any free space after the current chunk first. If the stream has no more data and there is - // nothing to skip, return without resetting the internal reader: resetting forgets the containers it - // has open, so at the end of the stream it would see a closing break byte (0xFF) at root level and throw. - if (bytesToSkip == 0 && _currentChunkSize < _buffer.Length) - { - int bytesAppended = _stream.Read(_buffer, _currentChunkSize, _buffer.Length - _currentChunkSize); - if (bytesAppended == 0) - { - _streamExhausted = true; - return; - } - - _currentChunkSize += bytesAppended; - leftoverBytesCount += bytesAppended; - } - - // If we are skipping bytes, we need to move the start forward - leftoverStartIndex += bytesToSkip; - leftoverBytesCount = leftoverBytesCount - bytesToSkip; - - // If the leftover data completely fills the buffer, grow it - if (leftoverBytesCount >= _buffer.Length) - { - int newSize = _buffer.Length * 2; - var newBuffer = ArrayPool.Shared.Rent(newSize); - - Buffer.BlockCopy(_buffer, leftoverStartIndex, newBuffer, 0, leftoverBytesCount); - ArrayPool.Shared.Return(_buffer); - _buffer = newBuffer; - } - else if (leftoverBytesCount > 0) // Shift leftovers (after skipping) to the beginning of the buffer - { - Buffer.BlockCopy(_buffer, leftoverStartIndex, _buffer, 0, leftoverBytesCount); - } - - // Read from stream into buffer after leftovers - int bytesReadFromStream = _stream.Read(_buffer, leftoverBytesCount, _buffer.Length - leftoverBytesCount); - if (bytesReadFromStream == 0) - _streamExhausted = true; - - // Update the total size of valid data in our buffer. - _currentChunkSize = leftoverBytesCount + bytesReadFromStream; - - var newMemorySlice = new ReadOnlyMemory(_buffer, 0, _currentChunkSize); - _internalCborReader.Reset(newMemorySlice); - - _logger.DebugFormat("Buffer refilled: read {0} byte(s), total in buffer now: {1}.", bytesReadFromStream, _currentChunkSize); - } /// - /// Executes a CBOR read operation, refilling the buffer and retrying if a CborContentException is thrown. + /// Skips the next value, which may be a subtree spanning several chunks. Uses TrySkipValue + /// (returns without consuming when more data is needed) so large values are + /// skipped by refilling in a loop rather than by catching exceptions. /// - /// The return type of the CBOR read operation. - /// A delegate representing the read operation to execute. It receives this - /// and the internal as arguments so the delegates - /// passed here don't capture state and are cached by the compiler instead of allocated per call. - /// The result of the read operation. - /// - /// Thrown if too many retries are attempted or if the stream ends unexpectedly. - /// - private T ExecuteRead(Func readOperation) - { - if (!_hasReadFromStream) - { - _hasReadFromStream = true; - RefillBuffer(); - } - - int maxRetries = 64; - int retryCount = 0; - - // Refill before the read when fewer bytes remain than the longest CBOR header, so a value that - // straddles the end of the chunk is read whole instead of failing and being retried. Each such - // failure throws, and exceptions cost far more than a refill. - if (_internalCborReader.BytesRemaining < MaxCborHeaderLength && !_streamExhausted) - { - RefillBuffer(); - } - - while (true) - { - try - { - return readOperation(this, _internalCborReader); - } - catch (Exception ex) when (ex is InvalidOperationException || ex is CborContentException) - { - // Both InvalidOperationException and CborContentException can occur when the reader - // runs out of buffered data mid-item or encounters an incomplete CBOR structure. - // - CborContentException: typically thrown when encountering invalid CBOR content, - // such as a premature break or truncated container. - // - InvalidOperationException: can happen when the reader attempts to interpret data - // as a different type due to hitting a buffer boundary before the full item is available. - // We catch both, trigger a buffer refill, and retry before deciding if it is a genuine error. - - if (_currentChunkSize == 0 && _internalCborReader.BytesRemaining == 0) - { - // Fail fast if we’ve already consumed all input and nothing remains to refill. - throw; - } - - if (++retryCount > maxRetries) - { - throw new CborContentException("Too many retries during CBOR stream parsing. Possible malformed or infinite data.", ex); - } - - _logger.Debug(ex, "CborContentException caught (attempt #{0}), attempting to refill buffer.", retryCount); - // Attempt to refill and retry the operation. - RefillBuffer(); - } - } - } - - - private bool IsNextByteEndOfContainer() - { - int unreadOffset = _currentChunkSize - _internalCborReader.BytesRemaining; - if (unreadOffset < _currentChunkSize) - return _buffer[unreadOffset] == 0xFF; // 0xFF indicates "break" in indefinite-length map/array - - return false; - } - - private void ReadEndContainer(CborContainerType expectedType) - { - // One static delegate per container type so neither captures the expected type. - if (expectedType == CborContainerType.Map) - ExecuteRead(static (self, r) => self.ReadEndContainerCore(r, CborContainerType.Map)); - else - ExecuteRead(static (self, r) => self.ReadEndContainerCore(r, CborContainerType.Array)); - } - - private bool ReadEndContainerCore(CborReader r, CborContainerType expectedType) + public void SkipValue() { - var expectedEndState = expectedType == CborContainerType.Map ? CborReaderState.EndMap : CborReaderState.EndArray; - if (_nestingStack.Count == 0 || _nestingStack.Peek().Type != expectedType) - throw new CborContentException($"Unexpected end of {expectedType.ToString().ToLowerInvariant()}."); + EnsureStarted(); + FillWhileNeedsMoreData(); - var state = CborReaderState.Finished; try { - state = r.PeekState(); - } - catch (CborContentException) - { - // This exception is expected in two cases: - // 1. When we've reached the end of the current CBOR buffer chunk and need to refill. - // 2. When the current buffer does not contain the start of the array/map we're trying to end. - // In both cases, the internal CborReader's state will be `Finished` and we will trigger a buffer refill. - // This is not a true error, but logging at Info level can help trace how we arrived at a certain state. - _logger.DebugFormat("CborContentException caught during PeekState while expecting end of {0}", expectedType.ToString().ToLowerInvariant()); - } - - if (state == expectedEndState) - { - if (expectedType == CborContainerType.Map) - r.ReadEndMap(); - else - r.ReadEndArray(); - _nestingStack.Pop(); - return true; - } - - if (state == CborReaderState.Finished) - { - // We got CborReaderState.Finished which means the reader has exhausted the bytes currently - // given to it and the next token may live in the next chunk that we haven't read yet. - // - // For indefinite-length containers the end of the container is a break marker byte (0xFF). - // If that break byte is the next byte in the stream, we must consume it, otherwise the reader - // will become desynchronized (it will still think we're inside a container while removed the - // container from the _nestingStack). - // This byte can exist in the next chunk, so when we hit Finished we first attempt to refill - // the buffer (no skip) so the reader can see the next byte(s). Only after refilling can we safely - // determine whether the next byte is a break byte that terminates the current container. - - // Try to refill first, maybe the break marker is in the next chunk. - RefillBuffer(0); - - if (!_nestingStack.Peek().RemainingItems.HasValue && IsNextByteEndOfContainer()) - { - // If the next raw byte after refill is the CBOR "break" (0xFF), that indicates an - // indefinite-length container terminator. We must explicitly consume that break byte - // so we can remove the item from the _nestingStack and parsing can continue correctly. - // - // We call RefillBuffer(1) to skip the break byte and rebuild the internal reader. - // It consumes the break byte from the leftover buffer so subsequent calls to the CborReader - // see the next item. - - RefillBuffer(1); // Skip the break marker (0xFF) - } - _nestingStack.Pop(); - return true; - } - else if (_nestingStack.Count > 0 && _nestingStack.Peek().IsComplete) - { - // Special handling for definite-length containers that are fully consumed: - // For definite-length maps/arrays, the internal CborReader does not provide an explicit - // end token. Once we have read all items (tracked via the _nestingStack frame's RemainingItems), - // PeekState may point to the next item rather than EndArray/EndMap. In that case, we can - // safely consider the container complete, pop the frame, and continue parsing. - // - // This prevents throwing a CborContentException when reading end container of a large - // container that spans multiple buffer chunks and ensures ReadEndContainer returns the correct - // logical end of the container. - _nestingStack.Pop(); - return true; - } - - throw new CborContentException($"Expected end of {expectedType.ToString().ToLowerInvariant()} but could not parse it."); - } - - - public void ReadEndMap() - { - ReadEndContainer(CborContainerType.Map); - // After reading the end of a container (map/array), we need to mark it as consumed - // in the parent container (if any). This ensures that nested arrays/maps count as - // a single item in their enclosing container. Without this, the parent container - // would incorrectly think there are remaining items, causing PeekState and - // item-count tracking to become desynchronized with the actual CBOR stream. - if (_nestingStack.Count > 0) - { - _nestingStack.Peek().ConsumeItem(); - } - } - - public void ReadEndArray() - { - ReadEndContainer(CborContainerType.Array); - // After reading the end of a container (map/array), we need to mark it as consumed - // in the parent container (if any). This ensures that nested arrays/maps count as - // a single item in their enclosing container. Without this, the parent container - // would incorrectly think there are remaining items, causing PeekState and - // item-count tracking to become desynchronized with the actual CBOR stream. - if (_nestingStack.Count > 0) - { - _nestingStack.Peek().ConsumeItem(); - } - } - - - public int? ReadStartMap() => ExecuteRead(static (self, reader) => - { - self.EnsureDefiniteLengthItemIsBuffered(2); - var count = reader.ReadStartMap(); - self._nestingStack.Push(new ContainerFrame(CborContainerType.Map, count)); - return count; - }); - - public int? ReadStartArray() => ExecuteRead(static (self, reader) => - { - self.EnsureDefiniteLengthItemIsBuffered(1); - var count = reader.ReadStartArray(); - self._nestingStack.Push(new ContainerFrame(CborContainerType.Array, count)); - return count; - }); - - - public CborReaderState PeekState() - { - return ExecuteRead(static (self, r) => self.PeekStateCore(r)); - } - - private CborReaderState PeekStateCore(CborReader r) - { - // Handle definite-length containers that silently complete - if (_nestingStack.Count > 0) - { - var frame = _nestingStack.Peek(); - if (frame.IsComplete) - { - return frame.Type == CborContainerType.Map - ? CborReaderState.EndMap - : CborReaderState.EndArray; - } - } - - // We need to Peek twice in case the first time failed because we are near the end of the current chunk and we just need to refill. - for (int attempt = 0; attempt < 2; attempt++) - { - try + while (!_internalCborReader.TrySkipValue()) { - var state = r.PeekState(); - if (state == CborReaderState.Finished && _nestingStack.Count > 0) + if (_streamExhausted) { - _logger.DebugFormat("PeekState returned Finished, but nesting stack is not empty. Attempting refill."); - RefillBuffer(0); - continue; + // Truncated payload: the value can never complete. + throw new CborContentException("Unexpected end of CBOR data stream while skipping a value."); } - return state; - } - catch (CborContentException ex) - { - // PeekState threw an exception, we will attempt to refill in case we aren't at the end of the stream. - _logger.Debug(ex, "PeekState threw exception (attempt #{0}). Attempting refill.", attempt + 1); - RefillBuffer(0); + Fill(); } } - - // If PeekState still fails after refilling, and we're truly at the end of the stream, - // only then consider inferring the state based on container nesting. - if (_nestingStack.Count > 0) - { - var inferredState = _nestingStack.Peek().Type == CborContainerType.Map - ? CborReaderState.EndMap - : CborReaderState.EndArray; - - _logger.DebugFormat("CborContentException during PeekState interpreted as {0} due to nesting stack.", inferredState); - return inferredState; - } - - throw new CborContentException("Unable to determine CBOR reader state after retries, and no containers remain."); - } - - /// - /// Executes the provided CBOR read operation and automatically updates the item count - /// of the current container (if any) to ensure that whenever an item is read, the parent - /// container's RemainingItems count is decremented. - /// - private T ExecuteValueRead(Func readOperation) - { - var result = ExecuteRead(readOperation); - if (_nestingStack.Count > 0) - { - _nestingStack.Peek().ConsumeItem(); - } - - return result; - } - - /// - /// Makes sure the buffer holds enough of an upcoming definite-length CBOR item for - /// to accept it: the whole payload of a string, or for a map or array at least one byte per element, - /// since rejects a declared length that can't fit in the bytes it holds. - /// If the item runs past the end of the current chunk, the buffer is refilled until the item is buffered - /// or the stream ends. grows the buffer at most 2x at a time, and only when it is - /// full of received bytes, so the allocation tracks the data that actually arrives rather than the declared - /// length; a corrupt or truncated header can't force an oversized allocation. - /// - /// This avoids the exception the read would otherwise throw when an item straddles the end of a chunk. If - /// the stream ends first, the read throws the same bounded as before. - /// - /// The minimum encoded size of one element of the declared length: - /// 1 for strings (bytes) and arrays (items), 2 for maps (a key and a value per entry). - private void EnsureDefiniteLengthItemIsBuffered(int minBytesPerElement) - { - const int AdditionalInfoMask = 0b0001_1111; - - var leftoverBytesCount = _internalCborReader.BytesRemaining; - var unreadOffset = _currentChunkSize - leftoverBytesCount; - - if (leftoverBytesCount == 0) - return; - - // Extract the additional information field which determines how the length is encoded. - var additionalInfo = _buffer[unreadOffset] & AdditionalInfoMask; - - int headerLength = - additionalInfo < 24 ? 1 : - additionalInfo == 24 ? 2 : - additionalInfo == 25 ? 3 : - additionalInfo == 26 ? 5 : - additionalInfo == 27 ? 9 : - 0; - - // Indefinite-length item (31) or unsupported additional info: the total size can't be precomputed. - // Also skip when the header itself isn't fully buffered; the read's retry handles that case. - if (headerLength == 0 || leftoverBytesCount < headerLength) - return; - - // The bytes following the initial type/length header. - var valueSpan = _buffer.AsSpan(unreadOffset + 1); - - ulong len64 = 0; - - // Decode the declared length according to CBOR major type encoding rules. - if (additionalInfo < 24) - { - len64 = (ulong)additionalInfo; // Length directly encoded in the additionalInfo bits. - } - else if (additionalInfo == 24) - { - len64 = valueSpan[0]; - } - else if (additionalInfo == 25) - { - len64 = BinaryPrimitives.ReadUInt16BigEndian(valueSpan); - } - else if (additionalInfo == 26) - { - len64 = BinaryPrimitives.ReadUInt32BigEndian(valueSpan); - } - else - { - len64 = BinaryPrimitives.ReadUInt64BigEndian(valueSpan); - } - - // Guard the multiplication below; lengths this large fall through to the size check after it. - if (len64 <= int.MaxValue) - len64 *= (ulong)minBytesPerElement; - - // The whole item (header + payload) is already in the buffer. - if (len64 <= (ulong)(leftoverBytesCount - headerLength)) - return; - - // Lengths that can't fit in a buffer are malformed or corrupt. Skip the refill and let the read fail. - if (len64 > (ulong)(int.MaxValue - headerLength)) - { - _logger.DebugFormat("Skipping refill ahead of read: declared length {0} too large.", len64); - return; - } - - // Bytes the reader needs buffered: the header plus the declared payload (or minimum element bytes). - int required = (int)len64 + headerLength; - - // Refill until the whole item is buffered or the stream runs dry. - while (!_streamExhausted && _internalCborReader.BytesRemaining < required) + catch (InvalidOperationException ex) { - RefillBuffer(); + throw new CborContentException(ex.Message, ex); } } - public string ReadTextString() => ExecuteValueRead(static (self, r) => { self.EnsureDefiniteLengthItemIsBuffered(1); return r.ReadTextString(); }); - public byte[] ReadByteString() => ExecuteValueRead(static (self, r) => { self.EnsureDefiniteLengthItemIsBuffered(1); return r.ReadByteString(); }); - public int ReadInt32() => ExecuteValueRead(static (_, r) => r.ReadInt32()); - public long ReadInt64() => ExecuteValueRead(static (_, r) => r.ReadInt64()); - public ulong ReadUInt64() => ExecuteValueRead(static (_, r) => r.ReadUInt64()); - public decimal ReadDecimal() => ExecuteValueRead(static (_, r) => r.ReadDecimal()); - public double ReadDouble() => ExecuteValueRead(static (_, r) => r.ReadDouble()); - public bool ReadBoolean() => ExecuteValueRead(static (_, r) => r.ReadBoolean()); - public float ReadSingle() => ExecuteValueRead(static (_, r) => r.ReadSingle()); - public void ReadNull() => ExecuteValueRead(static (_, r) => { r.ReadNull(); return true; }); - public void SkipValue() => ExecuteValueRead(static (_, r) => { r.SkipValue(); return true; }); - - // Tags annotate the next CBOR item but are not themselves counted - // as items within a definite-length array or map. For this reason, - // we do not call ExecuteValueRead() and only use ExecuteRead() here. public CborTag ReadTag() => ExecuteRead(static (_, r) => r.ReadTag()); + public int CurrentDepth => _internalCborReader.CurrentDepth; public void Dispose() @@ -797,6 +210,5 @@ public void Dispose() _buffer = null; } } -#endif } } diff --git a/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs b/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs index 5175e9734eb5..37bf9632998f 100644 --- a/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs +++ b/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs @@ -1280,7 +1280,6 @@ public void CorruptArrayHeader_DeclaringHugeLength_DoesNotGrowBufferToDeclaredLe var bytes = new byte[] { 0x9A, 0x05, 0xF5, 0xE1, 0x00, 0x01, 0x02, 0x03 }; using var reader = new CborStreamReader(new MemoryStream(bytes)); -#if NET11_0_OR_GREATER // The incremental reader validates element counts lazily: the array header is well-formed, so // ReadStartArray returns the declared count and the content error surfaces once the body runs out. reader.ReadStartArray(); @@ -1289,9 +1288,6 @@ public void CorruptArrayHeader_DeclaringHugeLength_DoesNotGrowBufferToDeclaredLe for (int i = 0; i < 100_000_000; i++) reader.ReadInt32(); }); -#else - Assert.Throws(() => reader.ReadStartArray()); -#endif // The buffer grows only as received bytes arrive, so a header can't force a ~100 MB allocation. var buffer = (byte[])typeof(CborStreamReader) @@ -1299,4 +1295,78 @@ public void CorruptArrayHeader_DeclaringHugeLength_DoesNotGrowBufferToDeclaredLe .GetValue(reader)!; Assert.True(buffer.Length <= 1024 * 1024, $"Read buffer grew to {buffer.Length} bytes for an 8-byte stream."); } + + private static byte[] GetBuffer(CborStreamReader reader) => + (byte[])typeof(CborStreamReader) + .GetField("_buffer", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)! + .GetValue(reader)!; + + [Fact] + public void MalformedByte_DoesNotBufferRestOfStream() + { + // [ 0x82 = array(2), 0x1C = reserved additional info (malformed) ] + 2 MB padding. + var bytes = new byte[2 * 1024 * 1024]; + bytes[0] = 0x82; + bytes[1] = 0x1C; + var stream = new MemoryStream(bytes); + using var reader = new CborStreamReader(stream); + + reader.ReadStartArray(); + Assert.Throws(() => reader.ReadInt32()); + + // Fail fast: a malformed token must not buffer the whole stream first. + Assert.True(stream.Position < 2 * AWSConfigs.CborReaderInitialBufferSize, + $"Stream advanced to {stream.Position} bytes on a malformed token; expected an early failure."); + Assert.True(GetBuffer(reader).Length < 2 * 1024 * 1024, + "Read buffer grew toward the full stream size on a malformed token."); + } + + [Fact] + public void SkipValue_OverLargeSubtree_SpanningManyChunks_Succeeds() + { + // A valid array far larger than the buffer, so SkipValue spans many refills. + var writer = new CborWriter(); + writer.WriteStartArray(500); + for (int i = 0; i < 500; i++) + writer.WriteTextString($"element-with-some-length-{i}"); + writer.WriteEndArray(); + var payload = writer.Encode(); + Assert.True(payload.Length > 10 * AWSConfigs.CborReaderInitialBufferSize); + + // Wrap so there is a value after the skipped one to confirm the position. + var outer = new CborWriter(); + outer.WriteStartArray(2); + outer.WriteEncodedValue(payload); + outer.WriteInt32(42); + outer.WriteEndArray(); + + using var reader = new CborStreamReader(new MemoryStream(outer.Encode())); + reader.ReadStartArray(); + reader.SkipValue(); + Assert.Equal(42, reader.ReadInt32()); + reader.ReadEndArray(); + } + + [Fact] + public void LargeValidPayload_StreamsWithoutBufferingWholePayload() + { + // A valid payload larger than the buffer should read correctly while the buffer stays bounded. + const int count = 2000; + var writer = new CborWriter(); + writer.WriteStartArray(count); + for (int i = 0; i < count; i++) + writer.WriteInt32(i); + writer.WriteEndArray(); + var payload = writer.Encode(); + Assert.True(payload.Length > 20 * AWSConfigs.CborReaderInitialBufferSize); + + using var reader = new CborStreamReader(new MemoryStream(payload)); + Assert.Equal(count, reader.ReadStartArray()); + for (int i = 0; i < count; i++) + Assert.Equal(i, reader.ReadInt32()); + reader.ReadEndArray(); + + Assert.True(GetBuffer(reader).Length < payload.Length, + $"Buffer ({GetBuffer(reader).Length}) should stay below the full payload size ({payload.Length})."); + } } diff --git a/sdk/Directory.Build.props b/sdk/Directory.Build.props index 4f5f92e48e08..b108f4caa1ca 100644 --- a/sdk/Directory.Build.props +++ b/sdk/Directory.Build.props @@ -47,7 +47,7 @@ --> netstandard2.0;net8.0;net10.0;net11.0 - net10.0 + net11.0 netcoreapp3.1;net5.0;net6.0;net7.0 diff --git a/sdk/test/Performance/SerdeBenchmarks/SerdeBenchmarksRunner/SerdeBenchmarksRunner.csproj b/sdk/test/Performance/SerdeBenchmarks/SerdeBenchmarksRunner/SerdeBenchmarksRunner.csproj index a75d24def91d..86192dea212a 100644 --- a/sdk/test/Performance/SerdeBenchmarks/SerdeBenchmarksRunner/SerdeBenchmarksRunner.csproj +++ b/sdk/test/Performance/SerdeBenchmarks/SerdeBenchmarksRunner/SerdeBenchmarksRunner.csproj @@ -2,9 +2,7 @@ Exe - - net11.0 + $(SdkPerformanceTargets) enable enable From 5f3ecb41e9146b141bd31c7b5a8e69113f0c69f4 Mon Sep 17 00:00:00 2001 From: Sichan Yoo Date: Wed, 28 Oct 2026 10:50:54 -0700 Subject: [PATCH 4/7] Reference System.Formats.Cbor 11.0.0-rc.2 for net472 and in the nuspec The incremental reader is no longer gated on net11, so every target needs the 11.0 package. The NetFramework project and the nuspec still referenced 9.0.5, which broke the net472 build and would have shipped a 9.0.5 dependency. --- .../AWSSDK.Extensions.CborProtocol.NetFramework.csproj | 2 +- .../AWSSDK.Extensions.CborProtocol.nuspec | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetFramework.csproj b/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetFramework.csproj index 8b7e6ef2a261..5e5b938f7e38 100644 --- a/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetFramework.csproj +++ b/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.NetFramework.csproj @@ -36,7 +36,7 @@ - + diff --git a/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.nuspec b/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.nuspec index be49c300b99f..894ff59e21cc 100644 --- a/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.nuspec +++ b/extensions/src/AWSSDK.Extensions.CborProtocol/AWSSDK.Extensions.CborProtocol.nuspec @@ -14,23 +14,23 @@ - + - + - + - + - + From c50abf7fedf23082126a51bbb6487a66d04018fb Mon Sep 17 00:00:00 2001 From: Sichan Yoo Date: Wed, 28 Oct 2026 11:47:13 -0700 Subject: [PATCH 5/7] Update zip artifact manifests for System.Formats.Cbor 11.0 System.Formats.Cbor 11.0 depends on System.ValueTuple 4.6.2, which treats net471+ as in-box, so net472 no longer publishes System.ValueTuple.dll. It also has no net8.0 asset, so net8.0 uses the netstandard2.0 build and now publishes Microsoft.Bcl.HashCode.dll. --- buildtools/ZipArtifactGenerator/manifests/net472.txt | 1 - buildtools/ZipArtifactGenerator/manifests/net8.0.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/buildtools/ZipArtifactGenerator/manifests/net472.txt b/buildtools/ZipArtifactGenerator/manifests/net472.txt index 0b36a053f646..5d652cdcc7c0 100644 --- a/buildtools/ZipArtifactGenerator/manifests/net472.txt +++ b/buildtools/ZipArtifactGenerator/manifests/net472.txt @@ -12,7 +12,6 @@ extensions\CborProtocol\System.Runtime.CompilerServices.Unsafe.dll extensions\CborProtocol\System.Text.Encodings.Web.dll extensions\CborProtocol\System.Text.Json.dll extensions\CborProtocol\System.Threading.Tasks.Extensions.dll -extensions\CborProtocol\System.ValueTuple.dll extensions\CloudFrontSigners\AWSSDK.Extensions.CloudFront.Signers.dll extensions\CloudFrontSigners\BouncyCastle.Cryptography.dll extensions\CloudFrontSigners\Microsoft.Bcl.AsyncInterfaces.dll diff --git a/buildtools/ZipArtifactGenerator/manifests/net8.0.txt b/buildtools/ZipArtifactGenerator/manifests/net8.0.txt index 567513239169..60a2bde3aae8 100644 --- a/buildtools/ZipArtifactGenerator/manifests/net8.0.txt +++ b/buildtools/ZipArtifactGenerator/manifests/net8.0.txt @@ -1,6 +1,7 @@ AWSSDK.Core.dll System.IO.Hashing.dll extensions\CborProtocol\AWSSDK.Extensions.CborProtocol.dll +extensions\CborProtocol\Microsoft.Bcl.HashCode.dll extensions\CborProtocol\System.Formats.Cbor.dll extensions\CborProtocol\System.IO.Hashing.dll extensions\CloudFrontSigners\AWSSDK.Extensions.CloudFront.Signers.dll From c487235e5e2d7d5102ed65747b17072a61004065 Mon Sep 17 00:00:00 2001 From: Sichan Yoo Date: Wed, 28 Oct 2026 12:45:30 -0700 Subject: [PATCH 6/7] Keep the performance runners on net10.0 The incremental reader is no longer gated on net11, so a net10.0 runner already exercises it through System.Formats.Cbor 11.0. The Catapult perf test hosts only have the .NET 10 runtime, so a net11.0 runner fails to start. --- sdk/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/Directory.Build.props b/sdk/Directory.Build.props index b108f4caa1ca..4f5f92e48e08 100644 --- a/sdk/Directory.Build.props +++ b/sdk/Directory.Build.props @@ -47,7 +47,7 @@ --> netstandard2.0;net8.0;net10.0;net11.0 - net11.0 + net10.0 netcoreapp3.1;net5.0;net6.0;net7.0 From 0394bb8f9b1314436b646033eb2c24dad30b00cb Mon Sep 17 00:00:00 2001 From: Sichan Yoo Date: Wed, 28 Oct 2026 15:17:28 -0700 Subject: [PATCH 7/7] Retry multi-item CBOR reads that straddle a chunk edge PeekState only waits for the next item's header, so indefinite-length strings and decimal fractions could still run past the buffered chunk and throw. For values starting with 0x5F, 0x7F or 0xC4, refill and retry until the read succeeds or the stream is exhausted. Adds boundary-sweep and truncation tests. --- .../Internal/CborStreamReader.cs | 55 ++++++--- .../CborStreamReaderTests.cs | 105 ++++++++++++++++++ 2 files changed, 147 insertions(+), 13 deletions(-) diff --git a/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs b/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs index 45f5409e7911..e1fe1725f1bd 100644 --- a/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs +++ b/extensions/src/AWSSDK.Extensions.CborProtocol/Internal/CborStreamReader.cs @@ -108,28 +108,57 @@ private void Fill() } /// - /// Executes a single-token CBOR read, refilling first while the reader reports - /// so the token is fully buffered. A - /// from the read is a malformed payload and propagates. + /// Executes a CBOR read, refilling while the reader reports . + /// That only covers the next item's header, so multi-item values (indefinite-length strings, decimal + /// fractions) are also retried after a refill until they succeed or the stream is exhausted. /// private T ExecuteRead(Func readOperation) { EnsureStarted(); - FillWhileNeedsMoreData(); - try - { - return readOperation(this, _internalCborReader); - } - catch (InvalidOperationException ex) + while (true) { - // The incremental reader raises InvalidOperationException for structural misuse, such as - // ending the wrong container type or reading a value as the wrong major type on a malformed - // payload. Surface it as a content error so callers see the same exception as on other targets. - throw new CborContentException(ex.Message, ex); + FillWhileNeedsMoreData(); + + try + { + return readOperation(this, _internalCborReader); + } + catch (Exception ex) when ((ex is CborContentException || ex is InvalidOperationException) + && !_streamExhausted && NextItemIsMultiItemValue()) + { + // A failed read leaves the reader at the start of the value. Scoped to multi-item values so a malformed + // ordinary value fails fast instead of pulling the rest of the stream into the buffer. + _logger.DebugFormat("Multi-item CBOR value spans a chunk boundary; refilling and retrying."); + Fill(); + } + catch (InvalidOperationException ex) + { + // The incremental reader raises InvalidOperationException for structural misuse, such as + // ending the wrong container type or reading a value as the wrong major type on a malformed + // payload. Surface it as a content error so callers see the same exception as with the previous reader. + throw new CborContentException(ex.Message, ex); + } } } + // Initial bytes of values the reader decodes as several CBOR items in one call (RFC 8949 §3.2.3, §3.4.4). + private const byte IndefiniteLengthByteString = 0x5F; + private const byte IndefiniteLengthTextString = 0x7F; + private const byte DecimalFractionTag = 0xC4; + + private bool NextItemIsMultiItemValue() + { + int remaining = _internalCborReader.BytesRemaining; + if (remaining <= 0) + return false; + + byte initialByte = _buffer[_slidLength - remaining]; + return initialByte == IndefiniteLengthByteString + || initialByte == IndefiniteLengthTextString + || initialByte == DecimalFractionTag; + } + private void EnsureStarted() { if (!_hasReadFromStream) diff --git a/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs b/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs index 37bf9632998f..3d7c2798eccd 100644 --- a/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs +++ b/extensions/test/CborProtocol.Tests/CborStreamReaderTests.cs @@ -1369,4 +1369,109 @@ public void LargeValidPayload_StreamsWithoutBufferingWholePayload() Assert.True(GetBuffer(reader).Length < payload.Length, $"Buffer ({GetBuffer(reader).Length}) should stay below the full payload size ({payload.Length})."); } + + // Indefinite-length strings and decimals span several CBOR items, so they can straddle a chunk edge even with + // the header buffered. Sweep the value across several buffer lengths to hit every boundary position. + public enum CompositeValueKind { IndefiniteTextString, IndefiniteByteString, Decimal } + + private const int SweepOffsets = 400; + private const int Sentinel = 424242; + private static readonly string[] TextChunks = { "alpha-", "bravo-", "charlie-", "delta-", "echo" }; + private static readonly byte[][] ByteChunks = { new byte[] { 1, 2, 3 }, Enumerable.Range(0, 40).Select(i => (byte)i).ToArray(), new byte[] { 9 } }; + private const decimal DecimalValue = 1234567890.123456789m; + + private static byte[] BuildCompositePayload(CompositeValueKind kind, int offset) + { + // [ padding byte string, value, sentinel ]. The padding length shifts where the value starts. + var writer = new CborWriter(CborConformanceMode.Lax); + writer.WriteStartArray(3); + writer.WriteByteString(new byte[offset]); + switch (kind) + { + case CompositeValueKind.IndefiniteTextString: + writer.WriteStartIndefiniteLengthTextString(); + foreach (var c in TextChunks) writer.WriteTextString(c); + writer.WriteEndIndefiniteLengthTextString(); + break; + case CompositeValueKind.IndefiniteByteString: + writer.WriteStartIndefiniteLengthByteString(); + foreach (var c in ByteChunks) writer.WriteByteString(c); + writer.WriteEndIndefiniteLengthByteString(); + break; + case CompositeValueKind.Decimal: + writer.WriteDecimal(DecimalValue); + break; + } + writer.WriteInt32(Sentinel); + writer.WriteEndArray(); + return writer.Encode(); + } + + private static void ReadAndAssertCompositeValue(CborStreamReader reader, CompositeValueKind kind) + { + switch (kind) + { + case CompositeValueKind.IndefiniteTextString: + Assert.Equal(string.Concat(TextChunks), reader.ReadTextString()); + break; + case CompositeValueKind.IndefiniteByteString: + Assert.Equal(ByteChunks.SelectMany(c => c).ToArray(), reader.ReadByteString()); + break; + case CompositeValueKind.Decimal: + Assert.Equal(DecimalValue, reader.ReadDecimal()); + break; + } + } + + [Theory] + [InlineData(CompositeValueKind.IndefiniteTextString)] + [InlineData(CompositeValueKind.IndefiniteByteString)] + [InlineData(CompositeValueKind.Decimal)] + public void CompositeValue_StraddlingChunkEdge_ReadsAtEveryOffset(CompositeValueKind kind) + { + var failures = new System.Collections.Generic.List(); + for (int offset = 0; offset < SweepOffsets; offset++) + { + try + { + using var reader = new CborStreamReader(new MemoryStream(BuildCompositePayload(kind, offset))); + Assert.Equal(3, reader.ReadStartArray()); + Assert.Equal(offset, reader.ReadByteString().Length); + ReadAndAssertCompositeValue(reader, kind); + Assert.Equal(Sentinel, reader.ReadInt32()); + reader.ReadEndArray(); + } + catch (Exception ex) + { + failures.Add($"offset {offset}: {ex.GetType().Name}: {ex.Message}"); + } + } + + Assert.True(failures.Count == 0, + $"{kind}: {failures.Count}/{SweepOffsets} offsets failed. First: {failures.FirstOrDefault()}"); + } + + [Theory] + [InlineData(CompositeValueKind.IndefiniteTextString)] + [InlineData(CompositeValueKind.IndefiniteByteString)] + [InlineData(CompositeValueKind.Decimal)] + public void CompositeValue_TruncatedInsideValue_ThrowsCborContentException(CompositeValueKind kind) + { + // Starts the value just before the first refill boundary, so the read must refill before it hits the truncation. + int offset = AWSConfigs.CborReaderInitialBufferSize - 4; + var truncated = BuildCompositePayload(kind, offset).Take(CompositeValueStart(offset) + 3).ToArray(); + + using var reader = new CborStreamReader(new MemoryStream(truncated)); + reader.ReadStartArray(); + reader.ReadByteString(); + Assert.Throws(() => ReadAndAssertCompositeValue(reader, kind)); + } + + private static int CompositeValueStart(int offset) + { + var writer = new CborWriter(CborConformanceMode.Lax); + writer.WriteStartArray(3); + writer.WriteByteString(new byte[offset]); + return writer.BytesWritten; + } }