From 290d201356165fe998946a745fe0f84e14d4b821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20IRMAK?= Date: Wed, 19 Aug 2026 15:42:59 +0300 Subject: [PATCH 1/2] implement EIP-7916 ProgressiveList and ProgressiveBitlist Add merkleizeProgressive() to lib.zig: a 0-terminated sequence of binary subtrees with leaf counts 1, 4, 16, 64, ..., reusing merkleize() for each fixed subtree. Refactor List/Bitlist into ListImpl/BitlistImpl taking a `limit: ?usize`, where null means progressive. List(T, N) and Bitlist(N) become thin wrappers, so their behaviour is unchanged. New public types: ProgressiveList(T), ProgressiveByteList and ProgressiveBitlist. Serialization is byte-identical to the bounded variants. Differences are confined to the capacity checks, which become no-ops, and merkleization: - maxInLength() returns error.NoMaxInLengthAvailable, since there is no static bound to check a payload against. This also changes Bitlist(N).maxInLength() from usize to !usize. - chunkCountLimit() is a @compileError, as a progressive tree has no fixed depth and so cannot be wrapped in TreeHasher. - ProgressiveBitlist does not trim trailing zero bytes before hashing. Bitlist(N) can, because zero chunks are padding in a fixed-depth tree, but dropping a chunk shifts the progressive subtree layout. Expected roots in the tests were generated by an independent Python transcription of the EIP pseudocode, covering chunk counts that straddle every subtree boundary plus deep cases at 1250 chunks and 20000 bits. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 35 +++++ src/lib.zig | 73 +++++++++++ src/tests.zig | 352 +++++++++++++++++++++++++++++++++++++++++++++++++- src/utils.zig | 129 +++++++++++++----- 4 files changed, 553 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 7f7c89e..82afd6e 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ Currently supported types: * **tagged** unions * `List[N]` * `Bitlist[N]` + * `ProgressiveList[T]` + * `ProgressiveBitlist` Ziglang has the limitation that it's not possible to determine which union field is active without tags. @@ -42,6 +44,8 @@ Supported types: * optionals * `List[N]` * `Bitlist[N]` + * `ProgressiveList[T]` + * `ProgressiveBitlist` ## Merkelization (experimental) @@ -59,6 +63,37 @@ Supported types: * unions * `List[N]` * `Bitlist[N]` + * `ProgressiveList[T]` + * `ProgressiveBitlist` + +## Progressive types (EIP-7916) + +`ProgressiveList(T)` and `ProgressiveBitlist` implement +[EIP-7916](https://eips.ethereum.org/EIPS/eip-7916). They serialize exactly like +`List(T, N)` and `Bitlist(N)`, but carry no capacity limit and merkleize with +`merkleizeProgressive`: a 0-terminated sequence of binary subtrees whose leaf +counts grow 1, 4, 16, 64, ... This costs fewer hashes for short lists and keeps +generalized indices stable as the list grows. + +```zig +const Transactions = ssz.utils.ProgressiveList(u64); +var txs = try Transactions.init(allocator); +defer txs.deinit(); +try txs.append(42); +try ssz.hashTreeRoot(Sha256, Transactions, txs, &root, allocator); +``` + +`ProgressiveByteList` is an alias for `ProgressiveList(u8)`. + +Two consequences of having no `N`: + + * `maxInLength` returns `error.NoMaxInLengthAvailable`, so `deserialize` cannot + reject an oversized payload up front. Decoding still allocates only in + proportion to the input, but callers that relied on `N` as a cheap sanity + bound should enforce their own context-specific limit, as the EIP recommends. + * `TreeHasher` cannot wrap a progressive type: a progressive tree has no fixed + depth, so the power-of-two Merkle cache does not apply. Using it is a compile + error. ## Using Custom Hash Functions diff --git a/src/lib.zig b/src/lib.zig index c10c268..9bbe72d 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -898,6 +898,79 @@ test "merkleize a bytes16 vector with one element" { // try std.testing.expect(std.mem.eql(u8, out[0..], expected[0..])); } +// merkleizeProgressive recursively calculates the root hash of an EIP-7916 +// progressive Merkle tree: a 0-terminated sequence of binary subtrees with leaf +// counts 1, 4, 16, 64, ... Callers start the recursion with `num_leaves = 1`. +// +// Trailing zero chunks are not padding here, so `chunks` must hold exactly +// ceil(serialized_len / BYTES_PER_CHUNK) entries. +pub fn merkleizeProgressive(Hasher: type, chunks: []chunk, num_leaves: usize, out: *[Hasher.digest_length]u8) anyerror!void { + // The 0-terminator. + if (chunks.len == 0) { + @memset(out[0..], 0); + return; + } + + // `merkleize` zero-pads the left subtree up to `num_leaves`. + const split = @min(num_leaves, chunks.len); + var buf: [Hasher.digest_length]u8 = undefined; + var digest = Hasher.init(Hasher.Options{}); + + try merkleize(Hasher, chunks[0..split], num_leaves, &buf); + digest.update(buf[0..]); + try merkleizeProgressive(Hasher, chunks[split..], num_leaves * 4, &buf); + digest.update(buf[0..]); + + digest.final(out); +} + +test "merkleizeProgressive of an empty slice is the zero chunk" { + const chunks = &[0][32]u8{}; + var out: [32]u8 = undefined; + try merkleizeProgressive(Sha256, chunks, 1, &out); + try std.testing.expectEqualSlices(u8, zero_chunk[0..], out[0..]); +} + +test "merkleizeProgressive subtree layout" { + var chunks: [5]chunk = undefined; + for (0..5) |i| chunks[i] = [_]u8{@intCast(i + 1)} ** 32; + + var got: [32]u8 = undefined; + var expected: [32]u8 = undefined; + var hasher = Sha256.init(Sha256.Options{}); + + // A single chunk fills the 1-leaf subtree, terminated by the zero chunk. + try merkleizeProgressive(Sha256, chunks[0..1], 1, &got); + hasher.update(chunks[0][0..]); + hasher.update(zero_chunk[0..]); + hasher.final(&expected); + try std.testing.expectEqualSlices(u8, expected[0..], got[0..]); + + // Five chunks fill the 1-leaf and the 4-leaf subtrees exactly. + try merkleizeProgressive(Sha256, chunks[0..5], 1, &got); + var second: [32]u8 = undefined; + try merkleize(Sha256, chunks[1..5], 4, &second); + var right: [32]u8 = undefined; + hasher = Sha256.init(Sha256.Options{}); + hasher.update(second[0..]); + hasher.update(zero_chunk[0..]); + hasher.final(&right); + hasher = Sha256.init(Sha256.Options{}); + hasher.update(chunks[0][0..]); + hasher.update(right[0..]); + hasher.final(&expected); + try std.testing.expectEqualSlices(u8, expected[0..], got[0..]); +} + +test "merkleizeProgressive is sensitive to trailing zero chunks" { + var chunks = [_]chunk{ [_]u8{0xAA} ** 32, zero_chunk }; + var one: [32]u8 = undefined; + var two: [32]u8 = undefined; + try merkleizeProgressive(Sha256, chunks[0..1], 1, &one); + try merkleizeProgressive(Sha256, chunks[0..2], 1, &two); + try std.testing.expect(!std.mem.eql(u8, one[0..], two[0..])); +} + fn packBits(bits: []const bool, l: *ArrayList(u8), allocator: Allocator) ![]chunk { var byte: u8 = 0; for (bits, 0..) |bit, bitidx| { diff --git a/src/tests.zig b/src/tests.zig index a583a73..58b5e52 100644 --- a/src/tests.zig +++ b/src/tests.zig @@ -1132,7 +1132,7 @@ test "maxInLength for fixed and variable types" { try expect(try ListU64.maxInLength() == 16 * 8); const Bitlist32 = utils.Bitlist(32); - try expect(Bitlist32.maxInLength() == (32 + 7 + 1) / 8); + try expect(try Bitlist32.maxInLength() == (32 + 7 + 1) / 8); const ListList = utils.List(utils.List(u8, 4), 2); try expect(try ListList.maxInLength() == 2 * 4 + 2 * (4 * 1)); @@ -3001,6 +3001,356 @@ test "utils.Bitlist.clone copies backing storage independently" { try expect((try data.get(0)) == true); } +// EIP-7916 ProgressiveList / ProgressiveBitlist. The expected roots below come +// from an independent Python transcription of the EIP pseudocode, not from this +// library. + +fn expectRootHex(expected_hex: []const u8, actual: *const [32]u8) !void { + var expected: [32]u8 = undefined; + _ = try std.fmt.hexToBytes(expected[0..], expected_hex); + try std.testing.expectEqualSlices(u8, expected[0..], actual[0..]); +} + +test "ProgressiveList(u64) tree root matches the EIP-7916 reference" { + const PList = utils.ProgressiveList(u64); + const cases = [_]struct { n: u64, root: []const u8 }{ + // Empty merkleizes to the zero chunk, so the root is hash(zero, zero). + .{ .n = 0, .root = "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" }, + .{ .n = 1, .root = "905efb51c2764c2c7a4efb0548e372569df06db82115c3b1896c186632f3fe5b" }, + // 4 uint64 fill the 1-leaf subtree; the 5th spills into the 4-leaf one. + .{ .n = 4, .root = "95a2f252ed2659ccf75e8821f05757c4663fce68e89d0290abf5c33d772935ae" }, + .{ .n = 5, .root = "29918e0447260511bc5be0f7dbb9817201e16e30c56af228b9cb931a16e8799d" }, + // 5 chunks exactly fill the 1- and 4-leaf subtrees; 6 reaches the 16-leaf one. + .{ .n = 20, .root = "c8a62a1a5fc7f814fafecb1d510213b25bda25425ab31c1ad7ff63c62c78307d" }, + .{ .n = 21, .root = "ed360c03ecbdfbb6f4b1cf5d9cbf6887038423e31121700797de968a9969aaed" }, + .{ .n = 22, .root = "61f3eebb593ca31c113a9dfec164edea6d13272e20a5f8d0ab641c6e3e2222a9" }, + }; + + for (cases) |c| { + var list = try PList.init(std.testing.allocator); + defer list.deinit(); + var i: u64 = 1; + while (i <= c.n) : (i += 1) try list.append(i); + + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PList, list, &root, std.testing.allocator); + try expectRootHex(c.root, &root); + } +} + +test "ProgressiveByteList tree root matches the EIP-7916 reference" { + const PBytes = utils.ProgressiveByteList; + + // Every chunk pack() produces counts, so all-zero payloads still change + // root with each added chunk. + const zero_cases = [_]struct { n: usize, root: []const u8 }{ + .{ .n = 0, .root = "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" }, + .{ .n = 1, .root = "e832d263aaa8f9417d9f45a702834f6961ee7b15ad4d3d27f2b0f4fe79d33031" }, + .{ .n = 32, .root = "e36306f41e65a19bc26226df4c969ef1ae6ac2e29edf4038761d553854385723" }, + .{ .n = 33, .root = "c33f5d2d028d955914dc4b72eb30c84ebb64b68df169054c5424b86833a8b171" }, + .{ .n = 100, .root = "c1757df1aa61464c2e0f033c88cb25aefde6c01a1cd41f2b421dffa178bce517" }, + .{ .n = 160, .root = "182743acd50e5d3765ba7c19e48f8a5c899b46ea95441045d1abb296106baa6a" }, + .{ .n = 161, .root = "f87031fc1210ed606d80f277dbbb765095d4b471720fe3371b8773fad46d02c0" }, + }; + for (zero_cases) |c| { + var list = try PBytes.init(std.testing.allocator); + defer list.deinit(); + for (0..c.n) |_| try list.append(0); + + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PBytes, list, &root, std.testing.allocator); + try expectRootHex(c.root, &root); + } + + const cases = [_]struct { n: usize, root: []const u8 }{ + .{ .n = 1, .root = "905efb51c2764c2c7a4efb0548e372569df06db82115c3b1896c186632f3fe5b" }, + .{ .n = 5, .root = "209ec0633411cff6970c26380d214e30985d43dcc50509c1b3b28f615d333939" }, + .{ .n = 33, .root = "bdb0c331db145d1efad9e022c70ab1f1c0896e7fc8bd8a83c6f0cd6ca89e1009" }, + .{ .n = 200, .root = "9152ead04c2a922ed04f51d7e2410c6f856f4ff670a99b86c880ae8cf92124a1" }, + }; + for (cases) |c| { + var list = try PBytes.init(std.testing.allocator); + defer list.deinit(); + for (0..c.n) |i| try list.append(@truncate(i + 1)); + + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PBytes, list, &root, std.testing.allocator); + try expectRootHex(c.root, &root); + } +} + +test "ProgressiveList of composite items tree root matches the EIP-7916 reference" { + // Bytes32 elements hash to themselves, so the chunks are the raw values. + const PList = utils.ProgressiveList([32]u8); + const cases = [_]struct { n: usize, root: []const u8 }{ + .{ .n = 1, .root = "a21da97c8a597221c87c9ea5ecdfbd860fcd52fd6fb5b001723f6437856c8df1" }, + .{ .n = 2, .root = "9a4badc45a45e9dd4b131c2c1aaff8a054527d8db40d2a7cd07e8f0f02a8232b" }, + .{ .n = 5, .root = "183886e81b2e887d5960b2fa49b3464eabee62ec55ff5e6ee6f7e0495d8a01d1" }, + .{ .n = 21, .root = "93589633f10a1e8fe51bef0481731c7c19d7a87269127b6c6a19720668ee47da" }, + .{ .n = 22, .root = "e79bdcda4e58dd09c4b855964e1f1c01c99e215b6e01602f5302763effaf8637" }, + }; + + for (cases) |c| { + var list = try PList.init(std.testing.allocator); + defer list.deinit(); + for (0..c.n) |i| try list.append([_]u8{@truncate(i + 1)} ** 32); + + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PList, list, &root, std.testing.allocator); + try expectRootHex(c.root, &root); + } +} + +test "ProgressiveBitlist tree root matches the EIP-7916 reference" { + const PBits = utils.ProgressiveBitlist; + + const set_cases = [_]struct { n: usize, root: []const u8 }{ + .{ .n = 0, .root = "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" }, + .{ .n = 1, .root = "905efb51c2764c2c7a4efb0548e372569df06db82115c3b1896c186632f3fe5b" }, + .{ .n = 8, .root = "89b4e102035da473eaf22c286e07d433e11cbd721578e55111e6e3381e44a485" }, + .{ .n = 256, .root = "b3327406854ffab96af59832dfa3f690f72c4f898e2ffd4ef3e90cc2fb876b43" }, + .{ .n = 257, .root = "be707c375a49431fdb06c00f7a4dcc9200d5613ea02999dc5e081913171bb8d0" }, + .{ .n = 300, .root = "8ab2de07a48c321a99ae0e54769d97d3b7f9d538c404ad6290db6ee40bcbd63d" }, + }; + for (set_cases) |c| { + var bits = try PBits.init(std.testing.allocator); + defer bits.deinit(); + for (0..c.n) |_| try bits.append(true); + + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PBits, bits, &root, std.testing.allocator); + try expectRootHex(c.root, &root); + } + + // Bitlist[N] trims trailing zero bytes before merkleizing; a progressive + // bitlist must not, since dropping a chunk changes the root. + const zero_cases = [_]struct { n: usize, root: []const u8 }{ + .{ .n = 1, .root = "e832d263aaa8f9417d9f45a702834f6961ee7b15ad4d3d27f2b0f4fe79d33031" }, + .{ .n = 8, .root = "8d709c6c23946fc63d47fefdf2466e87914380ddd1200753a093469535fdc776" }, + .{ .n = 256, .root = "09756b4ed11db307f098b2c1c543ae5348eadd79bd0413dcb941e4fbfe43592c" }, + .{ .n = 257, .root = "20833147423c1ffcdf357e57b00e48b2425dea1a96663333acbad1caeaa14653" }, + .{ .n = 300, .root = "143b6995489128f2afce0fcbbaf35c530d15af97ee2f6d11d4365c0200bef5de" }, + }; + for (zero_cases) |c| { + var bits = try PBits.init(std.testing.allocator); + defer bits.deinit(); + for (0..c.n) |_| try bits.append(false); + + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PBits, bits, &root, std.testing.allocator); + try expectRootHex(c.root, &root); + } + + var alternating = try PBits.init(std.testing.allocator); + defer alternating.deinit(); + for (0..13) |i| try alternating.append(i % 2 == 0); + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PBits, alternating, &root, std.testing.allocator); + try expectRootHex("a231c639a6becb4e98a86764f81e94b4a4ab7f47c1ff2da0bcb0b474319ea07e", &root); +} + +test "ProgressiveList is unbounded and merkleizes deep subtrees" { + // 5000 uint64 = 1250 chunks, reaching the sixth (1024-leaf) subtree. + const PList = utils.ProgressiveList(u64); + var list = try PList.init(std.testing.allocator); + defer list.deinit(); + for (0..5000) |i| try list.append(@intCast(i % 256)); + + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PList, list, &root, std.testing.allocator); + try expectRootHex("2ccd559b9db69d560a737ea1bd9a97a2d3557ef9eba116dd5a12c97d7079bd11", &root); + + const PBits = utils.ProgressiveBitlist; + var bits = try PBits.init(std.testing.allocator); + defer bits.deinit(); + for (0..20000) |i| try bits.append((i * 7) % 3 == 0); + + try hashTreeRoot(Sha256, PBits, bits, &root, std.testing.allocator); + try expectRootHex("fc22c6f8885c3813f8b0254e21aea8678954339685aef5a1bf49651bbbc2bd81", &root); +} + +test "ProgressiveList serialization is identical to List[N]" { + const PList = utils.ProgressiveList(u64); + const BList = utils.List(u64, 1024); + + var progressive = try PList.init(std.testing.allocator); + defer progressive.deinit(); + var bounded = try BList.init(std.testing.allocator); + defer bounded.deinit(); + for (0..37) |i| { + try progressive.append(@intCast(i * 7 + 1)); + try bounded.append(@intCast(i * 7 + 1)); + } + + var progressive_buf: ArrayList(u8) = .empty; + defer progressive_buf.deinit(std.testing.allocator); + var bounded_buf: ArrayList(u8) = .empty; + defer bounded_buf.deinit(std.testing.allocator); + try serialize(PList, progressive, &progressive_buf, std.testing.allocator); + try serialize(BList, bounded, &bounded_buf, std.testing.allocator); + + try expect(std.mem.eql(u8, progressive_buf.items, bounded_buf.items)); + try expect(try serializedSize(PList, progressive) == try serializedSize(BList, bounded)); + // The merkleization, on the other hand, must differ. + var progressive_root: [32]u8 = undefined; + var bounded_root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PList, progressive, &progressive_root, std.testing.allocator); + try hashTreeRoot(Sha256, BList, bounded, &bounded_root, std.testing.allocator); + try expect(!std.mem.eql(u8, &progressive_root, &bounded_root)); +} + +test "ProgressiveBitlist serialization is identical to Bitlist[N]" { + const PBits = utils.ProgressiveBitlist; + const BBits = utils.Bitlist(1024); + + inline for (.{ 0, 1, 7, 8, 9, 300 }) |n| { + var progressive = try PBits.init(std.testing.allocator); + defer progressive.deinit(); + var bounded = try BBits.init(std.testing.allocator); + defer bounded.deinit(); + for (0..n) |i| { + try progressive.append(i % 3 == 0); + try bounded.append(i % 3 == 0); + } + + var progressive_buf: ArrayList(u8) = .empty; + defer progressive_buf.deinit(std.testing.allocator); + var bounded_buf: ArrayList(u8) = .empty; + defer bounded_buf.deinit(std.testing.allocator); + try serialize(PBits, progressive, &progressive_buf, std.testing.allocator); + try serialize(BBits, bounded, &bounded_buf, std.testing.allocator); + + try expect(std.mem.eql(u8, progressive_buf.items, bounded_buf.items)); + try expect(progressive.serializedSize() == bounded.serializedSize()); + } +} + +test "(de)serialize ProgressiveList of fixed-length objects" { + const PList = utils.ProgressiveList(u64); + var list = try PList.init(std.testing.allocator); + defer list.deinit(); + for (0..3000) |i| try list.append(i * 100); + + var buf: ArrayList(u8) = .empty; + defer buf.deinit(std.testing.allocator); + try serialize(PList, list, &buf, std.testing.allocator); + + var deser = try PList.init(std.testing.allocator); + defer deser.deinit(); + try deserialize(PList, buf.items, &deser, std.testing.allocator); + try expect(list.eql(&deser)); +} + +test "(de)serialize ProgressiveList of variable-length objects" { + const PList = utils.ProgressiveList([]const u8); + var list = try PList.init(std.testing.allocator); + defer list.deinit(); + for (0..10) |i| { + try list.append(try std.fmt.allocPrint(std.testing.allocator, "count={}", .{i})); + } + defer for (0..list.len()) |i| { + std.testing.allocator.free(list.get(i) catch unreachable); + }; + + var buf: ArrayList(u8) = .empty; + defer buf.deinit(std.testing.allocator); + try serialize(PList, list, &buf, std.testing.allocator); + + var deser = try PList.init(std.testing.allocator); + defer deser.deinit(); + try deserialize(PList, buf.items, &deser, std.testing.allocator); + try expect(list.len() == deser.len()); + for (0..list.len()) |i| { + try expect(std.mem.eql(u8, try list.get(i), try deser.get(i))); + } +} + +test "(de)serialize ProgressiveBitlist" { + const PBits = utils.ProgressiveBitlist; + inline for (.{ 0, 1, 8, 300, 4097 }) |n| { + var bits = try PBits.init(std.testing.allocator); + defer bits.deinit(); + for (0..n) |i| try bits.append(i % 5 == 0); + + var buf: ArrayList(u8) = .empty; + defer buf.deinit(std.testing.allocator); + try serialize(PBits, bits, &buf, std.testing.allocator); + + var deser: PBits = undefined; + try deserialize(PBits, buf.items, &deser, std.testing.allocator); + defer deser.deinit(); + try expect(deser.len() == n); + try expect(bits.eql(&deser)); + } +} + +test "ProgressiveList as a struct field" { + const Block = struct { + slot: u64, + transactions: utils.ProgressiveList(u64), + flags: utils.ProgressiveBitlist, + }; + + var block = Block{ + .slot = 42, + .transactions = try utils.ProgressiveList(u64).init(std.testing.allocator), + .flags = try utils.ProgressiveBitlist.init(std.testing.allocator), + }; + defer block.transactions.deinit(); + defer block.flags.deinit(); + for (0..40) |i| try block.transactions.append(@intCast(i)); + for (0..40) |i| try block.flags.append(i % 2 == 0); + + try expect(!try isFixedSizeObject(Block)); + + var buf: ArrayList(u8) = .empty; + defer buf.deinit(std.testing.allocator); + try serialize(Block, block, &buf, std.testing.allocator); + + var deser: Block = undefined; + try deserialize(Block, buf.items, &deser, std.testing.allocator); + defer deser.transactions.deinit(); + defer deser.flags.deinit(); + + try expect(deser.slot == 42); + try expect(block.transactions.eql(&deser.transactions)); + try expect(block.flags.eql(&deser.flags)); + + var root: [32]u8 = undefined; + var deser_root: [32]u8 = undefined; + try hashTreeRoot(Sha256, Block, block, &root, std.testing.allocator); + try hashTreeRoot(Sha256, Block, deser, &deser_root, std.testing.allocator); + try expect(std.mem.eql(u8, &root, &deser_root)); +} + +test "progressive types have no static maxInLength" { + const PList = utils.ProgressiveList(u64); + const PBits = utils.ProgressiveBitlist; + + try expectError(error.NoMaxInLengthAvailable, libssz.maxInLength(PList)); + try expectError(error.NoMaxInLengthAvailable, libssz.maxInLength(PBits)); + try expect(try libssz.minInLength(PList) == 0); + try expect(try libssz.minInLength(PBits) == 1); + + // The error must propagate through a containing struct rather than + // silently yielding a bogus bound. + const S = struct { a: u32, b: PList }; + try expectError(error.NoMaxInLengthAvailable, libssz.maxInLength(S)); +} + +test "ProgressiveBitlist rejects malformed encodings but not long ones" { + const PBits = utils.ProgressiveBitlist; + + // The sentinel bit is still mandatory, a zero trailing byte still invalid. + try expectError(error.InvalidBitlistEncoding, PBits.validateBitlist(&[_]u8{})); + try expectError(error.BitlistTrailingByteZero, PBits.validateBitlist(&[_]u8{ 0xff, 0x00 })); + + // A payload that Bitlist(16) rejects on length grounds is fine here. + const long = [_]u8{0xff} ** 64; + try expectError(error.BitlistTooManyBytes, utils.Bitlist(16).validateBitlist(&long)); + try PBits.validateBitlist(&long); +} + test { _ = @import("beacon_tests.zig"); _ = @import("merkle_cache.zig"); diff --git a/src/utils.zig b/src/utils.zig index eda4f29..4670708 100644 --- a/src/utils.zig +++ b/src/utils.zig @@ -15,9 +15,28 @@ const zero_chunk = lib.zero_chunk; /// Implements the SSZ `List[N]` container. pub fn List(T: type, comptime N: usize) type { + return ListImpl(T, N); +} + +/// Implements the EIP-7916 `ProgressiveList[T]` container: serialized like +/// `List[T, N]`, merkleized progressively, with no capacity limit. +pub fn ProgressiveList(T: type) type { + return ListImpl(T, null); +} + +/// EIP-7916 `ProgressiveByteList`. +pub const ProgressiveByteList = ProgressiveList(u8); + +/// Backing implementation of `List` and `ProgressiveList`. `limit` is the +/// maximum element count, or `null` for a progressive list. +fn ListImpl(T: type, comptime limit: ?usize) type { // Compile-time check: List[bool, N] is not allowed, use Bitlist[N] instead if (T == bool) { - @compileError("List[bool, N] is not supported. Use Bitlist(" ++ std.fmt.comptimePrint("{}", .{N}) ++ ") instead for boolean lists."); + if (limit) |n| { + @compileError("List[bool, N] is not supported. Use Bitlist(" ++ std.fmt.comptimePrint("{}", .{n}) ++ ") instead for boolean lists."); + } else { + @compileError("ProgressiveList(bool) is not supported. Use ProgressiveBitlist instead for boolean lists."); + } } // Compile-time check: integer items must be a supported SSZ width. @@ -55,11 +74,13 @@ pub fn List(T: type, comptime N: usize) type { } /// Maximum serialized byte length for List(T, N) with at most N elements. + /// A `ProgressiveList` is unbounded, so it has no static maximum. pub fn maxInLength() !usize { + const n = limit orelse return error.NoMaxInLengthAvailable; if (try lib.isFixedSizeObject(Item)) { - return N * try lib.serializedFixedSize(Item); + return n * try lib.serializedFixedSize(Item); } - return N * @sizeOf(u32) + N * try lib.maxInLength(Item); + return n * @sizeOf(u32) + n * try lib.maxInLength(Item); } /// Minimum serialized byte length for List(T, N) (empty list). @@ -74,7 +95,7 @@ pub fn List(T: type, comptime N: usize) type { if (comptime Self.Item == u8) { // bulk-copy fast path: bytes are their own SSZ encoding - if (serialized.len > N) return error.OffsetExceedsSize; + if (limit) |n| if (serialized.len > n) return error.OffsetExceedsSize; try out.inner.ensureTotalCapacityPrecise(alloc, serialized.len); out.inner.appendSliceAssumeCapacity(serialized); return; @@ -96,7 +117,7 @@ pub fn List(T: type, comptime N: usize) type { const pitch = try lib.serializedFixedSize(Self.Item); if (serialized.len % pitch != 0) return error.OffsetOrdering; const n_items = serialized.len / pitch; - if (n_items > N) return error.OffsetExceedsSize; + if (limit) |n| if (n_items > n) return error.OffsetExceedsSize; for (0..n_items) |i| { var item: Self.Item = undefined; @@ -152,7 +173,7 @@ pub fn List(T: type, comptime N: usize) type { } pub fn append(self: *Self, item: Self.Item) error{ Overflow, OutOfMemory }!void { - if (self.inner.items.len >= N) return error.Overflow; + if (limit) |n| if (self.inner.items.len >= n) return error.Overflow; try self.inner.append(self.allocator, item); } @@ -165,7 +186,7 @@ pub fn List(T: type, comptime N: usize) type { } pub fn fromSlice(allocator: Allocator, m: []const T) !Self { - if (m.len > N) return error.Overflow; + if (limit) |n| if (m.len > n) return error.Overflow; var inner: Inner = .empty; try inner.appendSlice(allocator, m); return .{ .inner = inner, .allocator = allocator }; @@ -193,30 +214,39 @@ pub fn List(T: type, comptime N: usize) type { pub fn hashTreeRoot(self: *const Self, Hasher: type, out: *[Hasher.digest_length]u8, allocator: Allocator) !void { const items = self.constSlice(); + var tmp: chunk = undefined; + switch (@typeInfo(Item)) { .int => { var list: ArrayList(u8) = .empty; defer list.deinit(allocator); const chunks = try lib.pack([]const Item, items, &list, allocator); - var tmp: chunk = undefined; - try lib.merkleize(Hasher, chunks, chunkCountLimit(), &tmp); - lib.mixInLength2(Hasher, tmp, items.len, out); + if (limit == null) { + try lib.merkleizeProgressive(Hasher, chunks, 1, &tmp); + } else { + try lib.merkleize(Hasher, chunks, chunkCountLimit(), &tmp); + } }, else => { var chunks: ArrayList(chunk) = .empty; defer chunks.deinit(allocator); - var tmp: chunk = undefined; + var leaf: chunk = undefined; for (items) |item| { - try lib.hashTreeRoot(Hasher, Item, item, &tmp, allocator); - try chunks.append(allocator, tmp); + try lib.hashTreeRoot(Hasher, Item, item, &leaf, allocator); + try chunks.append(allocator, leaf); + } + if (limit) |n| { + // Always use N (max capacity) for merkleization, even when empty, + // This ensures proper tree depth according to SSZ specification + try lib.merkleize(Hasher, chunks.items, n, &tmp); + } else { + try lib.merkleizeProgressive(Hasher, chunks.items, 1, &tmp); } - // Always use N (max capacity) for merkleization, even when empty, - // This ensures proper tree depth according to SSZ specification - try lib.merkleize(Hasher, chunks.items, N, &tmp); - lib.mixInLength2(Hasher, tmp, items.len, out); }, } + + lib.mixInLength2(Hasher, tmp, items.len, out); } // Leaf protocol consumed by TreeHasher (cached hashing) @@ -232,9 +262,11 @@ pub fn List(T: type, comptime N: usize) type { /// SSZ chunk-count limit (used to derive the merkleization depth). pub fn chunkCountLimit() usize { + if (limit == null) @compileError("ProgressiveList has no fixed chunk-count limit; TreeHasher(ProgressiveList(T), Hasher) is not supported"); + const n = limit.?; return switch (@typeInfo(Item)) { - .int => (N * @sizeOf(Item) + BYTES_PER_CHUNK - 1) / BYTES_PER_CHUNK, - else => N, + .int => (n * @sizeOf(Item) + BYTES_PER_CHUNK - 1) / BYTES_PER_CHUNK, + else => n, }; } @@ -273,8 +305,10 @@ pub fn List(T: type, comptime N: usize) type { } const length = offset / OFFSET_SIZE; - if (length > N) { - return error.DynamicLengthExceedsMax; + if (limit) |n| { + if (length > n) { + return error.DynamicLengthExceedsMax; + } } return length; @@ -284,6 +318,16 @@ pub fn List(T: type, comptime N: usize) type { /// Implements the SSZ `Bitlist[N]` container. pub fn Bitlist(comptime N: usize) type { + return BitlistImpl(N); +} + +/// Implements the EIP-7916 `ProgressiveBitlist` container: serialized like +/// `Bitlist[N]`, merkleized progressively, with no capacity limit. +pub const ProgressiveBitlist = BitlistImpl(null); + +/// Backing implementation of `Bitlist` and `ProgressiveBitlist`. `limit` is the +/// maximum bit count, or `null` for a progressive bitlist. +fn BitlistImpl(comptime limit: ?usize) type { return struct { const Self = @This(); pub const Item = bool; @@ -367,8 +411,10 @@ pub fn Bitlist(comptime N: usize) type { } /// Maximum serialized byte length for Bitlist(N) (N bits + sentinel). - pub fn maxInLength() usize { - return (N + 7 + 1) / 8; + /// A `ProgressiveBitlist` is unbounded, so it has no static maximum. + pub fn maxInLength() !usize { + const n = limit orelse return error.NoMaxInLengthAvailable; + return (n + 7 + 1) / 8; } /// Minimum serialized byte length for Bitlist(N) (empty bitlist: one byte with sentinel). @@ -393,7 +439,7 @@ pub fn Bitlist(comptime N: usize) type { } pub fn append(self: *Self, item: bool) error{ Overflow, OutOfMemory, IndexOutOfBounds }!void { - if (self.length >= N) return error.Overflow; + if (limit) |n| if (self.length >= n) return error.Overflow; if (self.length % 8 == 0) { try self.inner.append(self.allocator, 0); } @@ -429,10 +475,14 @@ pub fn Bitlist(comptime N: usize) type { const sl = self.inner.items; try bitfield_bytes.appendSlice(allocator, sl[0..sl.len]); - // Remove trailing zeros but keep at least one byte - // This avoids the wasteful pattern of removing all zeros and then adding back a chunk - while (bitfield_bytes.items.len > 1 and bitfield_bytes.items[bitfield_bytes.items.len - 1] == 0) { - _ = bitfield_bytes.pop(); + // Only safe for a fixed-depth tree: dropping a zero chunk would + // shift the progressive subtree layout. + if (limit != null) { + // Remove trailing zeros but keep at least one byte + // This avoids the wasteful pattern of removing all zeros and then adding back a chunk + while (bitfield_bytes.items.len > 1 and bitfield_bytes.items[bitfield_bytes.items.len - 1] == 0) { + _ = bitfield_bytes.pop(); + } } } @@ -443,7 +493,11 @@ pub fn Bitlist(comptime N: usize) type { const chunks = std.mem.bytesAsSlice(chunk, bitfield_bytes.items); var tmp: chunk = undefined; - try lib.merkleize(Hasher, chunks, chunkCountLimit(), &tmp); + if (limit == null) { + try lib.merkleizeProgressive(Hasher, chunks, 1, &tmp); + } else { + try lib.merkleize(Hasher, chunks, chunkCountLimit(), &tmp); + } lib.mixInLength2(Hasher, tmp, bit_length, out); } @@ -455,7 +509,8 @@ pub fn Bitlist(comptime N: usize) type { } pub fn chunkCountLimit() usize { - return (N + 255) / 256; + if (limit == null) @compileError("ProgressiveBitlist has no fixed chunk-count limit; TreeHasher(ProgressiveBitlist, Hasher) is not supported"); + return (limit.? + 255) / 256; } pub fn getLeafBytes(self: *const Self, idx: usize, out: *chunk, comptime Hasher: type, allocator: Allocator) !void { @@ -478,9 +533,11 @@ pub fn Bitlist(comptime N: usize) type { if (byte_len == 0) return error.InvalidBitlistEncoding; // Maximum possible bytes in a bitlist with provided bitlimit. - const max_bytes = ((N + 7 + 1) >> 3); - if (byte_len > max_bytes) { - return error.BitlistTooManyBytes; + if (limit) |n| { + const max_bytes = ((n + 7 + 1) >> 3); + if (byte_len > max_bytes) { + return error.BitlistTooManyBytes; + } } // The most significant bit is present in the last byte in the array. @@ -498,8 +555,10 @@ pub fn Bitlist(comptime N: usize) type { // bit. Subtract this value by 1 to determine the length of the bitlist. const num_of_bits: u64 = @intCast(8 * (byte_len - 1) + msb_pos - 1); - if (num_of_bits > N) { - return error.BitlistTooManyBits; + if (limit) |n| { + if (num_of_bits > n) { + return error.BitlistTooManyBits; + } } } }; From 32aadce1122174dbb185129c62133ef2f88156a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20IRMAK?= Date: Wed, 19 Aug 2026 15:53:04 +0300 Subject: [PATCH 2/2] implement EIP-7495/7688 ProgressiveContainer merkleization A struct opts in by declaring `pub const ssz_progressive_container = true`. Serialization is untouched; hashTreeRoot becomes hash(merkleize_progressive(field_roots), pack_bits(active_fields)) Only the all-active form EIP-7688 mandates is supported: active_fields is derived from the field count, so a field cannot be marked inactive. Expected roots in the tests come from eth-remerkleable, the reference implementation execution-specs uses. The ProgressiveList vectors added in the previous commit were also cross-checked against it and all match. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 15 ++++++ src/lib.zig | 31 +++++++++++- src/tests.zig | 134 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 82afd6e..6064d6e 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,21 @@ try ssz.hashTreeRoot(Sha256, Transactions, txs, &root, allocator); `ProgressiveByteList` is an alias for `ProgressiveList(u8)`. +A struct opts in to EIP-7495 / EIP-7688 `ProgressiveContainer(active_fields=[1] * N)` +merkleization by declaring a marker. Serialization is unchanged; only the root +differs, becoming `hash(merkleize_progressive(field_roots), pack_bits(active_fields))`. + +```zig +pub const ExecutionPayload = struct { + pub const ssz_progressive_container = true; + parent_hash: [32]u8, + // ... +}; +``` + +Only the all-active form EIP-7688 mandates is supported; `active_fields` is +derived from the field count, so there is no way to mark a field inactive. + Two consequences of having no `N`: * `maxInLength` returns `error.NoMaxInLengthAvailable`, so `deserialize` cannot diff --git a/src/lib.zig b/src/lib.zig index 9bbe72d..4c5571c 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -699,6 +699,30 @@ test "mixInLength" { try std.testing.expect(std.mem.eql(u8, mixin[0..], expected[0..])); } +fn mixInActiveFields(Hasher: type, root: [Hasher.digest_length]u8, active_fields: chunk, out: *[Hasher.digest_length]u8) void { + var hasher = Hasher.init(Hasher.Options{}); + hasher.update(root[0..]); + hasher.update(active_fields[0..]); + hasher.final(out[0..]); +} + +/// A struct opts in to EIP-7495 `ProgressiveContainer(active_fields=[1] * N)` +/// merkleization, as mandated by EIP-7688, by declaring +/// `pub const ssz_progressive_container = true;`. Serialization is unchanged. +pub fn isProgressiveContainer(T: type) bool { + return @hasDecl(T, "ssz_progressive_container") and T.ssz_progressive_container; +} + +/// pack_bits(active_fields) for `n` all-active fields. EIP-7495 caps +/// active_fields at 256 bits, so the result is always a single chunk. +fn activeFieldsChunk(comptime n: usize) chunk { + if (n == 0) @compileError("a progressive container needs at least one field"); + if (n > 256) @compileError("a progressive container may have at most 256 fields"); + var c: chunk = zero_chunk; + for (0..n) |i| c[i / 8] |= @as(u8, 1) << @truncate(i % 8); + return c; +} + fn mixInSelector(Hasher: type, root: [Hasher.digest_length]u8, comptime selector: usize, out: *[Hasher.digest_length]u8) void { var hasher = Hasher.init(Hasher.Options{}); hasher.update(root[0..]); @@ -1075,7 +1099,12 @@ pub fn hashTreeRoot(Hasher: type, T: type, value: T, out: *[Hasher.digest_length try hashTreeRoot(Hasher, f.type, @field(value, f.name), &tmp, allocator); try chunks.append(allocator, tmp); } - try merkleize(Hasher, chunks.items, null, out); + if (comptime isProgressiveContainer(T)) { + try merkleizeProgressive(Hasher, chunks.items, 1, &tmp); + mixInActiveFields(Hasher, tmp, comptime activeFieldsChunk(str.fields.len), out); + } else { + try merkleize(Hasher, chunks.items, null, out); + } }, // An optional is a union with `None` as first value. .optional => |opt| if (value != null) { diff --git a/src/tests.zig b/src/tests.zig index 58b5e52..a8c039f 100644 --- a/src/tests.zig +++ b/src/tests.zig @@ -3011,6 +3011,12 @@ fn expectRootHex(expected_hex: []const u8, actual: *const [32]u8) !void { try std.testing.expectEqualSlices(u8, expected[0..], actual[0..]); } +fn expectBytesHex(expected_hex: []const u8, actual: []const u8) !void { + var buf: [256]u8 = undefined; + const expected = try std.fmt.hexToBytes(buf[0..], expected_hex); + try std.testing.expectEqualSlices(u8, expected, actual); +} + test "ProgressiveList(u64) tree root matches the EIP-7916 reference" { const PList = utils.ProgressiveList(u64); const cases = [_]struct { n: u64, root: []const u8 }{ @@ -3351,6 +3357,134 @@ test "ProgressiveBitlist rejects malformed encodings but not long ones" { try PBits.validateBitlist(&long); } +// EIP-7495/7688 ProgressiveContainer. Expected roots come from eth-remerkleable, +// the reference implementation used by execution-specs. + +test "ProgressiveContainer tree root matches the reference" { + const PC3 = struct { + pub const ssz_progressive_container = true; + a: u64, + b: [32]u8, + c: utils.ProgressiveByteList, + }; + + var c = try utils.ProgressiveByteList.init(std.testing.allocator); + defer c.deinit(); + for (0..40) |_| try c.append(0xaa); + const value = PC3{ .a = 7, .b = @splat(0x11), .c = c }; + + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PC3, value, &root, std.testing.allocator); + try expectRootHex("bfbf57e1e4548f1f776af7e2d996c4110e8e87d7d1e438c1c7bc3206b108f9ad", &root); + + // The same fields without the marker merkleize as a plain Container. + const Plain = struct { a: u64, b: [32]u8, c: utils.ProgressiveByteList }; + var plain_root: [32]u8 = undefined; + try hashTreeRoot(Sha256, Plain, Plain{ .a = 7, .b = @splat(0x11), .c = c }, &plain_root, std.testing.allocator); + try expectRootHex("2c1e19ea8cbe5da65b867b2c62f8a9be4f1f38f046b15a48a7c96bbc0c59de91", &plain_root); + + // Serialization is unaffected by the marker. + var pc_buf: ArrayList(u8) = .empty; + defer pc_buf.deinit(std.testing.allocator); + var plain_buf: ArrayList(u8) = .empty; + defer plain_buf.deinit(std.testing.allocator); + try serialize(PC3, value, &pc_buf, std.testing.allocator); + try serialize(Plain, Plain{ .a = 7, .b = @splat(0x11), .c = c }, &plain_buf, std.testing.allocator); + try expect(std.mem.eql(u8, pc_buf.items, plain_buf.items)); + + try expectBytesHex("070000000000000011111111111111111111111111111111111111111111111111111111111111112c000000" ++ "aa" ** 40, pc_buf.items); +} + +test "ProgressiveContainer with one and nineteen fields" { + const PC1 = struct { + pub const ssz_progressive_container = true; + a: u64, + }; + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PC1, PC1{ .a = 1 }, &root, std.testing.allocator); + try expectRootHex("905efb51c2764c2c7a4efb0548e372569df06db82115c3b1896c186632f3fe5b", &root); + + // 19 fields is the ExecutionPayload shape: active_fields packs to + // ff ff 07 followed by zeroes. + const PC19 = struct { + pub const ssz_progressive_container = true; + f0: u64, + f1: u64, + f2: u64, + f3: u64, + f4: u64, + f5: u64, + f6: u64, + f7: u64, + f8: u64, + f9: u64, + f10: u64, + f11: u64, + f12: u64, + f13: u64, + f14: u64, + f15: u64, + f16: u64, + f17: u64, + f18: u64, + }; + var v19: PC19 = undefined; + inline for (@typeInfo(PC19).@"struct".fields, 0..) |f, i| { + @field(v19, f.name) = i + 1; + } + try hashTreeRoot(Sha256, PC19, v19, &root, std.testing.allocator); + try expectRootHex("c6de7b2d3af2d92e3136b2228b93460506fab2907e467eff356c1dae7944d47b", &root); +} + +test "ProgressiveContainer of ProgressiveLists matches the reference" { + // The ExecutionRequests shape: 5 active fields, each a ProgressiveList of + // a fixed-size container. + const Req = struct { x: u64 }; + const ReqList = utils.ProgressiveList(Req); + const PC5 = struct { + pub const ssz_progressive_container = true; + deposits: ReqList, + withdrawals: ReqList, + consolidations: ReqList, + builder_deposits: ReqList, + builder_exits: ReqList, + }; + + var lists: [5]ReqList = undefined; + for (0..5) |i| lists[i] = try ReqList.init(std.testing.allocator); + defer for (0..5) |i| lists[i].deinit(); + try lists[0].append(.{ .x = 1 }); + try lists[0].append(.{ .x = 2 }); + try lists[2].append(.{ .x = 3 }); + try lists[4].append(.{ .x = 4 }); + try lists[4].append(.{ .x = 5 }); + try lists[4].append(.{ .x = 6 }); + + const value = PC5{ + .deposits = lists[0], + .withdrawals = lists[1], + .consolidations = lists[2], + .builder_deposits = lists[3], + .builder_exits = lists[4], + }; + + var root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PC5, value, &root, std.testing.allocator); + try expectRootHex("998b7a575ff838ca9816b0b12bd6883a779e98a0036745f4cb645fe72baa5776", &root); + + var buf: ArrayList(u8) = .empty; + defer buf.deinit(std.testing.allocator); + try serialize(PC5, value, &buf, std.testing.allocator); + try expectBytesHex("1400000024000000240000002c0000002c000000010000000000000002000000000000000300000000000000040000000000000005000000000000000600000000000000", buf.items); + + var deser: PC5 = undefined; + try deserialize(PC5, buf.items, &deser, std.testing.allocator); + defer inline for (@typeInfo(PC5).@"struct".fields) |f| @field(deser, f.name).deinit(); + var deser_root: [32]u8 = undefined; + try hashTreeRoot(Sha256, PC5, deser, &deser_root, std.testing.allocator); + try expect(std.mem.eql(u8, &root, &deser_root)); +} + test { _ = @import("beacon_tests.zig"); _ = @import("merkle_cache.zig");