evm-rpc: trace state diffs per-transaction when a block's whole-block response is too big

common/changes/@subsquid/evm-rpc/alert-fix-L8b2z3-base-sepolia-statediff-pertx_2026-06-24-05-28.json

@@ -0,0 +1 @@
+{"changes":[{"packageName":"@subsquid/evm-rpc","comment":"fall back to per-transaction debug_traceTransaction when a whole-block state diff exceeds the provider response size limit","type":"patch"}],"packageName":"@subsquid/evm-rpc"}

evm/evm-rpc/src/rpc.ts

@@ -16,6 +16,7 @@ import {
     GetBlock,
     Receipt,
     TraceFrame,
+    DebugStateDiff,
     DebugStateDiffResult,
     DebugFrameResult,
     TraceReplayTraces,
@@ -1018,6 +1019,7 @@ export class Rpc {
             validateError: info => {
                 if (info.message.includes('not found')) return null
                 if (info.message.includes('cannot query unfinalized data')) return null // Avalanche
+                if (isResponseTooBigError(info)) return RESPONSE_TOO_BIG as any
                 throw new RpcError(info)
             }
         })
@@ -1026,6 +1028,13 @@ export class Rpc {
         for (let i = 0; i < blocks.length; i++) {
             let block = blocks[i]
             let diffs = results[i]
+            if ((diffs as any) === RESPONSE_TOO_BIG) {
+                // The whole-block prestateTracer response exceeded the provider's
+                // response-size cap. Retrying/splitting the block batch can't help
+                // (a single block can't be split), so re-fetch the state diff one
+                // transaction at a time, where each response is small.
+                diffs = await this.traceStateDiffsPerTransaction(block, traceConfig)
+            }
             if (diffs == null) {
                 block._isInvalid = true
                 block._errorMessage = "failed to get debug state diffs for a block"
@@ -1037,6 +1046,39 @@ export class Rpc {
         }
     }
 
+    private async traceStateDiffsPerTransaction(
+        block: Block,
+        traceConfig: object
+    ): Promise<DebugStateDiffResult[] | null> {
+        let txs = block.block.transactions
+        this.log.warn(
+            `debug state diff for block ${block.number} exceeded the provider response size limit; ` +
+            `falling back to per-transaction tracing of ${txs.length} transactions`
+        )
+
+        // debug_traceTransaction returns a bare state diff per call (no { result, txHash }
+        // envelope), so we validate the diff itself and reattach the tx hash below.
+        let diffs = await this.reduceBatchOnRetry(txs.map(tx => ({
+            method: 'debug_traceTransaction',
+            params: [getTxHash(tx), traceConfig]
+        })), {
+            validateResult: getResultValidator(DebugStateDiff),
+            validateError: info => {
+                if (info.message.includes('not found')) return null
+                if (info.message.includes('cannot query unfinalized data')) return null // Avalanche
+                throw new RpcError(info)
+            }
+        })
+
+        let out: DebugStateDiffResult[] = new Array(txs.length)
+        for (let i = 0; i < txs.length; i++) {
+            let diff = diffs[i]
+            if (diff == null) return null // a transaction couldn't be traced; flag block for retry
+            out[i] = { result: diff, txHash: getTxHash(txs[i]) }
+        }
+        return out
+    }
+
     private async addDebugFrames(blocks: Block[], req: DataRequest): Promise<void> {
         let traceConfig = {
             tracer: 'callTracer',
@@ -1260,6 +1302,27 @@ function isEmpty(obj: object): boolean {
 }
 
 
+// Marks a debug-trace batch item whose whole-block response exceeded the
+// provider's response-size cap and must be re-fetched per transaction.
+const RESPONSE_TOO_BIG = Symbol('evm-rpc:response-too-big')
+
+
+// geth/erigon and managed providers (alchemy, dwellir, uniblock, …) cap a single
+// JSON-RPC response (commonly 160 MiB) and reject an oversized debug trace with
+// this error. Unlike the *transient* "-32020 response too large" handled in
+// EvmRpcClient.isResponseTooLargeError, a whole-block prestateTracer diff for a
+// very large block exceeds the cap *persistently* — every retry returns the same
+// error and the block batch can't be split below a single block — so the only way
+// forward is to trace the block one transaction at a time.
+function isResponseTooBigError(info: { code?: number, message: string }): boolean {
+    return (
+        info.code === -32008 ||
+        /response is too big/i.test(info.message) ||
+        /exceeded max limit/i.test(info.message)
+    )
+}
+
+
 /**
  * Recursively collects `logs` arrays from a callTracer frame and all its
  * nested sub-call frames. Only `address` and `topics` are needed by

evm/evm-rpc/test/helpers/mock-rpc-client.ts

@@ -1,3 +1,10 @@
+// A JSON-RPC error returned by an upstream as a fixture, so that
+// validateError() is exercised the same way the real client exercises it.
+class MockRpcErrorFixture {
+    constructor(public info: { code?: number, message: string, data?: any }) {}
+}
+
+
 export class MockRpcClient {
     private fixtures: Map<string, any>
 
@@ -13,32 +20,55 @@ export class MockRpcClient {
         return false
     }
 
-    async call<T = any>(method: string, params?: any[]): Promise<T> {
+    async call<T = any>(method: string, params?: any[], options?: {validateError?: (info: any, call?: any) => any}): Promise<T> {
+        const fixture = this.lookup(method, params)
+        if (fixture instanceof MockRpcErrorFixture) {
+            if (options?.validateError) {
+                return options.validateError(fixture.info, {method, params})
+            }
+            const err: any = new Error(fixture.info.message)
+            err.code = fixture.info.code
+            throw err
+        }
+        return fixture
+    }
+
+    private lookup(method: string, params?: any[]): any {
         const key = this.makeKey(method, params)
         if (this.fixtures.has(key)) {
             return this.fixtures.get(key)
         }
-
         // Try without params
         if (this.fixtures.has(method)) {
             return this.fixtures.get(method)
         }
-
         throw new Error(`No fixture found for method: ${method} with params: ${JSON.stringify(params)}`)
     }
 
-    async batchCall(batch: {method: string, params?: any[]}[], options?: {validateResult?: (result: any) => any}): Promise<any[]> {
+    async batchCall(batch: {method: string, params?: any[]}[], options?: {validateResult?: (result: any) => any, validateError?: (info: any, call?: any) => any}): Promise<any[]> {
         const results = []
         for (const req of batch) {
+            let fixture: any
             try {
-                let result = await this.call(req.method, req.params)
-                if (options?.validateResult) {
-                    result = options.validateResult(result)
-                }
-                results.push(result)
+                fixture = this.lookup(req.method, req.params)
             } catch (error: any) {
                 results.push({error: error.message})
+                continue
+            }
+            if (fixture instanceof MockRpcErrorFixture) {
+                // Mirror the real client: an error item is passed to validateError,
+                // whose return value becomes the item's result. If validateError throws
+                // (a non-recoverable error), the whole batch rejects.
+                if (options?.validateError) {
+                    results.push(options.validateError(fixture.info, req))
+                } else {
+                    const err: any = new Error(fixture.info.message)
+                    err.code = fixture.info.code
+                    throw err
+                }
+                continue
             }
+            results.push(options?.validateResult ? options.validateResult(fixture) : fixture)
         }
         return results
     }
@@ -48,6 +78,11 @@ export class MockRpcClient {
         this.fixtures.set(key, response)
     }
 
+    setErrorFixture(method: string, params: any[] | undefined, info: { code?: number, message: string, data?: any }): void {
+        const key = this.makeKey(method, params)
+        this.fixtures.set(key, new MockRpcErrorFixture(info))
+    }
+
     private makeKey(method: string, params?: any[]): string {
         if (!params || params.length === 0) {
             return method

evm/evm-rpc/test/state-diff-too-big.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect } from 'vitest'
+import { loadBlock } from './helpers/fixture-loader'
+import { MockRpcClient } from './helpers/mock-rpc-client'
+import { Rpc } from '../src/rpc'
+import { toQty, getTxHash } from '../src/util'
+
+// Regression for the base-sepolia "No Dumper Data" stall: a very large block's
+// whole-block prestateTracer state-diff response exceeds the provider's response
+// size cap (geth/erigon/alchemy/dwellir all return JSON-RPC error -32008
+// "Response is too big" / "Exceeded max limit of 167772160"). Splitting/retrying
+// the block batch can't help — a single block can't be split — so the dumper used
+// to stall/crash-loop forever on that block. The fix falls back to per-transaction
+// debug_traceTransaction, where each response is small.
+
+const BLOCK = 18000000
+
+// Must match the traceConfig that Rpc.addDebugStateDiffs builds verbatim, since the
+// mock keys fixtures by JSON.stringify(params).
+const TRACE_CONFIG = {
+    tracer: 'prestateTracer',
+    tracerConfig: { onlyTopCall: false, diffMode: true },
+    timeout: '20s'
+}
+
+const REQUEST = {
+    transactions: true,
+    stateDiffs: true,
+    useDebugApiForStateDiffs: true,
+    useDebugTraceBlockByNumber: true,
+    debugTraceTimeout: '20s'
+}
+
+function diffFor(i: number) {
+    // a small, valid per-tx state diff; storage key varies per tx so we can assert wiring
+    let slot = '0x' + i.toString(16).padStart(64, '0')
+    return {
+        pre: { '0x4200000000000000000000000000000000000015': { storage: { [slot]: '0x00' } } },
+        post: { '0x4200000000000000000000000000000000000015': { storage: { [slot]: '0x01' } } }
+    }
+}
+
+describe('debug state diff response-too-big handling', () => {
+    it('falls back to per-transaction tracing when the whole-block state diff exceeds the size cap', async () => {
+        const block = loadBlock('ethereum', BLOCK)
+        const txs = block.transactions as any[]
+        expect(txs.length).toBeGreaterThan(1)
+
+        const mockClient = new MockRpcClient()
+        mockClient.setFixture('eth_chainId', undefined, '0x1')
+        mockClient.setFixture('eth_getBlockByNumber', [toQty(BLOCK), true], block)
+
+        // whole-block call rejects with the provider's size-cap error
+        mockClient.setErrorFixture('debug_traceBlockByNumber', [block.number, TRACE_CONFIG], {
+            code: -32008,
+            message: 'Response is too big',
+            data: 'Exceeded max limit of 167772160'
+        })
+        // per-transaction calls succeed with small responses
+        txs.forEach((tx, i) => {
+            mockClient.setFixture('debug_traceTransaction', [getTxHash(tx), TRACE_CONFIG], diffFor(i))
+        })
+
+        const rpc = new Rpc({ client: mockClient as any })
+
+        const blocks = await rpc.getBlockBatch([BLOCK], REQUEST)
+        expect(blocks).toHaveLength(1)
+
+        const diffs = blocks[0].debugStateDiffs!
+        expect(diffs).toBeTruthy()
+        expect(diffs.length).toEqual(txs.length)
+        // every per-tx diff is reattached to its transaction in order
+        for (let i = 0; i < txs.length; i++) {
+            expect(diffs[i]!.txHash).toEqual(getTxHash(txs[i]))
+            expect(diffs[i]!.result).toEqual(diffFor(i))
+        }
+        expect(blocks[0]._isInvalid).toBeFalsy()
+    })
+
+    it('uses the whole-block response directly when it fits (no per-tx fallback)', async () => {
+        const block = loadBlock('ethereum', BLOCK)
+        const txs = block.transactions as any[]
+
+        const mockClient = new MockRpcClient()
+        mockClient.setFixture('eth_chainId', undefined, '0x1')
+        mockClient.setFixture('eth_getBlockByNumber', [toQty(BLOCK), true], block)
+        mockClient.setFixture(
+            'debug_traceBlockByNumber',
+            [block.number, TRACE_CONFIG],
+            txs.map((tx, i) => ({ result: diffFor(i), txHash: getTxHash(tx) }))
+        )
+        // deliberately NO debug_traceTransaction fixtures — if the fallback fired,
+        // the missing-fixture error would surface.
+
+        const rpc = new Rpc({ client: mockClient as any })
+
+        const blocks = await rpc.getBlockBatch([BLOCK], REQUEST)
+        expect(blocks).toHaveLength(1)
+        expect(blocks[0].debugStateDiffs!.length).toEqual(txs.length)
+        expect(blocks[0]._isInvalid).toBeFalsy()
+    })
+})