diff --git a/common/changes/@subsquid/evm-rpc/alert-fix-L8b2z3-base-sepolia-statediff-pertx_2026-06-24-05-28.json b/common/changes/@subsquid/evm-rpc/alert-fix-L8b2z3-base-sepolia-statediff-pertx_2026-06-24-05-28.json new file mode 100644 index 000000000..89640b827 --- /dev/null +++ b/common/changes/@subsquid/evm-rpc/alert-fix-L8b2z3-base-sepolia-statediff-pertx_2026-06-24-05-28.json @@ -0,0 +1 @@ +{"changes":[{"packageName":"@subsquid/evm-rpc","comment":"fall back to per-transaction debug_traceTransaction when a whole-block state diff exceeds the provider response size limit","type":"patch"}],"packageName":"@subsquid/evm-rpc"} diff --git a/evm/evm-rpc/src/rpc.ts b/evm/evm-rpc/src/rpc.ts index 14b3d236a..34c21dcec 100644 --- a/evm/evm-rpc/src/rpc.ts +++ b/evm/evm-rpc/src/rpc.ts @@ -16,6 +16,7 @@ import { GetBlock, Receipt, TraceFrame, + DebugStateDiff, DebugStateDiffResult, DebugFrameResult, TraceReplayTraces, @@ -1018,6 +1019,7 @@ export class Rpc { validateError: info => { if (info.message.includes('not found')) return null if (info.message.includes('cannot query unfinalized data')) return null // Avalanche + if (isResponseTooBigError(info)) return RESPONSE_TOO_BIG as any throw new RpcError(info) } }) @@ -1026,6 +1028,13 @@ export class Rpc { for (let i = 0; i < blocks.length; i++) { let block = blocks[i] let diffs = results[i] + if ((diffs as any) === RESPONSE_TOO_BIG) { + // The whole-block prestateTracer response exceeded the provider's + // response-size cap. Retrying/splitting the block batch can't help + // (a single block can't be split), so re-fetch the state diff one + // transaction at a time, where each response is small. + diffs = await this.traceStateDiffsPerTransaction(block, traceConfig) + } if (diffs == null) { block._isInvalid = true block._errorMessage = "failed to get debug state diffs for a block" @@ -1037,6 +1046,39 @@ export class Rpc { } } + private async traceStateDiffsPerTransaction( + block: Block, + traceConfig: object + ): Promise { + let txs = block.block.transactions + this.log.warn( + `debug state diff for block ${block.number} exceeded the provider response size limit; ` + + `falling back to per-transaction tracing of ${txs.length} transactions` + ) + + // debug_traceTransaction returns a bare state diff per call (no { result, txHash } + // envelope), so we validate the diff itself and reattach the tx hash below. + let diffs = await this.reduceBatchOnRetry(txs.map(tx => ({ + method: 'debug_traceTransaction', + params: [getTxHash(tx), traceConfig] + })), { + validateResult: getResultValidator(DebugStateDiff), + validateError: info => { + if (info.message.includes('not found')) return null + if (info.message.includes('cannot query unfinalized data')) return null // Avalanche + throw new RpcError(info) + } + }) + + let out: DebugStateDiffResult[] = new Array(txs.length) + for (let i = 0; i < txs.length; i++) { + let diff = diffs[i] + if (diff == null) return null // a transaction couldn't be traced; flag block for retry + out[i] = { result: diff, txHash: getTxHash(txs[i]) } + } + return out + } + private async addDebugFrames(blocks: Block[], req: DataRequest): Promise { let traceConfig = { tracer: 'callTracer', @@ -1260,6 +1302,27 @@ function isEmpty(obj: object): boolean { } +// Marks a debug-trace batch item whose whole-block response exceeded the +// provider's response-size cap and must be re-fetched per transaction. +const RESPONSE_TOO_BIG = Symbol('evm-rpc:response-too-big') + + +// geth/erigon and managed providers (alchemy, dwellir, uniblock, …) cap a single +// JSON-RPC response (commonly 160 MiB) and reject an oversized debug trace with +// this error. Unlike the *transient* "-32020 response too large" handled in +// EvmRpcClient.isResponseTooLargeError, a whole-block prestateTracer diff for a +// very large block exceeds the cap *persistently* — every retry returns the same +// error and the block batch can't be split below a single block — so the only way +// forward is to trace the block one transaction at a time. +function isResponseTooBigError(info: { code?: number, message: string }): boolean { + return ( + info.code === -32008 || + /response is too big/i.test(info.message) || + /exceeded max limit/i.test(info.message) + ) +} + + /** * Recursively collects `logs` arrays from a callTracer frame and all its * nested sub-call frames. Only `address` and `topics` are needed by diff --git a/evm/evm-rpc/test/helpers/mock-rpc-client.ts b/evm/evm-rpc/test/helpers/mock-rpc-client.ts index a29824694..9f6739537 100644 --- a/evm/evm-rpc/test/helpers/mock-rpc-client.ts +++ b/evm/evm-rpc/test/helpers/mock-rpc-client.ts @@ -1,3 +1,10 @@ +// A JSON-RPC error returned by an upstream as a fixture, so that +// validateError() is exercised the same way the real client exercises it. +class MockRpcErrorFixture { + constructor(public info: { code?: number, message: string, data?: any }) {} +} + + export class MockRpcClient { private fixtures: Map @@ -13,32 +20,55 @@ export class MockRpcClient { return false } - async call(method: string, params?: any[]): Promise { + async call(method: string, params?: any[], options?: {validateError?: (info: any, call?: any) => any}): Promise { + const fixture = this.lookup(method, params) + if (fixture instanceof MockRpcErrorFixture) { + if (options?.validateError) { + return options.validateError(fixture.info, {method, params}) + } + const err: any = new Error(fixture.info.message) + err.code = fixture.info.code + throw err + } + return fixture + } + + private lookup(method: string, params?: any[]): any { const key = this.makeKey(method, params) if (this.fixtures.has(key)) { return this.fixtures.get(key) } - // Try without params if (this.fixtures.has(method)) { return this.fixtures.get(method) } - throw new Error(`No fixture found for method: ${method} with params: ${JSON.stringify(params)}`) } - async batchCall(batch: {method: string, params?: any[]}[], options?: {validateResult?: (result: any) => any}): Promise { + async batchCall(batch: {method: string, params?: any[]}[], options?: {validateResult?: (result: any) => any, validateError?: (info: any, call?: any) => any}): Promise { const results = [] for (const req of batch) { + let fixture: any try { - let result = await this.call(req.method, req.params) - if (options?.validateResult) { - result = options.validateResult(result) - } - results.push(result) + fixture = this.lookup(req.method, req.params) } catch (error: any) { results.push({error: error.message}) + continue + } + if (fixture instanceof MockRpcErrorFixture) { + // Mirror the real client: an error item is passed to validateError, + // whose return value becomes the item's result. If validateError throws + // (a non-recoverable error), the whole batch rejects. + if (options?.validateError) { + results.push(options.validateError(fixture.info, req)) + } else { + const err: any = new Error(fixture.info.message) + err.code = fixture.info.code + throw err + } + continue } + results.push(options?.validateResult ? options.validateResult(fixture) : fixture) } return results } @@ -48,6 +78,11 @@ export class MockRpcClient { this.fixtures.set(key, response) } + setErrorFixture(method: string, params: any[] | undefined, info: { code?: number, message: string, data?: any }): void { + const key = this.makeKey(method, params) + this.fixtures.set(key, new MockRpcErrorFixture(info)) + } + private makeKey(method: string, params?: any[]): string { if (!params || params.length === 0) { return method diff --git a/evm/evm-rpc/test/state-diff-too-big.test.ts b/evm/evm-rpc/test/state-diff-too-big.test.ts new file mode 100644 index 000000000..4c4336472 --- /dev/null +++ b/evm/evm-rpc/test/state-diff-too-big.test.ts @@ -0,0 +1,101 @@ +import { describe, it, expect } from 'vitest' +import { loadBlock } from './helpers/fixture-loader' +import { MockRpcClient } from './helpers/mock-rpc-client' +import { Rpc } from '../src/rpc' +import { toQty, getTxHash } from '../src/util' + +// Regression for the base-sepolia "No Dumper Data" stall: a very large block's +// whole-block prestateTracer state-diff response exceeds the provider's response +// size cap (geth/erigon/alchemy/dwellir all return JSON-RPC error -32008 +// "Response is too big" / "Exceeded max limit of 167772160"). Splitting/retrying +// the block batch can't help — a single block can't be split — so the dumper used +// to stall/crash-loop forever on that block. The fix falls back to per-transaction +// debug_traceTransaction, where each response is small. + +const BLOCK = 18000000 + +// Must match the traceConfig that Rpc.addDebugStateDiffs builds verbatim, since the +// mock keys fixtures by JSON.stringify(params). +const TRACE_CONFIG = { + tracer: 'prestateTracer', + tracerConfig: { onlyTopCall: false, diffMode: true }, + timeout: '20s' +} + +const REQUEST = { + transactions: true, + stateDiffs: true, + useDebugApiForStateDiffs: true, + useDebugTraceBlockByNumber: true, + debugTraceTimeout: '20s' +} + +function diffFor(i: number) { + // a small, valid per-tx state diff; storage key varies per tx so we can assert wiring + let slot = '0x' + i.toString(16).padStart(64, '0') + return { + pre: { '0x4200000000000000000000000000000000000015': { storage: { [slot]: '0x00' } } }, + post: { '0x4200000000000000000000000000000000000015': { storage: { [slot]: '0x01' } } } + } +} + +describe('debug state diff response-too-big handling', () => { + it('falls back to per-transaction tracing when the whole-block state diff exceeds the size cap', async () => { + const block = loadBlock('ethereum', BLOCK) + const txs = block.transactions as any[] + expect(txs.length).toBeGreaterThan(1) + + const mockClient = new MockRpcClient() + mockClient.setFixture('eth_chainId', undefined, '0x1') + mockClient.setFixture('eth_getBlockByNumber', [toQty(BLOCK), true], block) + + // whole-block call rejects with the provider's size-cap error + mockClient.setErrorFixture('debug_traceBlockByNumber', [block.number, TRACE_CONFIG], { + code: -32008, + message: 'Response is too big', + data: 'Exceeded max limit of 167772160' + }) + // per-transaction calls succeed with small responses + txs.forEach((tx, i) => { + mockClient.setFixture('debug_traceTransaction', [getTxHash(tx), TRACE_CONFIG], diffFor(i)) + }) + + const rpc = new Rpc({ client: mockClient as any }) + + const blocks = await rpc.getBlockBatch([BLOCK], REQUEST) + expect(blocks).toHaveLength(1) + + const diffs = blocks[0].debugStateDiffs! + expect(diffs).toBeTruthy() + expect(diffs.length).toEqual(txs.length) + // every per-tx diff is reattached to its transaction in order + for (let i = 0; i < txs.length; i++) { + expect(diffs[i]!.txHash).toEqual(getTxHash(txs[i])) + expect(diffs[i]!.result).toEqual(diffFor(i)) + } + expect(blocks[0]._isInvalid).toBeFalsy() + }) + + it('uses the whole-block response directly when it fits (no per-tx fallback)', async () => { + const block = loadBlock('ethereum', BLOCK) + const txs = block.transactions as any[] + + const mockClient = new MockRpcClient() + mockClient.setFixture('eth_chainId', undefined, '0x1') + mockClient.setFixture('eth_getBlockByNumber', [toQty(BLOCK), true], block) + mockClient.setFixture( + 'debug_traceBlockByNumber', + [block.number, TRACE_CONFIG], + txs.map((tx, i) => ({ result: diffFor(i), txHash: getTxHash(tx) })) + ) + // deliberately NO debug_traceTransaction fixtures — if the fallback fired, + // the missing-fixture error would surface. + + const rpc = new Rpc({ client: mockClient as any }) + + const blocks = await rpc.getBlockBatch([BLOCK], REQUEST) + expect(blocks).toHaveLength(1) + expect(blocks[0].debugStateDiffs!.length).toEqual(txs.length) + expect(blocks[0]._isInvalid).toBeFalsy() + }) +})