fix: invalidate fork-choice head on FCU INVALID with latestValidHash

packages/beacon-node/src/chain/blocks/importBlock.ts

@@ -23,6 +23,7 @@ import {
 import {Attestation, BeaconBlock, altair, capella, electra, isGloasBeaconBlock, phase0, ssz} from "@lodestar/types";
 import {isErrorAborted, toRootHex} from "@lodestar/utils";
 import {ZERO_HASH_HEX} from "../../constants/index.js";
+import {isForkchoiceUpdateInvalidError} from "../../execution/engine/interface.js";
 import {callInNextEventLoop} from "../../util/eventLoop.js";
 import {isOptimisticBlock} from "../../util/forkChoice.js";
 import {isQueueErrorAborted} from "../../util/queue/index.js";
@@ -34,6 +35,7 @@ import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
 import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
 import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {getCheckpointFromState} from "./utils/checkpoint.js";
+import {invalidateForkchoiceHeadFromFcuInvalid} from "./utils/forkchoiceUpdateInvalid.js";
 
 /**
  * Fork-choice allows to import attestations from current (0) or past (1) epoch.
@@ -430,7 +432,8 @@ export async function importBlock(
      * - `headBlockHash !== null` -> Pre BELLATRIX_EPOCH
      * - `headBlockHash !== ZERO_HASH` -> Pre TTD
      */
-    const headBlockHash = this.forkChoice.getHead().executionPayloadBlockHash ?? ZERO_HASH_HEX;
+    const fcuHead = this.forkChoice.getHead();
+    const headBlockHash = fcuHead.executionPayloadBlockHash ?? ZERO_HASH_HEX;
     /**
      * After BELLATRIX_EPOCH and TTD it's okay to send a zero hash block hash for the finalized block. This will happen if
      * the current finalized block does not contain any execution payload at all (pre MERGE_EPOCH) or if it contains a
@@ -440,13 +443,12 @@ export async function importBlock(
     const finalizedBlockHash = this.forkChoice.getFinalizedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
     if (headBlockHash !== ZERO_HASH_HEX) {
       this.executionEngine
-        .notifyForkchoiceUpdate(
-          this.config.getForkName(this.forkChoice.getHead().slot),
-          headBlockHash,
-          safeBlockHash,
-          finalizedBlockHash
-        )
+        .notifyForkchoiceUpdate(this.config.getForkName(fcuHead.slot), headBlockHash, safeBlockHash, finalizedBlockHash)
         .catch((e) => {
+          if (isForkchoiceUpdateInvalidError(e)) {
+            invalidateForkchoiceHeadFromFcuInvalid(this, fcuHead.blockRoot, headBlockHash, e);
+            return;
+          }
           if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
             this.logger.error("Error pushing notifyForkchoiceUpdate()", {headBlockHash, finalizedBlockHash}, e);
           }

packages/beacon-node/src/chain/blocks/importExecutionPayload.ts

@@ -3,12 +3,13 @@ import {ExecutionStatus, PayloadExecutionStatus, getSafeExecutionBlockHash} from
 import {DataAvailabilityStatus, isStatePostGloas} from "@lodestar/state-transition";
 import {isErrorAborted} from "@lodestar/utils";
 import {ZERO_HASH_HEX} from "../../constants/index.js";
-import {ExecutionPayloadStatus} from "../../execution/index.js";
+import {ExecutionPayloadStatus, isForkchoiceUpdateInvalidError} from "../../execution/index.js";
 import {isQueueErrorAborted} from "../../util/queue/index.js";
 import {BeaconChain} from "../chain.js";
 import {RegenCaller} from "../regen/interface.js";
 import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
 import {ImportPayloadOpts} from "./types.js";
+import {invalidateForkchoiceHeadFromFcuInvalid} from "./utils/forkchoiceUpdateInvalid.js";
 import {
   verifyExecutionPayloadEnvelope,
   verifyExecutionPayloadEnvelopeSignature,
@@ -235,6 +236,10 @@ export async function importExecutionPayload(
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice);
     const finalizedBlockHash = this.forkChoice.getFinalizedBlock().executionPayloadBlockHash ?? ZERO_HASH_HEX;
     this.executionEngine.notifyForkchoiceUpdate(fork, blockHashHex, safeBlockHash, finalizedBlockHash).catch((e) => {
+      if (isForkchoiceUpdateInvalidError(e)) {
+        invalidateForkchoiceHeadFromFcuInvalid(this, head.blockRoot, blockHashHex, e);
+        return;
+      }
       if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
         this.logger.error("Error pushing notifyForkchoiceUpdate()", {blockHashHex, finalizedBlockHash}, e);
       }

packages/beacon-node/src/chain/blocks/utils/forkchoiceUpdateInvalid.ts

@@ -0,0 +1,71 @@
+import {ExecutionStatus} from "@lodestar/fork-choice";
+import {RootHex} from "@lodestar/types";
+import type {ForkchoiceUpdateError} from "../../../execution/engine/interface.js";
+import type {BeaconChain} from "../../chain.js";
+import {ForkchoiceCaller} from "../../forkChoice/index.js";
+
+/**
+ * Engine API spec: when EL responds INVALID to `engine_forkchoiceUpdated`, the CL must
+ * mark the offending head and its ancestors back to (but not including) `latestValidHash`
+ * as INVALID in fork choice and recompute the head. Without this, the CL's tree never
+ * abandons the bad branch — every slot it re-fires the same FCU, EL keeps responding
+ * INVALID, and the node stays wedged at the same head while the network advances.
+ *
+ * `validateLatestHash` walks up from `invalidateFromParentBlockRoot` (the invalid block
+ * itself, despite the name) marking ancestors invalid until reaching the LVH, then
+ * marks all descendants invalid in a second pass and recomputes scores. After that we
+ * trigger an explicit head recompute so subsequent attestations, proposals, and APIs
+ * see the corrected head immediately rather than waiting for the next `prepareNextSlot`
+ * or block import.
+ */
+export function invalidateForkchoiceHeadFromFcuInvalid(
+  chain: BeaconChain,
+  headBlockRoot: RootHex,
+  headBlockHash: RootHex,
+  e: ForkchoiceUpdateError
+): void {
+  const latestValidHash = e.type.latestValidHash;
+  chain.logger.warn(
+    "Invalidating head after FCU INVALID response",
+    {
+      headBlockRoot,
+      headBlockHash,
+      latestValidHash: latestValidHash ?? "null",
+      validationError: e.type.validationError ?? "",
+    },
+    e
+  );
+
+  try {
+    chain.forkChoice.validateLatestHash({
+      executionStatus: ExecutionStatus.Invalid,
+      latestValidExecHash: latestValidHash,
+      invalidateFromParentBlockRoot: headBlockRoot,
+      invalidateFromParentBlockHash: headBlockHash,
+    });
+  } catch (err) {
+    chain.logger.error(
+      "Failed to invalidate head after FCU INVALID response",
+      {headBlockRoot, headBlockHash, latestValidHash: latestValidHash ?? "null"},
+      err as Error
+    );
+    return;
+  }
+
+  try {
+    const newHead = chain.recomputeForkChoiceHead(ForkchoiceCaller.forkchoiceUpdateInvalid);
+    if (newHead.blockRoot !== headBlockRoot) {
+      chain.logger.info("Switched head after FCU INVALID invalidation", {
+        oldHeadBlockRoot: headBlockRoot,
+        newHeadBlockRoot: newHead.blockRoot,
+        newHeadSlot: newHead.slot,
+      });
+    }
+  } catch (err) {
+    chain.logger.error(
+      "Failed to recompute head after FCU INVALID invalidation",
+      {headBlockRoot, headBlockHash},
+      err as Error
+    );
+  }
+}

packages/beacon-node/src/chain/forkChoice/index.ts

@@ -32,6 +32,7 @@ export type ForkChoiceOpts = RawForkChoiceOpts & {
 export enum ForkchoiceCaller {
   prepareNextSlot = "prepare_next_slot",
   importBlock = "import_block",
+  forkchoiceUpdateInvalid = "forkchoice_update_invalid",
 }
 
 /**

packages/beacon-node/src/execution/engine/http.ts

@@ -14,6 +14,8 @@ import {
   ExecutePayloadResponse,
   ExecutionEngineState,
   ExecutionPayloadStatus,
+  ForkchoiceUpdateError,
+  ForkchoiceUpdateErrorCode,
   IExecutionEngine,
   PayloadAttributes,
   PayloadId,
@@ -370,7 +372,7 @@ export class ExecutionEngineHttp implements IExecutionEngine {
     }) as Promise<EngineApiRpcReturnTypes[typeof method]>;
 
     const {
-      payloadStatus: {status, latestValidHash: _latestValidHash, validationError},
+      payloadStatus: {status, latestValidHash, validationError},
       payloadId,
     } = await request;
 
@@ -398,7 +400,18 @@ export class ExecutionEngineHttp implements IExecutionEngine {
         return null;
 
       case ExecutionPayloadStatus.INVALID:
-        throw Error(
+        // Surface latestValidHash so the caller can invalidate the offending head and its
+        // ancestors back to (but not including) the LVH in the fork-choice tree, per Engine
+        // API spec. Without this, a CL whose fork-choice picks an invalid EL fork stays
+        // wedged: every slot re-fires the same FCU, EL keeps responding INVALID, and the
+        // tree never abandons the bad branch.
+        throw new ForkchoiceUpdateError(
+          {
+            code: ForkchoiceUpdateErrorCode.INVALID,
+            headBlockHash,
+            latestValidHash: latestValidHash ?? null,
+            validationError: validationError ?? null,
+          },
           `Invalid ${payloadAttributes ? "prepare payload" : "forkchoice request"}, validationError=${
             validationError ?? ""
           }`

packages/beacon-node/src/execution/engine/interface.ts

@@ -10,6 +10,7 @@ import {
 import {BlobsBundle, ExecutionPayload, ExecutionRequests, Root, RootHex, Wei, capella} from "@lodestar/types";
 import {BlobAndProof} from "@lodestar/types/deneb";
 import {BlobAndProofV2} from "@lodestar/types/fulu";
+import {LodestarError} from "@lodestar/utils";
 import {PayloadId, PayloadIdCache, WithdrawalV1} from "./payloadIdCache.js";
 import {ExecutionPayloadBody} from "./types.js";
 import {DATA} from "./utils.js";
@@ -79,6 +80,29 @@ export type ForkChoiceUpdateStatus =
   | ExecutionPayloadStatus.INVALID
   | ExecutionPayloadStatus.SYNCING;
 
+/**
+ * Thrown by `notifyForkchoiceUpdate` when the EL responds with INVALID. Carries the
+ * `latestValidHash` and the `headBlockHash` we asked it to set, so the caller can
+ * invalidate the offending head + ancestors back to (but not including) the LVH in
+ * the fork-choice tree per the Engine API spec.
+ */
+export enum ForkchoiceUpdateErrorCode {
+  INVALID = "FORKCHOICE_UPDATE_ERROR_INVALID",
+}
+
+export type ForkchoiceUpdateErrorType = {
+  code: ForkchoiceUpdateErrorCode.INVALID;
+  headBlockHash: RootHex;
+  latestValidHash: RootHex | null;
+  validationError: string | null;
+};
+
+export class ForkchoiceUpdateError extends LodestarError<ForkchoiceUpdateErrorType> {}
+
+export function isForkchoiceUpdateInvalidError(e: unknown): e is ForkchoiceUpdateError {
+  return e instanceof ForkchoiceUpdateError && e.type.code === ForkchoiceUpdateErrorCode.INVALID;
+}
+
 export type PayloadAttributes = {
   timestamp: number;
   prevRandao: Uint8Array;

packages/beacon-node/test/unit/chain/blocks/utils/forkchoiceUpdateInvalid.test.ts

@@ -0,0 +1,141 @@
+import {describe, expect, it, vi} from "vitest";
+import {ExecutionStatus} from "@lodestar/fork-choice";
+import {invalidateForkchoiceHeadFromFcuInvalid} from "../../../../../src/chain/blocks/utils/forkchoiceUpdateInvalid.js";
+import type {BeaconChain} from "../../../../../src/chain/chain.js";
+import {ForkchoiceCaller} from "../../../../../src/chain/forkChoice/index.js";
+import {ForkchoiceUpdateError, ForkchoiceUpdateErrorCode} from "../../../../../src/execution/engine/interface.js";
+
+describe("chain / blocks / utils / invalidateForkchoiceHeadFromFcuInvalid", () => {
+  const headBlockRoot = "0xbeac0a000000000000000000000000000000000000000000000000000000000a";
+  const headBlockHash = "0x19dc0a000000000000000000000000000000000000000000000000000000000a";
+  const lvh = "0x05a771b000000000000000000000000000000000000000000000000000000a05";
+  const newHeadBlockRoot = "0xbeac12e000000000000000000000000000000000000000000000000000000a12";
+
+  function makeChain(): {
+    chain: BeaconChain;
+    validateLatestHash: ReturnType<typeof vi.fn>;
+    recomputeForkChoiceHead: ReturnType<typeof vi.fn>;
+    logger: {warn: ReturnType<typeof vi.fn>; error: ReturnType<typeof vi.fn>; info: ReturnType<typeof vi.fn>};
+  } {
+    const validateLatestHash = vi.fn();
+    const recomputeForkChoiceHead = vi.fn().mockReturnValue({blockRoot: newHeadBlockRoot, slot: 5598});
+    const logger = {warn: vi.fn(), error: vi.fn(), info: vi.fn()};
+    const chain = {
+      forkChoice: {validateLatestHash},
+      logger,
+      recomputeForkChoiceHead,
+    } as unknown as BeaconChain;
+    return {chain, validateLatestHash, recomputeForkChoiceHead, logger};
+  }
+
+  it("calls forkChoice.validateLatestHash with the head as the invalid block and EL LVH", () => {
+    const {chain, validateLatestHash} = makeChain();
+    const e = new ForkchoiceUpdateError({
+      code: ForkchoiceUpdateErrorCode.INVALID,
+      headBlockHash,
+      latestValidHash: lvh,
+      validationError: "HeaderGasUsedMismatch",
+    });
+
+    invalidateForkchoiceHeadFromFcuInvalid(chain, headBlockRoot, headBlockHash, e);
+
+    expect(validateLatestHash).toHaveBeenCalledTimes(1);
+    expect(validateLatestHash).toHaveBeenCalledWith({
+      executionStatus: ExecutionStatus.Invalid,
+      latestValidExecHash: lvh,
+      invalidateFromParentBlockRoot: headBlockRoot,
+      invalidateFromParentBlockHash: headBlockHash,
+    });
+  });
+
+  it("recomputes head after invalidation so attestations and proposals see the corrected head", () => {
+    const {chain, recomputeForkChoiceHead, logger} = makeChain();
+    const e = new ForkchoiceUpdateError({
+      code: ForkchoiceUpdateErrorCode.INVALID,
+      headBlockHash,
+      latestValidHash: lvh,
+      validationError: null,
+    });
+
+    invalidateForkchoiceHeadFromFcuInvalid(chain, headBlockRoot, headBlockHash, e);
+
+    expect(recomputeForkChoiceHead).toHaveBeenCalledTimes(1);
+    expect(recomputeForkChoiceHead).toHaveBeenCalledWith(ForkchoiceCaller.forkchoiceUpdateInvalid);
+    // head changed → info log noting the switch
+    expect(logger.info).toHaveBeenCalledWith(
+      "Switched head after FCU INVALID invalidation",
+      expect.objectContaining({
+        oldHeadBlockRoot: headBlockRoot,
+        newHeadBlockRoot,
+      })
+    );
+  });
+
+  it("does not log a switch if head recompute returns the same root (no descendant alternative)", () => {
+    const {chain, recomputeForkChoiceHead, logger} = makeChain();
+    recomputeForkChoiceHead.mockReturnValueOnce({blockRoot: headBlockRoot, slot: 5597});
+    const e = new ForkchoiceUpdateError({
+      code: ForkchoiceUpdateErrorCode.INVALID,
+      headBlockHash,
+      latestValidHash: lvh,
+      validationError: null,
+    });
+
+    invalidateForkchoiceHeadFromFcuInvalid(chain, headBlockRoot, headBlockHash, e);
+
+    expect(recomputeForkChoiceHead).toHaveBeenCalledTimes(1);
+    expect(logger.info).not.toHaveBeenCalled();
+  });
+
+  it("forwards null LVH unchanged so protoArray can apply its 'unknown LVH' policy", () => {
+    const {chain, validateLatestHash} = makeChain();
+    const e = new ForkchoiceUpdateError({
+      code: ForkchoiceUpdateErrorCode.INVALID,
+      headBlockHash,
+      latestValidHash: null,
+      validationError: null,
+    });
+
+    invalidateForkchoiceHeadFromFcuInvalid(chain, headBlockRoot, headBlockHash, e);
+
+    expect(validateLatestHash).toHaveBeenCalledWith(
+      expect.objectContaining({
+        executionStatus: ExecutionStatus.Invalid,
+        latestValidExecHash: null,
+        invalidateFromParentBlockRoot: headBlockRoot,
+        invalidateFromParentBlockHash: headBlockHash,
+      })
+    );
+  });
+
+  it("swallows validateLatestHash failures and skips head recompute so the FCU catch handler does not crash the import path", () => {
+    const {chain, validateLatestHash, recomputeForkChoiceHead} = makeChain();
+    validateLatestHash.mockImplementationOnce(() => {
+      throw new Error("invalidateFromParentBlockRoot not in forkchoice");
+    });
+    const e = new ForkchoiceUpdateError({
+      code: ForkchoiceUpdateErrorCode.INVALID,
+      headBlockHash,
+      latestValidHash: lvh,
+      validationError: null,
+    });
+
+    expect(() => invalidateForkchoiceHeadFromFcuInvalid(chain, headBlockRoot, headBlockHash, e)).not.toThrow();
+    expect(recomputeForkChoiceHead).not.toHaveBeenCalled();
+  });
+
+  it("swallows recomputeForkChoiceHead failures so the FCU catch handler does not crash the import path", () => {
+    const {chain, recomputeForkChoiceHead} = makeChain();
+    recomputeForkChoiceHead.mockImplementationOnce(() => {
+      throw new Error("findHead failed");
+    });
+    const e = new ForkchoiceUpdateError({
+      code: ForkchoiceUpdateErrorCode.INVALID,
+      headBlockHash,
+      latestValidHash: lvh,
+      validationError: null,
+    });
+
+    expect(() => invalidateForkchoiceHeadFromFcuInvalid(chain, headBlockRoot, headBlockHash, e)).not.toThrow();
+  });
+});