Ensure valid ENSRainbow connection before indexing starts

.changeset/sharp-moons-shave.md

@@ -0,0 +1,5 @@
+---
+"ensindexer": minor
+---
+
+Introduced indexing event handler preconditions to optimize the cross-service availability in an ENSNode instance when ENSRainbow is performing a cold-start.

.changeset/whole-lines-smoke.md

@@ -0,0 +1,5 @@
+---
+"@ensnode/ensdb-sdk": minor
+---
+
+Extended the ENSNode Metadata with ENSRainbow Public Config.

apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.test.ts

@@ -261,21 +261,28 @@ describe("EnsDbWriterWorker", () => {
   });
 
   describe("interval behavior - snapshot upserts", () => {
-    it("continues upserting after snapshot validation errors", async () => {
+    it("upserts snapshots across different omnichain statuses", async () => {
       // arrange
       const unstartedSnapshot = createMockOmnichainSnapshot({
         omnichainStatus: OmnichainIndexingStatusIds.Unstarted,
       });
       const validSnapshot = createMockOmnichainSnapshot({
         omnichainIndexingCursor: 200,
       });
-      const crossChainSnapshot = createMockCrossChainSnapshot({
+      const unstartedCrossChainSnapshot = createMockCrossChainSnapshot({
+        slowestChainIndexingCursor: 100,
+        snapshotTime: 200,
+        omnichainSnapshot: unstartedSnapshot,
+      });
+      const validCrossChainSnapshot = createMockCrossChainSnapshot({
         slowestChainIndexingCursor: 200,
         snapshotTime: 300,
         omnichainSnapshot: validSnapshot,
       });
 
-      vi.mocked(buildCrossChainIndexingStatusSnapshotOmnichain).mockReturnValue(crossChainSnapshot);
+      vi.mocked(buildCrossChainIndexingStatusSnapshotOmnichain)
+        .mockReturnValueOnce(unstartedCrossChainSnapshot)
+        .mockReturnValueOnce(validCrossChainSnapshot);
 
       const ensDbClient = createMockEnsDbWriter();
       const indexingStatusBuilder = {
@@ -297,8 +304,15 @@ describe("EnsDbWriterWorker", () => {
 
       // assert
       expect(indexingStatusBuilder.getOmnichainIndexingStatusSnapshot).toHaveBeenCalledTimes(2);
-      expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledTimes(1);
-      expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledWith(crossChainSnapshot);
+      expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenCalledTimes(2);
+      expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenNthCalledWith(
+        1,
+        unstartedCrossChainSnapshot,
+      );
+      expect(ensDbClient.upsertIndexingStatusSnapshot).toHaveBeenNthCalledWith(
+        2,
+        validCrossChainSnapshot,
+      );
 
       // cleanup
       worker.stop();

apps/ensindexer/src/lib/ensdb-writer-worker/ensdb-writer-worker.ts

@@ -7,8 +7,6 @@ import {
   type CrossChainIndexingStatusSnapshot,
   type Duration,
   type EnsIndexerPublicConfig,
-  OmnichainIndexingStatusIds,
-  type OmnichainIndexingStatusSnapshot,
   validateEnsIndexerPublicConfigCompatibility,
 } from "@ensnode/ensnode-sdk";
 import type { LocalPonderClient } from "@ensnode/ponder-sdk";
@@ -231,7 +229,8 @@ export class EnsDbWriterWorker {
       // get system timestamp for the current iteration
       const snapshotTime = getUnixTime(new Date());
 
-      const omnichainSnapshot = await this.getValidatedIndexingStatusSnapshot();
+      const omnichainSnapshot =
+        await this.indexingStatusBuilder.getOmnichainIndexingStatusSnapshot();
 
       const crossChainSnapshot = buildCrossChainIndexingStatusSnapshotOmnichain(
         omnichainSnapshot,
@@ -248,24 +247,4 @@ export class EnsDbWriterWorker {
       // should not cause the ENSDb Writer Worker to stop functioning.
     }
   }
-
-  /**
-   * Get validated Omnichain Indexing Status Snapshot
-   *
-   * @returns Validated Omnichain Indexing Status Snapshot.
-   * @throws Error if the Omnichain Indexing Status is not in expected status yet.
-   */
-  private async getValidatedIndexingStatusSnapshot(): Promise<OmnichainIndexingStatusSnapshot> {
-    const omnichainSnapshot = await this.indexingStatusBuilder.getOmnichainIndexingStatusSnapshot();
-
-    // It only makes sense to write Indexing Status Snapshots into ENSDb once
-    // the indexing process has started, as before that there is no meaningful
-    // status to record.
-    // Invariant: the Omnichain Status must indicate that indexing has started already.
-    if (omnichainSnapshot.omnichainStatus === OmnichainIndexingStatusIds.Unstarted) {
-      throw new Error("Omnichain Status must not be 'Unstarted'.");
-    }
-
-    return omnichainSnapshot;
-  }
 }

apps/ensindexer/src/lib/ensrainbow/singleton.ts

@@ -0,0 +1,73 @@
+import config from "@/config";
+
+import pRetry from "p-retry";
+
+import { EnsRainbowApiClient } from "@ensnode/ensrainbow-sdk";
+
+const { ensRainbowUrl, labelSet } = config;
+
+if (ensRainbowUrl.href === EnsRainbowApiClient.defaultOptions().endpointUrl.href) {
+  console.warn(
+    `Using default public ENSRainbow server which may cause increased network latency. For production, use your own ENSRainbow server that runs on the same network as the ENSIndexer server.`,
+  );
+}
+
+/**
+ * Singleton ENSRainbow Client instance for ENSIndexer app.
+ */
+export const ensRainbowClient = new EnsRainbowApiClient({
+  endpointUrl: ensRainbowUrl,
+  labelSet,
+});
+
+/**
+ * Cached promise for waiting for ENSRainbow to be ready.
+ *
+ * This ensures that multiple concurrent calls to
+ * {@link waitForEnsRainbowToBeReady} will share the same underlying promise
+ * in order to use the same retry sequence.
+ */
+let waitForEnsRainbowToBeReadyPromise: Promise<void> | undefined;
+
+/**
+ * Wait for ENSRainbow to be ready
+ *
+ * Blocks execution until the ENSRainbow instance is ready to serve requests.
+ *
+ * Note: It may take 30+ minutes for the ENSRainbow instance to become ready in
+ * a cold start scenario. We use retries for the ENSRainbow health check with
+ * an exponential backoff strategy to handle this.
+ *
+ * @throws When ENSRainbow fails to become ready after all configured retry attempts.
+ *         This error will trigger termination of the ENSIndexer process.
+ */
+export function waitForEnsRainbowToBeReady(): Promise<void> {
+  if (waitForEnsRainbowToBeReadyPromise) {
+    return waitForEnsRainbowToBeReadyPromise;
+  }
+
+  console.log(`Waiting for ENSRainbow instance to be ready at '${ensRainbowUrl}'...`);
+
+  waitForEnsRainbowToBeReadyPromise = pRetry(async () => ensRainbowClient.health(), {
+    retries: 12, // This allows for a total of over 1 hour of retries with the exponential backoff strategy.
+    // 1 + 2 + 4 + ... + 2048 = 2^12 - 1 = 4,095s ≈ 1h 8m
+    onFailedAttempt: ({ error, attemptNumber, retriesLeft }) => {
+      console.warn(
+        `Attempt ${attemptNumber} failed for the ENSRainbow health check at '${ensRainbowUrl}' (${error.message}). ${retriesLeft} retries left.`,
+      );
+    },
+  })
+    .then(() => console.log(`ENSRainbow instance is ready at '${ensRainbowUrl}'.`))
+    .catch((error) => {
+      const errorMessage = error instanceof Error ? error.message : "Unknown error";
+
+      console.error(`ENSRainbow health check failed after multiple attempts: ${errorMessage}`);
+
+      // Throw the error to terminate the ENSIndexer process due to the failed health check of a critical dependency
+      throw new Error(errorMessage, {
+        cause: error instanceof Error ? error : undefined,
+      });
+    });
+
+  return waitForEnsRainbowToBeReadyPromise;
+}

apps/ensindexer/src/lib/graphnode-helpers.ts

@@ -3,9 +3,7 @@ import pRetry from "p-retry";
 import type { LabelHash, LiteralLabel } from "@ensnode/ensnode-sdk";
 import { type EnsRainbow, ErrorCode, isHealError } from "@ensnode/ensrainbow-sdk";
 
-import { getENSRainbowApiClient } from "@/lib/ensraibow-api-client";
-
-const ensRainbowApiClient = getENSRainbowApiClient();
+import { ensRainbowClient } from "@/lib/ensrainbow/singleton";
 
 /**
  * Attempt to heal a labelHash to its original label.
@@ -44,13 +42,13 @@ export async function labelByLabelHash(labelHash: LabelHash): Promise<LiteralLab
   // "last failure was HealServerError" (set) from "last failure was a network throw" (undefined).
   let lastServerError: EnsRainbow.HealServerError | undefined;
 
-  let response: Awaited<ReturnType<typeof ensRainbowApiClient.heal>>;
+  let response: EnsRainbow.HealResponse;
 
   try {
     response = await pRetry(
       async () => {
         lastServerError = undefined;
-        const result = await ensRainbowApiClient.heal(labelHash);
+        const result = await ensRainbowClient.heal(labelHash);
 
         if (isHealError(result) && result.errorCode === ErrorCode.ServerError) {
           lastServerError = result;
@@ -81,7 +79,7 @@ export async function labelByLabelHash(labelHash: LabelHash): Promise<LiteralLab
 
     // Not recoverable; causes the ENSIndexer process to terminate.
     if (error instanceof Error) {
-      error.message = `ENSRainbow Heal Request Failed: ENSRainbow unavailable at '${ensRainbowApiClient.getOptions().endpointUrl}'.`;
+      error.message = `ENSRainbow Heal Request Failed: ENSRainbow unavailable at '${ensRainbowClient.getOptions().endpointUrl}'.`;
     }
 
     throw error;