local env usability fixes

.github/workflows/fork-network.yml

@@ -15,6 +15,15 @@ name: fork-network
 #             federated-authority governance flow
 #   full    - production-shaped rehearsal: image rollout first, then the
 #             runtime upgrade via governance
+#
+# node_image / new_node_image take full Docker image references — anything
+# `docker pull` accepts. Git branches, commit shas, and repo URLs are NOT
+# accepted. Examples:
+#   ghcr.io/midnight-ntwrk/midnight-node:1.0.0                            (release tag)
+#   ghcr.io/midnight-ntwrk/midnight-node:1.0.1-dev-6587676a9bb2-amd64     (CI build of a commit)
+# CI tags a per-commit image <version>-dev-<12-char-tree-hash>-amd64; find it
+# in the commit's continuous-integration run (image_tags step) or compute the
+# hash with `git rev-parse HEAD^{tree} | cut -c1-12`.
 
 on:
   workflow_dispatch:
@@ -32,10 +41,10 @@ on:
         required: true
         default: "368fd98"
       node_image:
-        description: "midnight-node image to fork from (must match the snapshot's runtime host fns)"
+        description: "Full Docker image reference to fork from, e.g. ghcr.io/midnight-ntwrk/midnight-node:1.0.0 or a CI tag like ghcr.io/midnight-ntwrk/midnight-node:1.0.1-dev-<tree-hash>-amd64 — not a git branch/sha/URL. Must match the snapshot's runtime host fns"
         required: true
       new_node_image:
-        description: "midnight-node image to upgrade to (image rollout target and/or runtime wasm source, per upgrade_mode)"
+        description: "Full Docker image reference to upgrade to (image rollout target and/or runtime wasm source, per upgrade_mode). Same format as node_image"
         required: true
       upgrade_mode:
         description: "Upgrade shape: roll the node image, submit a runtime upgrade via governance, or both in sequence"

changes/node/added/local-env-from-genesis.md

@@ -0,0 +1,17 @@
+# Restore from-genesis bring-up for well-known networks in local-env tooling
+
+`npm run run:<network> -- --from-genesis` brings up a well-known network's base
+compose from block 0 again, instead of requiring `--from-snapshot` (the genesis
+path was removed together with the internal k8s/AWS integration in #1470).
+Nothing is mocked in this mode: validator seed phrases and a main-chain data
+source must be supplied via `--env-file` or the environment, and the CLI warns
+up front about compose variables left unset and about pre-existing chain data
+directories.
+
+Also clarifies the fork-network workflow's `node_image`/`new_node_image` inputs
+(full Docker image references with the expected release and CI tag formats, not
+git branches/shas/URLs) and documents how to find snapshot archives for each
+network from the backup system's public index in `docs/fork-testing.md`.
+
+PR:
+Issue:

docs/fork-testing.md

@@ -23,6 +23,37 @@ Local restore requires:
 - `tar`
 - `zstd` for `.zst` archives
 
+## Finding snapshot archives
+
+The backup system publishes snapshots of the well-known networks behind a
+CloudFront distribution. The manifest at
+`https://dg39snjayoq3t.cloudfront.net/index.json` lists every available
+archive, keyed by network name. To see which networks currently have
+snapshots:
+
+```bash
+curl -fsSL https://dg39snjayoq3t.cloudfront.net/index.json | jq 'keys'
+```
+
+As of this writing the index contains `devnet`, `qanet`, and `govnet`.
+`preview`, `preprod`, and `mainnet` snapshots are not published there yet;
+ask the node team if you need one.
+
+To resolve the latest archive URL for a network (this mirrors what the
+`fork-network` workflow's "Resolve latest snapshot URL" step does):
+
+```bash
+NETWORK=devnet
+curl -fsSL https://dg39snjayoq3t.cloudfront.net/index.json |
+  jq -r --arg net "$NETWORK" '
+    .[$net] | to_entries | map(.value[]) | sort_by(.timestamp) | last
+    | .s3_path | sub("^s3://[^/]+/"; "https://dg39snjayoq3t.cloudfront.net/")'
+```
+
+Pass the resulting URL to `--from-snapshot`. Drop the `last` selection to list
+every archive for the network (older snapshots stay useful for forking a
+specific block height or runtime version).
+
 ## Initial restore
 
 On the first bring-up of a well-known network, pass the snapshot URL to `run`,

local-environment/README.md

@@ -35,6 +35,10 @@ npm run run:preprod -- --from-snapshot https://example.com/snapshots/preprod-lat
 npm run run:mainnet -- --from-snapshot https://example.com/snapshots/mainnet-latest.tar.gz
 ```
 
+Real snapshot URLs for each network can be resolved from the backup system's
+public index — see
+[Finding snapshot archives](../docs/fork-testing.md#finding-snapshot-archives).
+
 After that initial restore, the same network can be restarted without
 `--from-snapshot` as long as the restored `data/` directories and generated
 mock-authorities output are still present:
@@ -48,6 +52,23 @@ image was built with the same `networkId` as the genesis used to produce the
 snapshot. Recent runtimes validate this at boot and the node will refuse to
 start on a mismatch.
 
+### Starting a well-known network from genesis
+
+`--from-genesis` skips the snapshot/fork flow entirely and brings up the
+network's base compose from block 0:
+
+```bash
+npm run run:devnet -- --from-genesis --env-file ../devnet.env
+```
+
+Unlike fork mode, nothing is mocked: each validator needs its real seed phrase
+(e.g. `MIDNIGHT_NODE_01_0_SEED`) and a reachable main-chain data source (the
+`DB_SYNC_POSTGRES_CONNECTION_STRING_NODE_*` vars) supplied via `--env-file` or
+the process environment. The CLI warns about any compose variables left unset,
+and about existing `data/` directories (nodes resume from existing chain data;
+wipe the network's `data/` directories first for a clean block-0 start).
+Restarting a genesis environment uses the same flag.
+
 ### Upgrade rehearsals
 
 `image-upgrade` rolls service containers from `NODE_IMAGE` to `NEW_NODE_IMAGE`.

local-environment/src/commands/run.ts

@@ -42,7 +42,8 @@ import {
  * Bring up a network locally:
  * - "local-env" runs the bundled local Cardano/PC stack from compose.
  * - Any well-known network (devnet/qanet/...) is forked from the
- *   provided snapshot via mock-authorities — there is no k8s-backed path.
+ *   provided snapshot via mock-authorities, or brought up from genesis
+ *   with --from-genesis — there is no k8s-backed path.
  */
 export async function run(network: string, runOptions: RunOptions) {
   if (network === "local-env") {
@@ -52,15 +53,20 @@ export async function run(network: string, runOptions: RunOptions) {
   }
 
   assertWellKnownNamespace(network);
-  console.log(
-    `Preparing ${network} local fork (mock-authorities-driven bring-up)`,
-  );
+  if (runOptions.fromGenesis) {
+    console.log(`Bringing up ${network} from genesis (base compose, no fork)`);
+  } else {
+    console.log(
+      `Preparing ${network} local fork (mock-authorities-driven bring-up)`,
+    );
+  }
   await runWellKnownNetwork(network, runOptions);
 }
 
 async function runWellKnownNetwork(namespace: string, runOptions: RunOptions) {
-  const networkConfig = loadNetworkConfig(namespace);
-  const mock = requireMockConfig(namespace, networkConfig);
+  if (runOptions.fromGenesis && runOptions.fromSnapshot) {
+    throw new Error("--from-genesis and --from-snapshot are mutually exclusive.");
+  }
 
   const composeFile = resolveComposeFile(namespace);
   const composeDir = path.dirname(composeFile);
@@ -75,6 +81,14 @@ async function runWellKnownNetwork(namespace: string, runOptions: RunOptions) {
     }
   }
 
+  if (runOptions.fromGenesis) {
+    await runFromGenesis(composeFile, env, runOptions);
+    return;
+  }
+
+  const networkConfig = loadNetworkConfig(namespace);
+  const mock = requireMockConfig(namespace, networkConfig);
+
   let overridePath: string;
   if (runOptions.fromSnapshot) {
     const restoredDirs = await restoreSnapshot({
@@ -134,6 +148,55 @@ async function runWellKnownNetwork(namespace: string, runOptions: RunOptions) {
   });
 }
 
+/**
+ * Bring up the network's base compose from block 0 — no snapshot restore, no
+ * mock-authorities rewrite. The base compose expects the real network inputs
+ * (validator seed phrases, a main-chain data source) via env/--env-file, so
+ * report anything left unset up front rather than letting the chain come up
+ * silently unable to produce blocks.
+ */
+async function runFromGenesis(
+  composeFile: string,
+  env: Record<string, string>,
+  runOptions: RunOptions,
+) {
+  const staleDataDirs = discoverComposeDataMounts(composeFile).filter((dir) =>
+    isNonEmptyDirectory(dir),
+  );
+  if (staleDataDirs.length > 0) {
+    console.warn(
+      `⚠️  Existing chain data found (${staleDataDirs.join(", ")}); nodes will resume from it rather than genesis. Delete these directories first for a clean block-0 start.`,
+    );
+  }
+
+  const unsetVars = collectUnsetComposeVars(composeFile, env);
+  if (unsetVars.length > 0) {
+    console.warn(
+      `⚠️  Env vars referenced by the compose file but not set: ${unsetVars.join(", ")}. From-genesis mode mocks nothing — validator seeds and a main-chain data source must be provided via --env-file or the environment for the chain to produce blocks.`,
+    );
+  }
+
+  await runDockerCompose({
+    composeFile,
+    env,
+    profiles: runOptions.profiles,
+    detach: true,
+  });
+}
+
+/** Env var names referenced by the compose file ($VAR or ${VAR...}) that are unset or blank in env. */
+function collectUnsetComposeVars(
+  composeFile: string,
+  env: Record<string, string>,
+): string[] {
+  const text = fs.readFileSync(composeFile, "utf8");
+  const names = new Set<string>();
+  for (const match of text.matchAll(/\$\{?([A-Z][A-Z0-9_]*)/g)) {
+    names.add(match[1]);
+  }
+  return [...names].filter((name) => !env[name]).sort();
+}
+
 function assertReusableForkState(
   namespace: string,
   composeFile: string,

local-environment/src/index.ts

@@ -84,8 +84,12 @@ program
     "--from-snapshot <uri>",
     "http(s):// snapshot URI to restore before the first well-known-network bring-up. Later runs can omit it to reuse existing local fork state.",
   )
+  .option(
+    "--from-genesis",
+    "Bring up the well-known network's base compose from block 0 instead of forking a snapshot. Requires the network's validator seeds and main-chain data-source env vars (see README).",
+  )
   .description(
-    "Bring up a forked well-known network from a snapshot using mock-authorities, reuse an existing local fork, or run the local-env target.",
+    "Bring up a forked well-known network from a snapshot using mock-authorities, reuse an existing local fork, start a well-known network from genesis, or run the local-env target.",
   )
   .action(async (network: string, options: RunOptions) => {
     await run(network, options);

local-environment/src/lib/types.ts

@@ -22,6 +22,13 @@ export interface RunOptions {
    * output.
    */
   fromSnapshot?: string;
+  /**
+   * Bring the well-known network's base compose up from block 0 instead of
+   * forking a snapshot. Nothing is mocked in this mode: validator seed
+   * phrases and a main-chain data source must be supplied via env/--env-file.
+   * Mutually exclusive with fromSnapshot.
+   */
+  fromGenesis?: boolean;
 }
 
 export interface ImageUpgradeOptions extends RunOptions {