start API server after chain init, add health/ready endpoints

pkgs/api/README.md

@@ -2,15 +2,19 @@
 
 ## Overview
 
-This package provides the HTTP API server for the Zeam node with the following endpoints:
+This package provides two HTTP servers for the Zeam node:
 
-- Server-Sent Events (SSE) stream for real-time chain events at `/events`
-- Prometheus metrics endpoint at `/metrics`
-- Health check at `/lean/v0/health`
-- Finalized checkpoint state at `/lean/v0/states/finalized` (for checkpoint sync)
-- Justified checkpoint information at `/lean/v0/checkpoints/justified`
-- Fork choice state at `/lean/v0/fork_choice` (full fork choice snapshot as JSON)
-- Fork choice graph visualization at `/api/forkchoice/graph` (Grafana node-graph compatible)
+**Metrics Server** (port 9668) - starts immediately:
+- `/metrics` - Prometheus metrics
+- `/lean/v0/health` - Liveness check
+
+**API Server** (port 9667) - starts after chain init:
+- `/lean/v0/ready` - Readiness check
+- `/lean/v0/states/finalized` - Checkpoint state (for checkpoint sync)
+- `/lean/v0/checkpoints/justified` - Justified checkpoint info
+- `/lean/v0/fork_choice` - Full fork choice snapshot (JSON)
+- `/api/forkchoice/graph` - Fork choice visualization (Grafana compatible)
+- `/events` - SSE stream for real-time chain events
 
 ## Package Components
 
@@ -33,9 +37,10 @@ Provides real-time chain event streaming via Server-Sent Events:
 - `new_justification` - New justified checkpoint
 - `new_finalization` - New finalized checkpoint
 
-### 2. Health Checks
+### 2. Health & Readiness Checks
 
-Simple health check endpoint at `/lean/v0/health`.
+- `/lean/v0/health` on metrics server (9668) - Liveness check, available immediately
+- `/lean/v0/ready` on API server (9667) - Readiness check, available after chain init
 
 ## Event System
 
@@ -101,12 +106,20 @@ Streams real-time chain events (head, justification, finalization).
 curl -N http://localhost:9667/events
 ```
 
-### `/lean/v0/health`
+### `/lean/v0/health` (Metrics Server)
+
+Returns liveness status. Available immediately on metrics port.
+
+```sh
+curl http://localhost:9668/lean/v0/health
+```
+
+### `/lean/v0/ready` (API Server)
 
-Returns node health status.
+Returns readiness status. Available after chain initialization.
 
 ```sh
-curl http://localhost:9667/lean/v0/health
+curl http://localhost:9667/lean/v0/ready
 ```
 
 ### `/api/forkchoice/graph`
@@ -190,25 +203,28 @@ The API system is initialized at startup in `pkgs/cli/src/main.zig`:
 // Initialize metrics
 try api.init(allocator);
 
-// Start HTTP server in background thread
-// chain can be null for early startup (chain-dependent endpoints return 503 until set)
-var handle = try api_server.startAPIServer(allocator, port, logger_config, chain);
+// Start metrics server early (no chain dependency)
+var metrics_handle = try metrics_server.startMetricsServer(allocator, metrics_port, logger_config);
 
-// Later, set chain if started with null
-handle.setChain(beam_chain);
+// After chain initialization, start API server
+var api_handle = try api_server.startAPIServer(allocator, api_port, logger_config, chain);
 
 // Graceful shutdown
-handle.stop();
+api_handle.stop();
+metrics_handle.stop();
 ```
 
-The server exposes:
-- SSE at `/events`
-- Metrics at `/metrics`
-- Health at `/lean/v0/health`
-- Checkpoint state at `/lean/v0/states/finalized`
-- Justified checkpoint at `/lean/v0/checkpoints/justified`
-- Fork choice state at `/lean/v0/fork_choice`
-- Fork choice visualization at `/api/forkchoice/graph`
+**Metrics Server** (port 9668) - starts immediately:
+- `/metrics` - Prometheus metrics (JSON)
+- `/lean/v0/health` - Liveness check (JSON)
+
+**API Server** (port 9667) - starts after chain init:
+- `/lean/v0/ready` - Readiness check (JSON)
+- `/lean/v0/states/finalized` - Checkpoint state (SSZ)
+- `/lean/v0/checkpoints/justified` - Justified checkpoint (JSON)
+- `/lean/v0/fork_choice` - Full fork choice snapshot (JSON)
+- `/api/forkchoice/graph` - Fork choice visualization (JSON)
+- `/events` - SSE event streaming
 
 **Note**: On freestanding targets (ZKVM), the HTTP server is automatically disabled.
 
@@ -252,29 +268,32 @@ pkgs/cli/src/api_server.zig ← HTTP server (serves via endpoints)
 Start a node:
 
 ```sh
-./zig-out/bin/zeam beam --mockNetwork --api-port 9668
+./zig-out/bin/zeam beam --mockNetwork
 ```
 
 Test endpoints:
 
 ```sh
-# SSE events
-curl -N http://localhost:9668/events
+# Health (metrics server - port 9668)
+curl http://localhost:9668/lean/v0/health
 
-# Metrics
+# Metrics (metrics server - port 9668)
 curl http://localhost:9668/metrics
 
-# Health
-curl http://localhost:9668/lean/v0/health
+# Readiness (API server - port 9667)
+curl http://localhost:9667/lean/v0/ready
 
-# Checkpoint state
-curl http://localhost:9668/lean/v0/states/finalized -o state.ssz
+# SSE events (API server - port 9667)
+curl -N http://localhost:9667/events
 
-# Justified checkpoint
-curl http://localhost:9668/lean/v0/checkpoints/justified
+# Checkpoint state (API server - port 9667)
+curl http://localhost:9667/lean/v0/states/finalized -o state.ssz
 
-# Fork choice state
-curl http://localhost:9668/lean/v0/fork_choice
+# Justified checkpoint (API server - port 9667)
+curl http://localhost:9667/lean/v0/checkpoints/justified
+
+# Fork choice state (API server - port 9667)
+curl http://localhost:9667/lean/v0/fork_choice
 ```
 
 ## Visualization with Prometheus & Grafana

pkgs/cli/src/api_server.zig

@@ -33,11 +33,10 @@ const StartupStatus = enum(u8) {
 };
 
 /// API server that runs in a background thread
-/// Handles SSE events, health checks, forkchoice graph, and checkpoint state endpoints
-/// chain is optional - if null, chain-dependent endpoints will return 503
-/// (API server starts before chain initialization, so chain may not be available yet)
+/// Handles SSE events, forkchoice graph, and checkpoint state endpoints
+/// Note: Health checks are served by the metrics_server (for early liveness checks)
 /// Note: Metrics are served by the separate metrics_server on a different port
-pub fn startAPIServer(allocator: std.mem.Allocator, port: u16, logger_config: *LoggerConfig, chain: ?*BeamChain) !*ApiServer {
+pub fn startAPIServer(allocator: std.mem.Allocator, port: u16, logger_config: *LoggerConfig, chain: *BeamChain) !*ApiServer {
     // Initialize the global event broadcaster for SSE events
     // This is idempotent - safe to call even if already initialized elsewhere (e.g., node.zig)
     try event_broadcaster.initGlobalBroadcaster(allocator);
@@ -56,7 +55,7 @@ pub fn startAPIServer(allocator: std.mem.Allocator, port: u16, logger_config: *L
         .allocator = allocator,
         .port = port,
         .logger = logger,
-        .chain = std.atomic.Value(?*BeamChain).init(chain),
+        .chain = chain,
         .stopped = std.atomic.Value(bool).init(false),
         .startup_status = std.atomic.Value(StartupStatus).init(.pending),
         .sse_active = 0,
@@ -127,8 +126,8 @@ fn routeConnection(connection: std.net.Server.Connection, allocator: std.mem.All
         defer arena.deinit();
         const request_allocator = arena.allocator();
 
-        if (std.mem.eql(u8, request.head.target, "/lean/v0/health")) {
-            ctx.handleHealth(&request);
+        if (std.mem.eql(u8, request.head.target, "/lean/v0/ready")) {
+            ctx.handleReady(&request);
         } else if (std.mem.eql(u8, request.head.target, "/lean/v0/states/finalized")) {
             ctx.handleFinalizedCheckpointState(&request) catch |err| {
                 ctx.logger.warn("failed to handle finalized checkpoint state request: {}", .{err});
@@ -145,16 +144,11 @@ fn routeConnection(connection: std.net.Server.Connection, allocator: std.mem.All
                 _ = request.respond("Internal Server Error\n", .{ .status = .internal_server_error }) catch {};
             };
         } else if (std.mem.startsWith(u8, request.head.target, "/api/forkchoice/graph")) {
-            const chain = ctx.getChain() orelse {
-                _ = request.respond("Service Unavailable: Chain not initialized\n", .{ .status = .service_unavailable }) catch {};
-                connection.stream.close();
-                return;
-            };
             if (!ctx.rate_limiter.allow(connection.address) or !ctx.tryAcquireGraph()) {
                 _ = request.respond("Too Many Requests\n", .{ .status = .too_many_requests }) catch {};
             } else {
                 defer ctx.releaseGraph();
-                handleForkChoiceGraph(&request, request_allocator, chain) catch |err| {
+                handleForkChoiceGraph(&request, request_allocator, ctx.chain) catch |err| {
                     ctx.logger.warn("fork choice graph request failed: {}", .{err});
                     _ = request.respond("Internal Server Error\n", .{}) catch {};
                 };
@@ -171,7 +165,7 @@ pub const ApiServer = struct {
     allocator: std.mem.Allocator,
     port: u16,
     logger: ModuleLogger,
-    chain: std.atomic.Value(?*BeamChain),
+    chain: *BeamChain,
     stopped: std.atomic.Value(bool),
     startup_status: std.atomic.Value(StartupStatus),
     sse_active: usize,
@@ -192,14 +186,6 @@ pub const ApiServer = struct {
         self.allocator.destroy(self);
     }
 
-    pub fn setChain(self: *Self, chain: *BeamChain) void {
-        self.chain.store(chain, .release);
-    }
-
-    fn getChain(self: *const Self) ?*BeamChain {
-        return self.chain.load(.acquire);
-    }
-
     fn run(self: *Self) void {
         const address = std.net.Address.parseIp4("0.0.0.0", self.port) catch |err| {
             self.logger.err("failed to parse server address 0.0.0.0:{d}: {}", .{ self.port, err });
@@ -241,10 +227,10 @@ pub const ApiServer = struct {
         }
     }
 
-    /// Handle health check endpoint
-    fn handleHealth(_: *const Self, request: *std.http.Server.Request) void {
-        const response = "{\"status\":\"healthy\",\"service\":\"zeam-api\"}";
-        _ = request.respond(response, .{
+    /// Handle readiness check endpoint
+    /// Returns 200 when API server is running (chain is always initialized at this point)
+    fn handleReady(_: *const Self, request: *std.http.Server.Request) void {
+        _ = request.respond("{\"ready\":true}", .{
             .extra_headers = &.{
                 .{ .name = "content-type", .value = "application/json; charset=utf-8" },
             },
@@ -254,14 +240,8 @@ pub const ApiServer = struct {
     /// Handle finalized checkpoint state endpoint
     /// Serves the finalized checkpoint lean state (BeamState) as SSZ octet-stream at /lean/v0/states/finalized
     fn handleFinalizedCheckpointState(self: *const Self, request: *std.http.Server.Request) !void {
-        // Get the chain (may be null if API server started before chain initialization)
-        const chain = self.getChain() orelse {
-            _ = request.respond("Service Unavailable: Chain not initialized\n", .{ .status = .service_unavailable }) catch {};
-            return;
-        };
-
         // Get finalized state from chain (chain handles its own locking internally)
-        const finalized_lean_state = chain.getFinalizedState() orelse {
+        const finalized_lean_state = self.chain.getFinalizedState() orelse {
             _ = request.respond("Not Found: Finalized checkpoint lean state not available\n", .{ .status = .not_found }) catch {};
             return;
         };
@@ -296,14 +276,8 @@ pub const ApiServer = struct {
     /// Returns checkpoint info as JSON at /lean/v0/checkpoints/justified
     /// Useful for monitoring consensus progress and fork choice state
     fn handleJustifiedCheckpoint(self: *const Self, request: *std.http.Server.Request) !void {
-        // Get the chain (may be null if API server started before chain initialization)
-        const chain = self.getChain() orelse {
-            _ = request.respond("Service Unavailable: Chain not initialized\n", .{ .status = .service_unavailable }) catch {};
-            return;
-        };
-
         // Get justified checkpoint from chain (chain handles its own locking internally)
-        const justified_checkpoint = chain.getJustifiedCheckpoint();
+        const justified_checkpoint = self.chain.getJustifiedCheckpoint();
 
         // Convert checkpoint to JSON string
         const json_string = justified_checkpoint.toJsonString(self.allocator) catch |err| {
@@ -328,10 +302,7 @@ pub const ApiServer = struct {
     /// Returns full fork choice state as JSON at /lean/v0/fork_choice
     /// Includes head, justified, finalized checkpoints, safe target, and all proto nodes
     fn handleForkChoice(self: *const Self, request: *std.http.Server.Request, allocator: std.mem.Allocator) !void {
-        const chain = self.getChain() orelse {
-            _ = request.respond("Service Unavailable: Chain not initialized\n", .{ .status = .service_unavailable }) catch {};
-            return;
-        };
+        const chain = self.chain;
 
         const snapshot = chain.forkChoice.snapshot(allocator) catch |err| {
             self.logger.err("failed to get fork choice snapshot: {}", .{err});

pkgs/cli/src/main.zig

@@ -380,12 +380,6 @@ fn mainInner() !void {
                 return err;
             };
 
-            // Start API server early. Pass null for chain - in .beam command mode, chains are created later
-            api_server_handle = api_server.startAPIServer(allocator, beamcmd.@"api-port", &api_logger_config, null) catch |err| {
-                ErrorHandler.logErrorWithDetails(err, "start API server", .{ .port = beamcmd.@"api-port" });
-                return err;
-            };
-
             std.debug.print("beam={any}\n", .{beamcmd});
 
             const mock_network = beamcmd.mockNetwork;
@@ -604,9 +598,11 @@ fn mainInner() !void {
                 .is_aggregator = beamcmd.@"is-aggregator",
             });
 
-            if (api_server_handle) |handle| {
-                handle.setChain(beam_node_1.chain);
-            }
+            // Start API server after chain initialization so chain can be passed directly
+            api_server_handle = api_server.startAPIServer(allocator, beamcmd.@"api-port", &api_logger_config, beam_node_1.chain) catch |err| {
+                ErrorHandler.logErrorWithDetails(err, "start API server", .{ .port = beamcmd.@"api-port" });
+                return err;
+            };
 
             var beam_node_2: BeamNode = undefined;
             try beam_node_2.init(allocator, .{

pkgs/cli/src/metrics_server.zig

@@ -7,9 +7,9 @@ const ModuleLogger = utils_lib.ModuleLogger;
 const ACCEPT_POLL_NS: u64 = 50 * std.time.ns_per_ms;
 const STARTUP_POLL_NS: u64 = 1 * std.time.ns_per_ms;
 
-/// Simple metrics server that only serves Prometheus metrics at /metrics endpoint.
-/// This is a lightweight server separate from the main API server.
-/// It has no rate limiting, SSE support, or chain dependency.
+/// Lightweight server for metrics and health endpoints.
+/// Serves /metrics (Prometheus) and /lean/v0/health (liveness check).
+/// Starts early, before chain initialization, with no chain dependency.
 pub fn startMetricsServer(
     allocator: std.mem.Allocator,
     port: u16,
@@ -131,6 +131,8 @@ pub const MetricsServer = struct {
 
         if (std.mem.eql(u8, request.head.target, "/metrics")) {
             self.handleMetrics(&request);
+        } else if (std.mem.eql(u8, request.head.target, "/lean/v0/health")) {
+            handleHealth(&request);
         } else {
             _ = request.respond("Not Found\n", .{ .status = .not_found }) catch {};
         }
@@ -156,3 +158,13 @@ pub const MetricsServer = struct {
         }) catch {};
     }
 };
+
+/// Handle health check endpoint (liveness) - always returns healthy
+fn handleHealth(request: *std.http.Server.Request) void {
+    const response = "{\"status\":\"healthy\",\"service\":\"zeam\"}";
+    _ = request.respond(response, .{
+        .extra_headers = &.{
+            .{ .name = "content-type", .value = "application/json; charset=utf-8" },
+        },
+    }) catch {};
+}

pkgs/cli/test/integration.zig

@@ -176,8 +176,9 @@ const ZeamRequest = struct {
     }
 
     /// Make a request to the /lean/v0/health endpoint and return the response
+    /// Note: Health checks are served on the metrics port (default: 9668)
     fn getHealth(self: ZeamRequest) ![]u8 {
-        return self.makeRequestToPort("/lean/v0/health", constants.DEFAULT_API_PORT);
+        return self.makeRequestToPort("/lean/v0/health", constants.DEFAULT_METRICS_PORT);
     }
 
     /// Internal helper to make HTTP requests to any endpoint on the specified port