feat: gateway health monitoring

CHANGELOG.md

@@ -3,11 +3,13 @@
 ### Added
 
 - Gateway: per-relay `blockfrost_gateway_relay_healthy`, `blockfrost_gateway_relay_data_node_up`, and `blockfrost_gateway_relay_info` metrics in `GET /metrics` (and the same data points in `GET /stats`)
+- Gateway: `blockfrost_gateway_healthy` metric in `GET /metrics`, mirroring the `healthy` field of `GET /`
 - New endpoints proxied to the data node: `/accounts/{stake_address}/utxos`, `/addresses/{address}`, and `/blocks/slot/{slot_number}`
 - `--max-response-body-bytes` to configure the maximum proxied response body size (default 10 MiB)
 
 ### Fixed
 
+- Gateway: `GET /` no longer always reports `healthy: true` — it now reflects periodic health checks of PostgreSQL connectivity and the Blockfrost API
 - Raised the proxied body limit from 1 MiB to 10 MiB
 - Restored `local_address` bind for unspecified IPs like `0.0.0.0`, fixing an IPv6 regression that broke the IPv4-forcing behavior of `--server-address 0.0.0.0`
 - Inverted the metrics answer in the `--init` config prompt

crates/gateway/CHANGELOG.md

@@ -3,6 +3,7 @@
 ### Added
 
 - Per-relay `blockfrost_gateway_relay_healthy`, `blockfrost_gateway_relay_data_node_up`, and `blockfrost_gateway_relay_info` metrics in `GET /metrics` (and the same data points in `GET /stats`)
+- `blockfrost_gateway_healthy` metric in `GET /metrics`, mirroring the `healthy` field of `GET /`
 - Prometheus metrics endpoint `GET /metrics` exposing per-relay stats (connection status, WebSocket RTT, connected-since timestamp, request/response counters) and PostgreSQL connection-pool gauges (max size, open, available, waiting)
 - Prometheus counter `blockfrost_gateway_http_requests_total` with `method`, `route`, and `status_code` labels for Gateway API requests
 - `blockfrost_gateway_build_info` metric exposing the Gateway version and git revision
@@ -12,6 +13,7 @@
 
 ### Fixed
 
+- `GET /` no longer always reports `healthy: true` — it now reflects periodic health checks of PostgreSQL connectivity and the Blockfrost API
 - The underlying Blockfrost API error (e.g. rate limiting) is now logged when the license NFT check fails during registration
 
 ### Removed

crates/gateway/src/api/metrics.rs

@@ -1,4 +1,5 @@
 use crate::db::{DB, PoolStatus};
+use crate::health_monitor::HealthMonitor;
 use crate::load_balancer::LoadBalancerState;
 use axum::{Extension, http::StatusCode, response::IntoResponse};
 use metrics::{describe_counter, describe_gauge, gauge};
@@ -122,6 +123,7 @@ const RELAY_METRICS: &[RelayMetric] = &[
 pub(crate) async fn render_prometheus(
     load_balancer: &LoadBalancerState,
     db_pool: &PoolStatus,
+    healthy: bool,
 ) -> Result<String, std::fmt::Error> {
     let now_chrono = chrono::Utc::now();
     let now_instant = std::time::Instant::now();
@@ -160,6 +162,13 @@ pub(crate) async fn render_prometheus(
 
     let mut out = String::new();
 
+    writeln!(
+        out,
+        "# HELP blockfrost_gateway_healthy Whether the Gateway is healthy, i.e. its periodic health checks (PostgreSQL connectivity, Blockfrost API) succeeded. Mirrors the `healthy` field of `GET /`."
+    )?;
+    writeln!(out, "# TYPE blockfrost_gateway_healthy gauge")?;
+    writeln!(out, "blockfrost_gateway_healthy {}", u8::from(healthy))?;
+
     writeln!(
         out,
         "# HELP blockfrost_gateway_connected_relays Number of relays currently connected via WebSocket."
@@ -231,11 +240,13 @@ pub(crate) async fn render_prometheus(
 pub async fn route(
     Extension(load_balancer): Extension<LoadBalancerState>,
     Extension(db): Extension<DB>,
+    Extension(health_monitor): Extension<HealthMonitor>,
     Extension(prometheus_handle): Extension<PrometheusHandle>,
 ) -> Result<impl IntoResponse, StatusCode> {
+    let healthy = health_monitor.current_status().await.healthy;
     let mut body = prometheus_handle.render();
     body.push_str(
-        &render_prometheus(&load_balancer, &db.pool_status())
+        &render_prometheus(&load_balancer, &db.pool_status(), healthy)
             .await
             .map_err(|e| {
                 error!("failed to render gateway metrics: {e}");
@@ -304,10 +315,12 @@ mod tests {
         });
         lb.active_relays.lock().await.insert(uuid, relay);
 
-        let out = render_prometheus(&lb, &test_pool_status())
+        let out = render_prometheus(&lb, &test_pool_status(), true)
             .await
             .expect("render metrics");
 
+        assert!(out.contains("# TYPE blockfrost_gateway_healthy gauge"));
+        assert!(out.contains("\nblockfrost_gateway_healthy 1\n"));
         assert!(out.contains("# TYPE blockfrost_gateway_connected_relays gauge"));
         assert!(out.contains("\nblockfrost_gateway_connected_relays 1\n"));
         assert!(out.contains("# TYPE blockfrost_gateway_db_pool_max_size gauge"));
@@ -348,7 +361,7 @@ mod tests {
             .await
             .insert(uuid, test_relay_state("Icebreaker2"));
 
-        let out = render_prometheus(&lb, &test_pool_status())
+        let out = render_prometheus(&lb, &test_pool_status(), true)
             .await
             .expect("render metrics");
 
@@ -370,12 +383,21 @@ mod tests {
     #[tokio::test]
     async fn reports_zero_when_no_relays() {
         let lb = LoadBalancerState::new(None, test_key());
-        let out = render_prometheus(&lb, &test_pool_status())
+        let out = render_prometheus(&lb, &test_pool_status(), true)
             .await
             .expect("render metrics");
         assert!(out.contains("\nblockfrost_gateway_connected_relays 0\n"));
     }
 
+    #[tokio::test]
+    async fn reports_unhealthy_gateway() {
+        let lb = LoadBalancerState::new(None, test_key());
+        let out = render_prometheus(&lb, &test_pool_status(), false)
+            .await
+            .expect("render metrics");
+        assert!(out.contains("\nblockfrost_gateway_healthy 0\n"));
+    }
+
     #[tokio::test]
     async fn escapes_label_values() {
         let lb = LoadBalancerState::new(None, test_key());
@@ -385,7 +407,7 @@ mod tests {
             .await
             .insert(uuid, test_relay_state("we\"ird\\name"));
 
-        let out = render_prometheus(&lb, &test_pool_status())
+        let out = render_prometheus(&lb, &test_pool_status(), true)
             .await
             .expect("render metrics");
         assert!(out.contains("relay=\"we\\\"ird\\\\name\""));

crates/gateway/src/api/root.rs

@@ -1,5 +1,6 @@
 use crate::config::Config;
-use axum::{Extension, Json};
+use crate::health_monitor::HealthMonitor;
+use axum::{Extension, Json, http::StatusCode, response::IntoResponse};
 use serde::Serialize;
 
 #[derive(Serialize)]
@@ -8,15 +9,28 @@ pub struct Response {
     pub version: String,
     pub healthy: bool,
     pub commit: &'static str,
+    pub errors: Vec<String>,
 }
 
-pub async fn route(Extension(config): Extension<Config>) -> Json<Response> {
+pub async fn route(
+    Extension(config): Extension<Config>,
+    Extension(health_monitor): Extension<HealthMonitor>,
+) -> impl IntoResponse {
+    let status = health_monitor.current_status().await;
+
+    let http_status = if status.healthy {
+        StatusCode::OK
+    } else {
+        StatusCode::SERVICE_UNAVAILABLE
+    };
+
     let response = Response {
         url: config.server.url.clone(),
         version: env!("CARGO_PKG_VERSION").to_string(),
         commit: env!("GIT_REVISION"),
-        healthy: true,
+        healthy: status.healthy,
+        errors: status.errors,
     };
 
-    Json(response)
+    (http_status, Json(response))
 }

crates/gateway/src/blockfrost.rs

@@ -23,6 +23,24 @@ impl BlockfrostAPI {
         }
     }
 
+    /// Verifies Blockfrost API connectivity
+    pub async fn ping(&self, timeout: std::time::Duration) -> Result<(), String> {
+        if cfg!(feature = "dev_mock_db") {
+            return Ok(());
+        }
+
+        tokio::time::timeout(timeout, self.api.blocks_latest())
+            .await
+            .map_err(|_| {
+                format!(
+                    "Blockfrost API health check timed out after {}s",
+                    timeout.as_secs()
+                )
+            })?
+            .map(|_| ())
+            .map_err(|e| format!("Blockfrost API error: {e}"))
+    }
+
     // Parse asset from the unit
     async fn parse_asset(&self, unit: &str) -> Result<Asset, APIError> {
         if unit.len() < self.policy_id_size {

crates/gateway/src/db.rs

@@ -46,6 +46,36 @@ impl DB {
         Self { pool }
     }
 
+    /// Verifies database connectivity by running a trivial query through the connection pool
+    pub async fn ping(&self, timeout: std::time::Duration) -> Result<(), String> {
+        if cfg!(feature = "dev_mock_db") {
+            return Ok(());
+        }
+
+        let check = async {
+            let connection = self
+                .pool
+                .get()
+                .await
+                .map_err(|e| format!("database pool error: {e}"))?;
+            connection
+                .interact(|c| diesel::sql_query("SELECT 1").execute(c))
+                .await
+                .map_err(|e| format!("database interaction error: {e}"))?
+                .map_err(|e| format!("database query error: {e}"))?;
+            Ok(())
+        };
+
+        tokio::time::timeout(timeout, check)
+            .await
+            .unwrap_or_else(|_| {
+                Err(format!(
+                    "database health check timed out after {}s (pool exhausted or database unreachable)",
+                    timeout.as_secs()
+                ))
+            })
+    }
+
     pub fn pool_status(&self) -> PoolStatus {
         let status = self.pool.status();
         PoolStatus {

crates/gateway/src/health_monitor.rs

@@ -0,0 +1,95 @@
+use crate::blockfrost::BlockfrostAPI;
+use crate::db::DB;
+use std::sync::Arc;
+use tokio::sync::{Mutex, Notify};
+use tokio::time::{self, Duration, Instant};
+
+#[derive(Clone)]
+pub struct HealthStatus {
+    pub healthy: bool,
+    pub errors: Vec<String>,
+}
+
+#[derive(Clone)]
+pub struct HealthMonitor {
+    status: Arc<Mutex<HealthStatus>>,
+}
+
+impl HealthMonitor {
+    const DB_PING_TIMEOUT: Duration = Duration::from_secs(5);
+    const BLOCKFROST_PING_TIMEOUT: Duration = Duration::from_secs(5);
+    /// The Blockfrost check runs on its own, slower cadence to keep the
+    /// request-quota usage of the health checks negligible.
+    const BLOCKFROST_CHECK_INTERVAL: Duration = Duration::from_secs(60);
+    const HEALTHY_CHECK_INTERVAL: Duration = Duration::from_secs(10);
+    const UNHEALTHY_CHECK_INTERVAL: Duration = Duration::from_secs(2);
+
+    pub async fn current_status(&self) -> HealthStatus {
+        self.status.lock().await.clone()
+    }
+
+    pub fn new_static(status: HealthStatus) -> Self {
+        Self {
+            status: Arc::new(Mutex::new(status)),
+        }
+    }
+
+    pub async fn spawn(db: DB, blockfrost_api: BlockfrostAPI) -> Self {
+        let self_ = Self::new_static(HealthStatus {
+            healthy: true,
+            errors: vec![],
+        });
+        let status = self_.status.clone();
+
+        let first_check_done = Arc::new(Notify::new());
+        let first_check_done_ = first_check_done.clone();
+
+        tokio::spawn(async move {
+            let mut previously_healthy = true;
+            let mut blockfrost_error: Option<String> = None;
+            let mut last_blockfrost_check: Option<Instant> = None;
+            loop {
+                let db_error = db.ping(Self::DB_PING_TIMEOUT).await.err();
+
+                if last_blockfrost_check
+                    .is_none_or(|at| at.elapsed() >= Self::BLOCKFROST_CHECK_INTERVAL)
+                {
+                    blockfrost_error = blockfrost_api
+                        .ping(Self::BLOCKFROST_PING_TIMEOUT)
+                        .await
+                        .err();
+                    last_blockfrost_check = Some(Instant::now());
+                }
+
+                let errors: Vec<String> = [db_error, blockfrost_error.clone()]
+                    .into_iter()
+                    .flatten()
+                    .collect();
+                let healthy = errors.is_empty();
+
+                if previously_healthy && !healthy {
+                    tracing::warn!("Gateway became unhealthy: {}", errors.join("; "));
+                } else if !previously_healthy && healthy {
+                    tracing::warn!("Gateway became healthy again.");
+                }
+
+                previously_healthy = healthy;
+
+                *status.lock().await = HealthStatus { healthy, errors };
+
+                first_check_done_.notify_one();
+
+                let delay = if healthy {
+                    Self::HEALTHY_CHECK_INTERVAL
+                } else {
+                    Self::UNHEALTHY_CHECK_INTERVAL
+                };
+
+                time::sleep(delay).await;
+            }
+        });
+
+        first_check_done.notified().await;
+        self_
+    }
+}

crates/gateway/src/lib.rs

@@ -3,6 +3,7 @@ pub mod blockfrost;
 pub mod config;
 pub mod db;
 pub mod errors;
+pub mod health_monitor;
 pub mod hydra_server_bridge;
 pub mod hydra_server_platform;
 pub mod load_balancer;

crates/gateway/src/main.rs

@@ -7,8 +7,8 @@ use axum::{
 };
 use bf_common::tracing::setup_tracing;
 use blockfrost_gateway::{
-    api, blockfrost, config, db, hydra_server_bridge, hydra_server_platform, load_balancer,
-    middlewares, rate_limit, sdk_bridge_ws,
+    api, blockfrost, config, db, health_monitor, hydra_server_bridge, hydra_server_platform,
+    load_balancer, middlewares, rate_limit, sdk_bridge_ws,
 };
 use clap::Parser;
 use colored::Colorize;
@@ -42,6 +42,8 @@ async fn main() -> Result<()> {
     )
     .await;
     let blockfrost_api = blockfrost::BlockfrostAPI::new(&config.blockfrost.project_id);
+    let health_monitor =
+        health_monitor::HealthMonitor::spawn(pool.clone(), blockfrost_api.clone()).await;
     let hydras_manager = if let Some(hydra_platform_config) = &config.hydra_platform {
         Some(
             hydra_server_platform::HydrasManager::new(
@@ -103,6 +105,7 @@ async fn main() -> Result<()> {
         .layer(Extension(load_balancer))
         .layer(Extension(config.clone()))
         .layer(Extension(pool))
+        .layer(Extension(health_monitor))
         .layer(Extension(blockfrost_api))
         .layer(Extension(register_rate_limiter))
         .layer(Extension(prometheus_handle));