diff --git a/crates/sandstorm-core/src/master/api.rs b/crates/sandstorm-core/src/master/api.rs index 4fe6e163..2c226cf6 100644 --- a/crates/sandstorm-core/src/master/api.rs +++ b/crates/sandstorm-core/src/master/api.rs @@ -2548,9 +2548,13 @@ fn record_host( /// store lock is never held across an await. pub(super) fn diff(st: &St, req: &PollRequest) -> Result, ApiErr> { let store = st.store(); - store + let accepted = store .set_node_health(&req.node_id, req.free_memory_mib, req.pressure_some_avg10) .map_err(|e| err(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + // Certificate validation may predate a long-poll hold and revocation. + if !accepted { + return Err(err(StatusCode::FORBIDDEN, "node is absent or revoked")); + } // An empty list from an early wake report says nothing; keep the last. if !req.features.is_empty() { st.metrics diff --git a/crates/sandstorm-core/src/state.rs b/crates/sandstorm-core/src/state.rs index 3c6daac1..0e17964d 100644 --- a/crates/sandstorm-core/src/state.rs +++ b/crates/sandstorm-core/src/state.rs @@ -776,13 +776,14 @@ impl Store { Ok(n > 0) } - pub fn set_node_health(&self, id: &str, free_mib: u64, psi: f64) -> Result<()> { - self.conn.execute( + /// False if the node is absent or revoked; a stale report cannot reconnect it. + pub fn set_node_health(&self, id: &str, free_mib: u64, psi: f64) -> Result { + let updated = self.conn.execute( "UPDATE nodes SET connected=1, last_seen_at=?2, free_mib=?3, psi_some_avg10=?4 - WHERE id=?1", + WHERE id=?1 AND revoked=0", params![id, now_secs(), free_mib as i64, psi], )?; - Ok(()) + Ok(updated > 0) } pub fn list_nodes(&self) -> Result> { @@ -1884,11 +1885,22 @@ mod tests { psi_some_avg10: 0.0, }) .unwrap(); - s.set_node_health("bm-1", 408, 15.6).unwrap(); + assert!(s.set_node_health("bm-1", 408, 15.6).unwrap()); let n = &s.list_nodes().unwrap()[0]; assert_eq!(n.free_mib, 408); assert!((n.psi_some_avg10 - 15.6).abs() < 1e-9); assert!(n.connected, "reporting health means it is connected"); + + s.revoke_node("bm-1").unwrap(); + assert!(!s.set_node_health("bm-1", 1024, 0.0).unwrap()); + let n = &s.list_nodes().unwrap()[0]; + assert!(!n.connected, "a revoked node stays offline"); + assert_eq!(n.free_mib, 408, "a revoked report does not update health"); + assert!((n.psi_some_avg10 - 15.6).abs() < 1e-9); + + s.set_node_cert("bm-1", "new-certificate").unwrap(); + assert!(s.set_node_health("bm-1", 1024, 0.0).unwrap()); + assert!(s.list_nodes().unwrap()[0].connected); } /// A restart must not inherit the last run's connectivity. diff --git a/crates/sandstorm-core/tests/node_identity.rs b/crates/sandstorm-core/tests/node_identity.rs index 0274690c..479e046e 100644 --- a/crates/sandstorm-core/tests/node_identity.rs +++ b/crates/sandstorm-core/tests/node_identity.rs @@ -13,12 +13,17 @@ use std::net::SocketAddr; use std::sync::Arc; +use std::time::Duration; +use axum::body::Body; +use axum::http::{Request, StatusCode}; use sandstorm_core::identity::Ca; use sandstorm_core::master::{router, MasterOptions, MasterState}; +use sandstorm_core::proto::{Command, PollResponse}; use sandstorm_core::state::Store; use sandstorm_core::tls::{self, PeerCert}; use tempfile::TempDir; +use tower::ServiceExt; struct Harness { base: String, @@ -107,6 +112,42 @@ async fn poll_status(base: &str, node_id: &str) -> u16 { .as_u16() } +/// Return only after this authenticated, idle request has entered its hold. +async fn park_poll( + st: Arc, + fingerprint: String, +) -> tokio::task::JoinHandle { + let app = router(st.clone()).layer(axum::Extension(PeerCert(Some(fingerprint)))); + let request = Request::post("/v1/node/poll") + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ + "node_id": "node-a", "observed": [], "hold_ms": 55_000, + "free_memory_mib": 1024, "pressure_some_avg10": 0.0, + }) + .to_string(), + )) + .unwrap(); + let (parked, waiting) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let response = app.oneshot(request); + tokio::pin!(response); + // With an in-memory body, authentication and the first diff are + // synchronous: the first Pending is the long-poll wait. + assert!(futures_util::poll!(&mut response).is_pending()); + parked.send(()).unwrap(); + response.await.unwrap() + }); + tokio::time::timeout(Duration::from_secs(5), waiting) + .await + .expect("the poll must enter its hold") + .unwrap(); + let node = &st.store().list_nodes().unwrap()[0]; + assert!(node.connected); + assert_eq!(node.free_mib, 1024, "the first diff has recorded health"); + task +} + // ---- which node is this ------------------------------------------------- #[tokio::test] @@ -200,6 +241,98 @@ async fn identity_revoking_stops_the_node_at_once() { assert_eq!(poll_status(&base, "node-a").await, 403); } +#[tokio::test] +async fn identity_revoking_a_parked_poll_keeps_the_node_offline() { + let setup = start_with_presented(None).await; + let (cert, _) = register(&setup, "node-a").await; + let fp = tls::fingerprint_of_pem(&cert).unwrap(); + setup + .st + .store() + .put_template("python", "rootfs", "{}") + .unwrap(); + let parked = park_poll(setup.st.clone(), fp.clone()).await; + + let revoked = reqwest::Client::new() + .post(format!("{}/v1/node/revoke/node-a", setup.base)) + .send() + .await + .unwrap(); + assert_eq!(revoked.status(), StatusCode::OK); + let connected_after_revoke = setup.st.store().list_nodes().unwrap()[0].connected; + + let response = tokio::time::timeout(Duration::from_secs(5), parked) + .await + .expect("revocation must wake the held poll") + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN, "the held poll"); + assert!(!connected_after_revoke, "revocation keeps the node offline"); + assert!(!setup.st.store().list_nodes().unwrap()[0].connected); + + let app = router(setup.st.clone()).layer(axum::Extension(PeerCert(Some(fp)))); + let response = app + .oneshot( + Request::post("/v1/node/poll") + .header("content-type", "application/json") + .body(Body::from( + serde_json::json!({ + "node_id": "node-a", "observed": [], "hold_ms": 0, + "free_memory_mib": 1024, "pressure_some_avg10": 0.0, + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN, "a new poll"); + assert!(!setup.st.store().list_nodes().unwrap()[0].connected); + + let created = reqwest::Client::new() + .post(format!("{}/v1/sandboxes", setup.base)) + .json(&serde_json::json!({ "template": "python" })) + .send() + .await + .unwrap(); + assert_eq!(created.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(!setup.st.store().list_nodes().unwrap()[0].connected); +} + +#[tokio::test] +async fn identity_an_unrevoked_parked_poll_is_woken_with_commands() { + let setup = start_with_presented(None).await; + let (cert, _) = register(&setup, "node-a").await; + let fp = tls::fingerprint_of_pem(&cert).unwrap(); + setup + .st + .store() + .put_template("python", "rootfs", "{}") + .unwrap(); + let parked = park_poll(setup.st.clone(), fp).await; + + let created = reqwest::Client::new() + .post(format!("{}/v1/sandboxes", setup.base)) + .json(&serde_json::json!({ "template": "python" })) + .send() + .await + .unwrap(); + assert_eq!(created.status(), StatusCode::CREATED); + let created: serde_json::Value = created.json().await.unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), parked) + .await + .expect("creating work must wake the held poll") + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let response: PollResponse = serde_json::from_slice(&body).unwrap(); + assert!( + matches!(response.commands.as_slice(), [Command::Create { id, .. }] if id == created["id"].as_str().unwrap()) + ); + assert!(setup.st.store().list_nodes().unwrap()[0].connected); +} + #[tokio::test] async fn identity_re_registering_retires_the_old_certificate() { let setup = start_with_presented(None).await;