Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion crates/sandstorm-core/src/master/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2548,9 +2548,13 @@ fn record_host(
/// store lock is never held across an await.
pub(super) fn diff(st: &St, req: &PollRequest) -> Result<Vec<Command>, ApiErr> {
let store = st.store();
store
let accepted = store
.set_node_health(&req.node_id, req.free_memory_mib, req.pressure_some_avg10)
.map_err(|e| err(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
// Certificate validation may predate a long-poll hold and revocation.
if !accepted {
return Err(err(StatusCode::FORBIDDEN, "node is absent or revoked"));
}
// An empty list from an early wake report says nothing; keep the last.
if !req.features.is_empty() {
st.metrics
Expand Down
22 changes: 17 additions & 5 deletions crates/sandstorm-core/src/state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -776,13 +776,14 @@ impl Store {
Ok(n > 0)
}

pub fn set_node_health(&self, id: &str, free_mib: u64, psi: f64) -> Result<()> {
self.conn.execute(
/// False if the node is absent or revoked; a stale report cannot reconnect it.
pub fn set_node_health(&self, id: &str, free_mib: u64, psi: f64) -> Result<bool> {
let updated = self.conn.execute(
"UPDATE nodes SET connected=1, last_seen_at=?2, free_mib=?3, psi_some_avg10=?4
WHERE id=?1",
WHERE id=?1 AND revoked=0",
params![id, now_secs(), free_mib as i64, psi],
)?;
Ok(())
Ok(updated > 0)
}

pub fn list_nodes(&self) -> Result<Vec<Node>> {
Expand Down Expand Up @@ -1884,11 +1885,22 @@ mod tests {
psi_some_avg10: 0.0,
})
.unwrap();
s.set_node_health("bm-1", 408, 15.6).unwrap();
assert!(s.set_node_health("bm-1", 408, 15.6).unwrap());
let n = &s.list_nodes().unwrap()[0];
assert_eq!(n.free_mib, 408);
assert!((n.psi_some_avg10 - 15.6).abs() < 1e-9);
assert!(n.connected, "reporting health means it is connected");

s.revoke_node("bm-1").unwrap();
assert!(!s.set_node_health("bm-1", 1024, 0.0).unwrap());
let n = &s.list_nodes().unwrap()[0];
assert!(!n.connected, "a revoked node stays offline");
assert_eq!(n.free_mib, 408, "a revoked report does not update health");
assert!((n.psi_some_avg10 - 15.6).abs() < 1e-9);

s.set_node_cert("bm-1", "new-certificate").unwrap();
assert!(s.set_node_health("bm-1", 1024, 0.0).unwrap());
assert!(s.list_nodes().unwrap()[0].connected);
}

/// A restart must not inherit the last run's connectivity.
Expand Down
133 changes: 133 additions & 0 deletions crates/sandstorm-core/tests/node_identity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,17 @@

use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;

use axum::body::Body;
use axum::http::{Request, StatusCode};
use sandstorm_core::identity::Ca;
use sandstorm_core::master::{router, MasterOptions, MasterState};
use sandstorm_core::proto::{Command, PollResponse};
use sandstorm_core::state::Store;
use sandstorm_core::tls::{self, PeerCert};
use tempfile::TempDir;
use tower::ServiceExt;

struct Harness {
base: String,
Expand Down Expand Up @@ -107,6 +112,42 @@ async fn poll_status(base: &str, node_id: &str) -> u16 {
.as_u16()
}

/// Return only after this authenticated, idle request has entered its hold.
async fn park_poll(
st: Arc<MasterState>,
fingerprint: String,
) -> tokio::task::JoinHandle<axum::response::Response> {
let app = router(st.clone()).layer(axum::Extension(PeerCert(Some(fingerprint))));
let request = Request::post("/v1/node/poll")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"node_id": "node-a", "observed": [], "hold_ms": 55_000,
"free_memory_mib": 1024, "pressure_some_avg10": 0.0,
})
.to_string(),
))
.unwrap();
let (parked, waiting) = tokio::sync::oneshot::channel();
let task = tokio::spawn(async move {
let response = app.oneshot(request);
tokio::pin!(response);
// With an in-memory body, authentication and the first diff are
// synchronous: the first Pending is the long-poll wait.
assert!(futures_util::poll!(&mut response).is_pending());
parked.send(()).unwrap();
response.await.unwrap()
});
tokio::time::timeout(Duration::from_secs(5), waiting)
.await
.expect("the poll must enter its hold")
.unwrap();
let node = &st.store().list_nodes().unwrap()[0];
assert!(node.connected);
assert_eq!(node.free_mib, 1024, "the first diff has recorded health");
task
}

// ---- which node is this -------------------------------------------------

#[tokio::test]
Expand Down Expand Up @@ -200,6 +241,98 @@ async fn identity_revoking_stops_the_node_at_once() {
assert_eq!(poll_status(&base, "node-a").await, 403);
}

#[tokio::test]
async fn identity_revoking_a_parked_poll_keeps_the_node_offline() {
let setup = start_with_presented(None).await;
let (cert, _) = register(&setup, "node-a").await;
let fp = tls::fingerprint_of_pem(&cert).unwrap();
setup
.st
.store()
.put_template("python", "rootfs", "{}")
.unwrap();
let parked = park_poll(setup.st.clone(), fp.clone()).await;

let revoked = reqwest::Client::new()
.post(format!("{}/v1/node/revoke/node-a", setup.base))
.send()
.await
.unwrap();
assert_eq!(revoked.status(), StatusCode::OK);
let connected_after_revoke = setup.st.store().list_nodes().unwrap()[0].connected;

let response = tokio::time::timeout(Duration::from_secs(5), parked)
.await
.expect("revocation must wake the held poll")
.unwrap();
assert_eq!(response.status(), StatusCode::FORBIDDEN, "the held poll");
assert!(!connected_after_revoke, "revocation keeps the node offline");
assert!(!setup.st.store().list_nodes().unwrap()[0].connected);

let app = router(setup.st.clone()).layer(axum::Extension(PeerCert(Some(fp))));
let response = app
.oneshot(
Request::post("/v1/node/poll")
.header("content-type", "application/json")
.body(Body::from(
serde_json::json!({
"node_id": "node-a", "observed": [], "hold_ms": 0,
"free_memory_mib": 1024, "pressure_some_avg10": 0.0,
})
.to_string(),
))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::FORBIDDEN, "a new poll");
assert!(!setup.st.store().list_nodes().unwrap()[0].connected);

let created = reqwest::Client::new()
.post(format!("{}/v1/sandboxes", setup.base))
.json(&serde_json::json!({ "template": "python" }))
.send()
.await
.unwrap();
assert_eq!(created.status(), StatusCode::SERVICE_UNAVAILABLE);
assert!(!setup.st.store().list_nodes().unwrap()[0].connected);
}

#[tokio::test]
async fn identity_an_unrevoked_parked_poll_is_woken_with_commands() {
let setup = start_with_presented(None).await;
let (cert, _) = register(&setup, "node-a").await;
let fp = tls::fingerprint_of_pem(&cert).unwrap();
setup
.st
.store()
.put_template("python", "rootfs", "{}")
.unwrap();
let parked = park_poll(setup.st.clone(), fp).await;

let created = reqwest::Client::new()
.post(format!("{}/v1/sandboxes", setup.base))
.json(&serde_json::json!({ "template": "python" }))
.send()
.await
.unwrap();
assert_eq!(created.status(), StatusCode::CREATED);
let created: serde_json::Value = created.json().await.unwrap();
let response = tokio::time::timeout(Duration::from_secs(5), parked)
.await
.expect("creating work must wake the held poll")
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.unwrap();
let response: PollResponse = serde_json::from_slice(&body).unwrap();
assert!(
matches!(response.commands.as_slice(), [Command::Create { id, .. }] if id == created["id"].as_str().unwrap())
);
assert!(setup.st.store().list_nodes().unwrap()[0].connected);
}

#[tokio::test]
async fn identity_re_registering_retires_the_old_certificate() {
let setup = start_with_presented(None).await;
Expand Down
Loading