From 50490cf55ca2b78b5dc3a13920da94fdf626a90b Mon Sep 17 00:00:00 2001 From: dada-yan Date: Sat, 14 Nov 2026 16:11:21 +1100 Subject: [PATCH] fix(node): retry gateway activity after failed early reports Signed-off-by: dada-yan --- crates/sandstorm-core/src/master/api.rs | 24 +++- crates/sandstorm-core/src/node/gateway.rs | 7 +- crates/sandstorm-core/src/node/mod.rs | 157 +++++++++++++++++++-- crates/sandstorm-core/src/node/wake.rs | 72 +++++++--- crates/sandstorm-core/src/proto.rs | 27 ++++ crates/sandstorm-core/tests/listen_mode.rs | 1 + crates/sandstorm-core/tests/master_api.rs | 106 +++++++++++++- 7 files changed, 355 insertions(+), 39 deletions(-) diff --git a/crates/sandstorm-core/src/master/api.rs b/crates/sandstorm-core/src/master/api.rs index 4fe6e163..2312e359 100644 --- a/crates/sandstorm-core/src/master/api.rs +++ b/crates/sandstorm-core/src/master/api.rs @@ -19,8 +19,8 @@ use crate::auth::{require_token, AuthConfig}; use crate::identity::{random_token, Ca}; use crate::node::BackendKind; use crate::proto::{ - Command, CommandResult, Egress, FetchFrom, PollRequest, PollResponse, RegisterRequest, - RegisterResponse, + Command, CommandResult, Egress, FetchFrom, GatewayActivity, PollRequest, PollResponse, + RegisterRequest, RegisterResponse, }; use crate::state::{now_secs, Node, Sandbox, SnapshotTag, Store}; use crate::tier::{IdlePolicy, Tier}; @@ -2459,12 +2459,21 @@ pub(super) fn record_failures(st: &St, req: &PollRequest) -> Result<(), ApiErr> /// that request's late report. Late traffic still moves the idle clock; /// it just does not overrule a demotion that came after it. pub(super) fn apply_traffic(st: &St, req: &PollRequest) -> Result<(), ApiErr> { - if req.traffic.is_empty() { + if req.traffic.is_empty() && req.gateway_activity.is_empty() { return Ok(()); } + let mut by_id: HashMap<&str, Vec<&str>> = HashMap::new(); + for event in &req.gateway_activity { + by_id.entry(&event.id).or_default().push(&event.token_hash); + } + // A scoped report is authoritative for its ID even if none of its + // events match. Use the legacy IDs only for unscoped entries. + for id in &req.traffic { + by_id.entry(id).or_default(); + } let store = st.store(); let ise = |e: anyhow::Error| err(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()); - for id in &req.traffic { + for (id, token_hashes) in by_id { let Some(sb) = store.get_sandbox(id).map_err(ise)? else { continue; }; @@ -2472,11 +2481,16 @@ pub(super) fn apply_traffic(st: &St, req: &PollRequest) -> Result<(), ApiErr> { tracing::warn!(sandbox = %id, node = %req.node_id, holder = %sb.node_id, "ignored traffic reported by a node that does not hold the sandbox"); continue; } + if !token_hashes.is_empty() + && !token_hashes.contains(&GatewayActivity::hash_token(&sb.token).as_str()) + { + continue; + } store.touch(id).map_err(ise)?; let serving = req .observed .iter() - .any(|o| &o.id == id && o.tier.is_serving()); + .any(|o| o.id == id && o.tier.is_serving()); let asked_for = st .explicit_tier .lock() diff --git a/crates/sandstorm-core/src/node/gateway.rs b/crates/sandstorm-core/src/node/gateway.rs index 7e8d25a8..413e8b1a 100644 --- a/crates/sandstorm-core/src/node/gateway.rs +++ b/crates/sandstorm-core/src/node/gateway.rs @@ -411,7 +411,7 @@ async fn proxy(State(t): State>, req: Request) -> Response { // in flight is deferred rather than pulling the sandbox out from under // it. if let Some(w) = &t.wake { - w.saw(&t.id); + w.saw(&t.id, &expected); } // Held to the end of the answer, the wake included: a sandbox with a // request in flight is never put away to make room. @@ -766,7 +766,10 @@ mod tests { assert_eq!(woken.load(std::sync::atomic::Ordering::SeqCst), 1); // Both requests are reported to the master, once. - assert_eq!(w.take_traffic(), vec!["sb-cold".to_string()]); + let (ids, activity) = w.take_traffic(true); + assert_eq!(ids, ["sb-cold"]); + assert_eq!(activity.len(), 1); + assert_eq!(activity[0].id, "sb-cold"); g.reconcile(&[]).await; } diff --git a/crates/sandstorm-core/src/node/mod.rs b/crates/sandstorm-core/src/node/mod.rs index a38d162a..26268b49 100644 --- a/crates/sandstorm-core/src/node/mod.rs +++ b/crates/sandstorm-core/src/node/mod.rs @@ -615,6 +615,7 @@ async fn report_wakes( } }; let (free_memory_mib, pressure_some_avg10) = runtime::host_pressure(); + let (traffic, gateway_activity) = wake.take_traffic(false); let req = PollRequest { node_id: node_id.clone(), observed, @@ -623,7 +624,8 @@ async fn report_wakes( hold_ms: None, failures: Vec::new(), deferred: Vec::new(), - traffic: wake.take_traffic(), + traffic, + gateway_activity, evicted: wake.recently_evicted(Instant::now()), features: Vec::new(), tags: Vec::new(), @@ -632,17 +634,27 @@ async fn report_wakes( templates: None, pulls: Vec::new(), }; - let mut rb = client - .post(&url) - .timeout(Duration::from_secs(10)) - .json(&req); - if let Some(t) = api_token.as_deref().filter(|t| !t.is_empty()) { - rb = rb.bearer_auth(t); - } - if let Err(e) = rb.send().await.and_then(|r| r.error_for_status()) { - tracing::debug!(error = %e, "could not report a wake early; it goes with the next poll"); - } + send_wake_report(&client, &url, api_token.as_deref(), req, &wake).await; + } +} + +async fn send_wake_report( + client: &reqwest::Client, + url: &str, + api_token: Option<&str>, + req: PollRequest, + wake: &Wakeups, +) { + let mut request = client.post(url).timeout(Duration::from_secs(10)).json(&req); + if let Some(token) = api_token.filter(|t| !t.is_empty()) { + request = request.bearer_auth(token); + } + match request.send().await { + Ok(response) if response.status().is_success() => return, + Ok(response) => tracing::debug!(status = %response.status(), "early wake report refused"), + Err(error) => tracing::debug!(%error, "early wake report could not reach the master"), } + wake.retry_traffic(req.gateway_activity); } /// Returns the API token the master handed back, if any. @@ -757,6 +769,7 @@ async fn report( }) .collect(); + let (traffic, gateway_activity) = wake.take_traffic(true); Ok(PollRequest { node_id: cfg.node_id.clone(), observed, @@ -765,7 +778,8 @@ async fn report( hold_ms: hold.map(|h| h.as_millis() as u64), failures, deferred: Vec::new(), - traffic: wake.take_traffic(), + traffic, + gateway_activity, evicted: wake.recently_evicted(Instant::now()), features: cfg.features.clone(), tags: tags::held(&rt.tags_dir(), tags::REPORT_MIN_AGE), @@ -1459,6 +1473,7 @@ mod tests { #[cfg(test)] mod wake_tests { use super::*; + use crate::proto::GatewayActivity; use crate::tier::Tier; use std::path::{Path, PathBuf}; use std::sync::Mutex; @@ -1497,6 +1512,122 @@ mod wake_tests { } } + async fn report_traffic_after_early_wake(status: reqwest::StatusCode) -> PollRequest { + use axum::{routing::post, Json, Router}; + use tokio::sync::{mpsc, Notify}; + + let dir = tempfile::tempdir().unwrap(); + let (cfg, rt) = node(dir.path()); + let wake = Arc::new(Wakeups::new(|_, _, _| Ok(()))); + let mut gw = Gateway::waking(wake.clone()); + let failing = HashMap::new(); + let mut early = report(&cfg, &rt, &failing, &mut gw, &wake, None) + .await + .unwrap(); + wake.saw("repeat", "repeat-token"); + wake.saw("old-only", "old-token"); + wake.saw("recreated", "removed-token"); + (early.traffic, early.gateway_activity) = wake.take_traffic(false); + + let (send_received, mut received) = mpsc::channel(1); + let release = Arc::new(Notify::new()); + let server_release = release.clone(); + let app = Router::new().route( + "/v1/node/traffic", + post(move |Json(req): Json| { + let send_received = send_received.clone(); + let release = server_release.clone(); + async move { + send_received.send(req).await.unwrap(); + release.notified().await; + status + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/v1/node/traffic", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let client = reqwest::Client::builder().no_proxy().build().unwrap(); + let record_concurrent_activity = async { + let sent = tokio::time::timeout(Duration::from_secs(5), received.recv()) + .await + .expect("early wake report did not arrive") + .unwrap(); + wake.saw("repeat", "repeat-token"); + wake.saw("new-only", "new-token"); + wake.saw("recreated", "replacement-token"); + release.notify_one(); + sent + }; + + let (_, sent) = tokio::join!( + send_wake_report(&client, &url, None, early, &wake), + record_concurrent_activity, + ); + server.abort(); + assert_eq!(sent.traffic, ["old-only", "recreated", "repeat"]); + assert_activity( + &sent, + &[ + ("old-only", "old-token"), + ("recreated", "removed-token"), + ("repeat", "repeat-token"), + ], + ); + + let normal = report(&cfg, &rt, &failing, &mut gw, &wake, Some(Duration::ZERO)) + .await + .unwrap(); + let next = report(&cfg, &rt, &failing, &mut gw, &wake, Some(Duration::ZERO)) + .await + .unwrap(); + assert!(next.gateway_activity.is_empty()); + assert!(next.traffic.is_empty()); + normal + } + + fn assert_activity(report: &PollRequest, expected: &[(&str, &str)]) { + let mut expected: Vec<_> = expected + .iter() + .map(|(id, token)| GatewayActivity::new(id, token)) + .collect(); + expected.sort_unstable(); + let mut actual = report.gateway_activity.clone(); + actual.sort_unstable(); + assert_eq!(actual, expected); + } + + #[tokio::test] + async fn a_failed_early_wake_report_retries_without_losing_concurrent_activity() { + let normal = + report_traffic_after_early_wake(reqwest::StatusCode::SERVICE_UNAVAILABLE).await; + assert_eq!(normal.traffic, ["new-only", "recreated", "repeat"]); + assert_activity( + &normal, + &[ + ("new-only", "new-token"), + ("old-only", "old-token"), + ("recreated", "removed-token"), + ("recreated", "replacement-token"), + ("repeat", "repeat-token"), + ], + ); + } + + #[tokio::test] + async fn a_successful_early_wake_report_leaves_only_concurrent_activity() { + let normal = report_traffic_after_early_wake(reqwest::StatusCode::NO_CONTENT).await; + assert_eq!(normal.traffic, ["new-only", "recreated", "repeat"]); + assert_activity( + &normal, + &[ + ("new-only", "new-token"), + ("recreated", "replacement-token"), + ("repeat", "repeat-token"), + ], + ); + } + /// A demotion issued before the master heard about a request must not /// undo the wake that request caused. It is deferred, not failed: the /// failure book stays empty, so nothing shows up as an error and the @@ -1507,7 +1638,7 @@ mod wake_tests { let (cfg, rt) = node(dir.path()); let w = Arc::new(Wakeups::new(|_, _, _| Ok(()))); let mut failing = HashMap::new(); - w.saw("sb"); + w.saw("sb", "token"); for tier in [Tier::Frozen, Tier::Snapshot, Tier::Detached] { match run_one(&cfg, &rt, &mut failing, &w, &set_tier(tier), Instant::now()) { diff --git a/crates/sandstorm-core/src/node/wake.rs b/crates/sandstorm-core/src/node/wake.rs index 507652d9..9a0b771a 100644 --- a/crates/sandstorm-core/src/node/wake.rs +++ b/crates/sandstorm-core/src/node/wake.rs @@ -33,7 +33,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; -use crate::proto::ObservedSandbox; +use crate::proto::{GatewayActivity, ObservedSandbox}; use crate::tier::Tier; /// How long a sandbox that has just come up -- created, or restored for the @@ -110,7 +110,7 @@ pub struct Wakeups { /// When each sandbox last had a request through the gateway. seen: Mutex>, /// Requests the master has not been told about yet. - unreported: Mutex>, + unreported: Mutex, /// When each sandbox last served a request, or came up. Unlike `seen` /// it is kept for the sandbox's life: it is how a wake that needs /// memory picks which idle sandbox to put away. @@ -148,6 +148,12 @@ pub struct Wakeups { waiting: AtomicUsize, } +#[derive(Default)] +struct Traffic { + fresh: HashSet, + retry: HashSet, +} + impl Wakeups { /// `wake` brings a sandbox to Hot and returns when it is serving. It /// blocks, and is called with that sandbox's lock held and how long it @@ -196,8 +202,12 @@ impl Wakeups { } /// A request for this sandbox reached the gateway. - pub fn saw(&self, id: &str) { + pub fn saw(&self, id: &str, token: &str) { self.saw_at(id, Instant::now()); + self.unreported + .lock() + .fresh + .insert(GatewayActivity::new(id, token)); } pub(super) fn saw_at(&self, id: &str, now: Instant) { @@ -207,7 +217,6 @@ impl Wakeups { drop(demand); let prev = self.seen.lock().insert(id.to_string(), now); self.used.lock().insert(id.to_string(), now); - self.unreported.lock().insert(id.to_string()); // `seen` forgets anything older than GRACE, so no entry also means // a quiet spell. if prev.is_none_or(|p| now.saturating_duration_since(p) >= GRACE) { @@ -267,17 +276,32 @@ impl Wakeups { .filter(|d| !d.is_zero()) } - /// The sandboxes to report as used, emptied as it is read. - pub fn take_traffic(&self) -> Vec { + /// Drain fresh activity. Only a regular poll also takes failed early + /// reports, and only fresh activity goes into the legacy ID list. + pub fn take_traffic(&self, include_retries: bool) -> (Vec, Vec) { let now = Instant::now(); // Pruned here because this runs on every report: nothing outside // the grace window is consulted again. self.seen .lock() .retain(|_, at| now.saturating_duration_since(*at) < GRACE); - let mut ids: Vec = self.unreported.lock().drain().collect(); - ids.sort(); - ids + let (mut ids, mut activity) = { + let mut pending = self.unreported.lock(); + let ids: Vec<_> = pending.fresh.iter().map(|e| e.id.clone()).collect(); + let mut activity = std::mem::take(&mut pending.fresh); + if include_retries { + activity.extend(pending.retry.drain()); + } + (ids, activity.into_iter().collect::>()) + }; + ids.sort_unstable(); + ids.dedup(); + activity.sort_unstable(); + (ids, activity) + } + + pub fn retry_traffic(&self, activity: Vec) { + self.unreported.lock().retry.extend(activity); } /// A sandbox just woke for a request, or just answered one. It counts @@ -404,7 +428,10 @@ impl Wakeups { self.fresh.lock().remove(id); self.demand.lock().remove(id); self.evicted.lock().remove(id); - self.unreported.lock().remove(id); + let mut pending = self.unreported.lock(); + pending.fresh.retain(|e| e.id != id); + pending.retry.retain(|e| e.id != id); + drop(pending); self.arrivals.lock().remove(id); self.prefetched.lock().remove(id); } @@ -521,18 +548,27 @@ mod tests { #[test] fn traffic_is_reported_once() { let w = Wakeups::new(|_, _, _| Ok(())); - w.saw("a"); - w.saw("b"); - w.saw("a"); - assert_eq!(w.take_traffic(), vec!["a".to_string(), "b".to_string()]); - assert!(w.take_traffic().is_empty(), "already reported"); + w.saw("a", "token"); + w.saw("b", "token"); + w.saw("a", "token"); + assert_eq!( + w.take_traffic(true), + ( + vec!["a".to_string(), "b".to_string()], + vec![ + GatewayActivity::new("a", "token"), + GatewayActivity::new("b", "token") + ], + ) + ); + assert!(w.take_traffic(true).1.is_empty(), "already reported"); } #[test] fn grace_runs_out() { let w = Wakeups::new(|_, _, _| Ok(())); assert_eq!(w.in_grace("a", Instant::now()), None, "never seen"); - w.saw("a"); + w.saw("a", "token"); let left = w.in_grace("a", Instant::now()).expect("just seen"); assert!(left <= GRACE && left > GRACE - Duration::from_secs(1)); assert_eq!(w.in_grace("a", Instant::now() + GRACE), None); @@ -646,13 +682,13 @@ mod tests { #[test] fn forgetting_drops_everything() { let w = Wakeups::new(|_, _, _| Ok(())); - w.saw("a"); + w.saw("a", "token"); w.started("a"); w.forget("a"); assert!(w.demand.lock().is_empty()); assert!(w.fresh.lock().is_empty()); assert_eq!(w.in_grace("a", Instant::now()), None); - assert!(w.take_traffic().is_empty()); + assert!(w.take_traffic(true).1.is_empty()); } /// The point of the per-sandbox lock: a wake must not run runsc while diff --git a/crates/sandstorm-core/src/proto.rs b/crates/sandstorm-core/src/proto.rs index e259a531..0a373990 100644 --- a/crates/sandstorm-core/src/proto.rs +++ b/crates/sandstorm-core/src/proto.rs @@ -140,6 +140,11 @@ pub struct PollRequest { /// does not put it straight back to sleep. #[serde(default)] pub traffic: Vec, + /// Gateway activity bound to the instance that received the request. + /// Retries use only this field so an older master cannot apply them + /// to a replacement that has the same sandbox ID. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub gateway_activity: Vec, /// Sandboxes this node put to disk on its own, to make room for a wake. /// /// The master takes the node's word for these and moves desired state @@ -264,6 +269,28 @@ pub struct CommandFailure { pub held_back: bool, } +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct GatewayActivity { + pub id: String, + /// The creation token identifies the instance. Send a digest so this + /// report does not carry a credential that can access the sandbox. + pub token_hash: String, +} + +impl GatewayActivity { + pub fn new(id: &str, token: &str) -> Self { + Self { + id: id.to_string(), + token_hash: Self::hash_token(token), + } + } + + pub(crate) fn hash_token(token: &str) -> String { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(token.as_bytes())) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ObservedSandbox { pub id: String, diff --git a/crates/sandstorm-core/tests/listen_mode.rs b/crates/sandstorm-core/tests/listen_mode.rs index f9c47295..3119c584 100644 --- a/crates/sandstorm-core/tests/listen_mode.rs +++ b/crates/sandstorm-core/tests/listen_mode.rs @@ -115,6 +115,7 @@ async fn fake_sync( failures: vec![], deferred: vec![], traffic: vec![], + gateway_activity: vec![], evicted: vec![], features: vec![], tags: vec![], diff --git a/crates/sandstorm-core/tests/master_api.rs b/crates/sandstorm-core/tests/master_api.rs index 637b8e41..7ddf1abf 100644 --- a/crates/sandstorm-core/tests/master_api.rs +++ b/crates/sandstorm-core/tests/master_api.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use sandstorm_core::identity::Ca; use sandstorm_core::master::{router, MasterState}; use sandstorm_core::proto::{ - Command, Egress, ImageConfig, ObservedSandbox, PollResponse, TemplatePull, + Command, Egress, GatewayActivity, ImageConfig, ObservedSandbox, PollResponse, TemplatePull, }; use sandstorm_core::state::Store; use sandstorm_core::tier::Tier; @@ -2839,6 +2839,110 @@ async fn traffic_at_the_gateway_keeps_a_woken_sandbox_awake() { assert_eq!(tier().await, "frozen"); } +/// A retry must follow the sandbox instance whose gateway saw the request. +#[tokio::test] +async fn gateway_activity_cannot_wake_a_replacement_with_the_same_id() { + let h = start().await; + let node = "bm-1"; + let id = "reused-id"; + register(&h, node, "default").await; + put_template(&h, "python").await; + let c = reqwest::Client::new(); + let url = format!("{}/v1/sandboxes/{id}", h.base); + let create = || async { + let created = c + .post(format!("{}/v1/sandboxes", h.base)) + .json(&serde_json::json!({ "template": "python", "id": id, "node_id": node })) + .send() + .await + .unwrap(); + assert_eq!(created.status(), 201); + let body: serde_json::Value = created.json().await.unwrap(); + assert_eq!(body["id"], id); + assert_eq!(body["node_id"], node); + body["token"].as_str().unwrap().to_owned() + }; + let old_token = create().await; + assert_eq!(c.delete(&url).send().await.unwrap().status(), 204); + let current_token = create().await; + assert_ne!(old_token, current_token); + let resp = c + .post(format!("{url}/tier")) + .json(&serde_json::json!({ "tier": "snapshot" })) + .send() + .await + .unwrap(); + assert!(resp.status().is_success()); + + let db = rusqlite::Connection::open(h._td.path().join("state.db")).unwrap(); + db.execute("UPDATE sandboxes SET last_active_at=1 WHERE id=?1", [id]) + .unwrap(); + let poll = |activity: GatewayActivity, legacy: &[&str]| { + let body = serde_json::json!({ + "node_id": node, + "observed": [ObservedSandbox { + id: id.to_owned(), + tier: Tier::Hot, + serving: true, + ip: None, + net_index: None, + egress: None, + }], + "free_memory_mib": 2000, + "pressure_some_avg10": 0.0, + "hold_ms": 0, + "traffic": legacy, + "gateway_activity": [activity], + }); + let request = c.post(format!("{}/v1/node/poll", h.base)).json(&body); + async move { + request + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json::() + .await + .unwrap() + } + }; + let inspect = || async { + let sb: serde_json::Value = c + .get(&url) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + sb + }; + + // The typed entry must override its mirrored ID-only entry, even when + // the node now observes the replacement hot. + let stale = poll(GatewayActivity::new(id, &old_token), &[id]).await; + let replacement = inspect().await; + assert_eq!(replacement["tier"], "snapshot"); + assert_eq!(replacement["last_active_at"], 1); + assert!( + stale.commands.iter().any(|command| matches!( + command, + Command::SetTier { id: command_id, tier: Tier::Snapshot } if command_id == id + )), + "got {:?}", + stale.commands + ); + + let current = poll(GatewayActivity::new(id, ¤t_token), &[]).await; + let replacement = inspect().await; + assert_eq!(replacement["tier"], "hot"); + assert!(replacement["last_active_at"].as_i64().unwrap() > 1); + assert!(current.commands.is_empty(), "got {:?}", current.commands); +} + /// A polling node reports a wake out of turn, so the master's view is right /// within a round trip rather than a whole hold later. #[tokio::test]