diff --git a/crates/sandstorm-core/src/master/api.rs b/crates/sandstorm-core/src/master/api.rs index 4fe6e163..a9c58a3d 100644 --- a/crates/sandstorm-core/src/master/api.rs +++ b/crates/sandstorm-core/src/master/api.rs @@ -1928,7 +1928,7 @@ async fn run_exec( args: Vec, timeout_secs: u64, ) -> Result { - let (node_id, tier) = { + let node_id = { let store = st.store(); let Some(sb) = store .get_sandbox(id) @@ -1936,17 +1936,24 @@ async fn run_exec( else { return Err(err(StatusCode::NOT_FOUND, "no such sandbox")); }; - (sb.node_id, sb.tier) + // A command can only run in a sandbox that is running. Promoting first + // would hide a multi-second wait inside what looks like a quick call, + // so say so instead and let the caller decide. + if sb.tier != Tier::Hot { + return Err(err( + StatusCode::CONFLICT, + format!( + "sandbox is {}; POST /v1/sandboxes/{id}/touch to bring it hot first", + sb.tier + ), + )); + } + // Keep the hot check and touch atomic with respect to the idle sweep. + store + .touch(id) + .map_err(|e| err(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + sb.node_id }; - // A command can only run in a sandbox that is running. Promoting first - // would hide a multi-second wait inside what looks like a quick call, - // so say so instead and let the caller decide. - if tier != Tier::Hot { - return Err(err( - StatusCode::CONFLICT, - format!("sandbox is {tier}; POST /v1/sandboxes/{id}/touch to bring it hot first"), - )); - } let exec_id = random_token(); let (tx, rx) = oneshot::channel(); @@ -1968,7 +1975,14 @@ async fn run_exec( // The node's own timeout plus room for a round trip and a loaded node. let budget = Duration::from_secs(timeout_secs.saturating_add(20)); match tokio::time::timeout(budget, rx).await { - Ok(Ok(r)) => Ok(r), + Ok(Ok(r)) => { + // Idle starts at completion, including non-zero exits. Only touch: + // a manual freeze while this command ran must remain in force. + st.store() + .touch(id) + .map_err(|e| err(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + Ok(r) + } // Sender dropped, or nothing came back in time. Either way the // caller gets an answer rather than hanging. _ => { diff --git a/crates/sandstorm-core/src/master/demote.rs b/crates/sandstorm-core/src/master/demote.rs index e6ba62ec..b9cc98cc 100644 --- a/crates/sandstorm-core/src/master/demote.rs +++ b/crates/sandstorm-core/src/master/demote.rs @@ -58,3 +58,240 @@ fn sweep(st: &Arc) -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use axum::Router; + use futures_util::FutureExt; + use http_body_util::BodyExt; + use rusqlite::Connection; + use serde_json::{json, Value}; + use tower::ServiceExt; + + use crate::identity::Ca; + use crate::master::api::{deliver_result, take_execs}; + use crate::master::{router, MasterOptions, SANDBOX_TOKEN_HEADER}; + use crate::proto::{Command, CommandResult}; + use crate::state::{now_secs, Sandbox, Store}; + use crate::tier::{IdlePolicy, Tier}; + + struct Harness { + st: Arc, + app: Router, + db: Connection, + _dir: tempfile::TempDir, + } + + impl Harness { + fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.db"); + let store = Store::open(&path).unwrap(); + let now = now_secs(); + store + .insert_sandbox(&Sandbox { + id: "sb1".into(), + node_id: "node1".into(), + pool: "default".into(), + template: "python".into(), + tier: Tier::Hot, + token: "sandbox-token".into(), + volume: None, + port: None, + gateway_port: None, + created_at: now, + last_active_at: now, + last_error: None, + error_attempts: 0, + net_index: None, + ip: None, + from_tag: None, + memory_limit_mib: None, + egress: Default::default(), + }) + .unwrap(); + let st = Arc::new(MasterState::new( + store, + Ca::open_or_create(dir.path().join("pki")).unwrap(), + MasterOptions { + idle: IdlePolicy { + to_frozen: Duration::from_secs(100), + to_snapshot: Duration::from_secs(200), + to_detached: Duration::from_secs(300), + }, + api_token: Some("api-token".into()), + ..Default::default() + }, + )); + Self { + app: router(st.clone()), + st, + db: Connection::open(path).unwrap(), + _dir: dir, + } + } + + fn sandbox(&self) -> Sandbox { + self.st.store().get_sandbox("sb1").unwrap().unwrap() + } + + fn age_by(&self, seconds: i64) { + // Move stored times together so elapsed idle accumulates unless a request touches it. + self.db + .execute( + "UPDATE sandboxes SET created_at=created_at-?1, last_active_at=last_active_at-?1", + [seconds], + ) + .unwrap(); + } + + fn sweep_to(&self, tier: Tier) { + sweep(&self.st).unwrap(); + assert_eq!(self.sandbox().tier, tier); + } + + fn exec_id(&self) -> String { + let commands = take_execs(&self.st, "node1"); + assert_eq!(commands.len(), 1); + match commands.into_iter().next().unwrap() { + Command::Exec { exec_id, .. } => exec_id, + other => panic!("expected Exec, got {other:?}"), + } + } + + fn complete(&self, result_id: String, exit_code: i64) { + assert!(deliver_result( + &self.st, + CommandResult { + result_id, + stdout: "done".into(), + stderr: "failed".into(), + exit_code, + size_bytes: 0, + }, + )); + } + + async fn rejected(&self, request: Request, status: StatusCode) { + let last_active = self.sandbox().last_active_at; + let response = self.app.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), status); + assert_eq!(self.sandbox().last_active_at, last_active); + assert!(take_execs(&self.st, "node1").is_empty()); + } + } + + fn request(endpoint: &str) -> Request { + let body = match endpoint { + "exec" => json!({"args": ["false"]}), + "eval" => json!({"code": "raise RuntimeError()"}), + "tier" => json!({"tier": "frozen"}), + _ => panic!("unknown endpoint {endpoint}"), + }; + Request::post(format!("/v1/sandboxes/sb1/{endpoint}")) + .header("content-type", "application/json") + .header("authorization", "Bearer api-token") + .header(SANDBOX_TOKEN_HEADER, "sandbox-token") + .body(Body::from(body.to_string())) + .unwrap() + } + + async fn continuous_calls_keep_hot(endpoint: &str) { + let h = Harness::new(); + for exit_code in [0, 1, 0] { + h.age_by(20); + h.sweep_to(Tier::Hot); + let mut response = Box::pin(h.app.clone().oneshot(request(endpoint))); + assert!(response.as_mut().now_or_never().is_none()); + let exec_id = h.exec_id(); + h.age_by(30); + h.sweep_to(Tier::Hot); + h.complete(exec_id, exit_code); + let response = response.await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let result: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(result["exit_code"], exit_code); + } + assert!(now_secs() - h.sandbox().created_at >= 150); + + // Idle starts at completion, and a new execution refreshes it before waiting. + h.age_by(90); + h.sweep_to(Tier::Hot); + let mut response = Box::pin(h.app.clone().oneshot(request(endpoint))); + assert!(response.as_mut().now_or_never().is_none()); + let exec_id = h.exec_id(); + h.age_by(20); + h.sweep_to(Tier::Hot); + h.complete(exec_id, 1); + assert_eq!(response.await.unwrap().status(), StatusCode::OK); + + h.age_by(90); + h.sweep_to(Tier::Hot); + h.age_by(20); + h.sweep_to(Tier::Frozen); + h.age_by(100); + h.sweep_to(Tier::Snapshot); + h.age_by(100); + h.sweep_to(Tier::Detached); + } + + #[tokio::test] + async fn continuous_exec_stays_hot_then_demotes_after_the_last_result() { + continuous_calls_keep_hot("exec").await; + } + + #[tokio::test] + async fn continuous_eval_stays_hot_then_demotes_after_the_last_result() { + continuous_calls_keep_hot("eval").await; + } + + #[tokio::test] + async fn rejected_exec_and_eval_do_not_extend_idle() { + for endpoint in ["exec", "eval"] { + let h = Harness::new(); + h.age_by(90); + let mut unauthenticated = request(endpoint); + unauthenticated.headers_mut().remove("authorization"); + h.rejected(unauthenticated, StatusCode::UNAUTHORIZED).await; + + let mut wrong_sandbox_token = request(endpoint); + wrong_sandbox_token + .headers_mut() + .remove(SANDBOX_TOKEN_HEADER); + h.rejected(wrong_sandbox_token, StatusCode::FORBIDDEN).await; + + let mut invalid = request(endpoint); + *invalid.body_mut() = Body::from("{}"); + h.rejected(invalid, StatusCode::UNPROCESSABLE_ENTITY).await; + if endpoint == "exec" { + let mut empty = request(endpoint); + *empty.body_mut() = Body::from(r#"{"args":[]}"#); + h.rejected(empty, StatusCode::BAD_REQUEST).await; + } + + h.age_by(20); + h.sweep_to(Tier::Frozen); + h.rejected(request(endpoint), StatusCode::CONFLICT).await; + assert_eq!(h.sandbox().tier, Tier::Frozen); + } + } + + #[tokio::test] + async fn late_exec_and_eval_results_do_not_undo_a_manual_freeze() { + for endpoint in ["exec", "eval"] { + let h = Harness::new(); + let mut response = Box::pin(h.app.clone().oneshot(request(endpoint))); + assert!(response.as_mut().now_or_never().is_none()); + let exec_id = h.exec_id(); + let frozen = h.app.clone().oneshot(request("tier")).await.unwrap(); + assert_eq!(frozen.status(), StatusCode::OK); + h.complete(exec_id, 0); + assert_eq!(response.await.unwrap().status(), StatusCode::OK); + assert_eq!(h.sandbox().tier, Tier::Frozen); + } + } +}