From 5630fc0d9d021b8bdc4476e92460aa0105345d8c Mon Sep 17 00:00:00 2001 From: dada-yan Date: Sat, 14 Nov 2026 16:15:09 +1100 Subject: [PATCH] fix(fleet): fail seed and verify on HTTP errors Signed-off-by: dada-yan --- .github/workflows/ci.yml | 2 + scripts/fleet.sh | 51 ++++++--- scripts/tests/test_fleet_http.py | 176 +++++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 scripts/tests/test_fleet_http.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b62b1886..cfc44757 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,8 @@ jobs: run: npm test - name: test the installer's backend choice run: sh packaging/tests/install_backend.sh + - name: test fleet script HTTP errors + run: python3 scripts/tests/test_fleet_http.py - name: cargo fmt run: cargo fmt --all -- --check - name: cargo clippy diff --git a/scripts/fleet.sh b/scripts/fleet.sh index 35d4e14e..89751c7e 100755 --- a/scripts/fleet.sh +++ b/scripts/fleet.sh @@ -155,7 +155,7 @@ snapshot_host() { getval() { sed -n "s/^$2=//p" "$1" | head -1; } -# curl the master, from the master, over loopback. The workstation may +# curl the master, from the master. The workstation may # have no route to the control plane -- a fleet behind NAT is the normal # case -- and nothing here needs one. mapi() { @@ -163,8 +163,8 @@ mapi() { # Built as one string for the REMOTE shell to parse, with the quotes # the remote needs. An array expanded with ${a[*]} would arrive as # bare words and "Bearer " would become two arguments. - local cmd - cmd="sudo curl -sS --cacert /var/lib/sandstorm/pki/ca.pem" + local cmd response code + cmd="sudo curl -sS --write-out '\n%{http_code}' --cacert /var/lib/sandstorm/pki/ca.pem" cmd="$cmd -H \"authorization: Bearer \$(sudo cat /var/lib/sandstorm/pki/api-token)\"" cmd="$cmd -H 'content-type: application/json' -X $method" [ -n "$body" ] && cmd="$cmd --data-raw '$body'" @@ -172,7 +172,25 @@ mapi() { # by default now, so 127.0.0.1 answers nothing. Its certificate carries # the bind address as a SAN, so this verifies. cmd="$cmd 'https://${MASTER_BIND:-127.0.0.1}:$MASTER_PORT$path'" - sshq "$MASTER_HOST" "$cmd" + if ! response=$(sshq "$MASTER_HOST" "$cmd"); then + printf '%s %s: could not complete the request\n' "$method" "$path" >&2 + return 1 + fi + # Strip only the status curl appended. Leave the body on stdout for + # successful callers, including an empty body after HTTP 204. + code=${response##*$'\n'} + case "$response" in + *$'\n'[0-9][0-9][0-9]) response=${response%$'\n'*} ;; + *) printf '%s %s: missing HTTP status\n' "$method" "$path" >&2; return 1 ;; + esac + case "$code" in + 2[0-9][0-9]) printf '%s' "$response" ;; + *) + printf '%s %s: HTTP %s\n' "$method" "$path" "$code" >&2 + [ -z "$response" ] || printf '%.200s\n' "$response" >&2 + return 1 + ;; + esac } preflight() { @@ -557,7 +575,7 @@ EOF say "registering it on the master" local body body="{\"name\":\"$NAME\",\"rootfs\":\"rootfs\",\"spec\":{\"args\":[\"/usr/bin/python3\",\"/app/s.py\"],\"memory_max_mib\":$MEM}}" - if mapi POST /v1/templates "$body" >/dev/null 2>&1; then + if mapi POST /v1/templates "$body" >/dev/null; then ok "template '$NAME' registered" say "now: python3 bench/fleet/loadtest.py --master https://$MASTER_ADDR:$PORT --ca $CA \\" echo " --token-file <(printf %s \"\$(sed -n s/^TOKEN=//p $STATE)\") --template $NAME --total 50" @@ -587,25 +605,32 @@ cmd_verify() { say "creating one '$NAME' sandbox and waiting for the node to start it" local made id - made=$(mapi POST /v1/sandboxes "{\"template\":\"$NAME\",\"pool\":\"$POOL\"}") + made=$(mapi POST /v1/sandboxes "{\"template\":\"$NAME\",\"pool\":\"$POOL\"}") \ + || die "the master would not create a probe sandbox" id=$(printf '%s' "$made" | python3 -c 'import json,sys -try: print(json.load(sys.stdin)["id"]) -except Exception: pass' 2>/dev/null) - [ -n "$id" ] || die "the master would not create one: $(printf '%s' "$made" | head -c 200)" +sid = json.load(sys.stdin)["id"] +if isinstance(sid, str): print(sid)' 2>/dev/null) + [ -n "$id" ] || die "the master did not return a probe sandbox ID" - local i state + local i state reply for i in $(seq 1 30); do sleep 2 - state=$(mapi GET "/v1/sandboxes/$id" | python3 -c 'import json,sys + if ! reply=$(mapi GET "/v1/sandboxes/$id") || + ! state=$(printf '%s' "$reply" | python3 -c 'import json,sys s = json.load(sys.stdin) if s.get("ip"): print("up|" + s["ip"]) elif s.get("last_error"): print("failed|" + s["last_error"].split(chr(10))[0]) -else: print("pending|")' 2>/dev/null) +else: print("pending|")' 2>/dev/null); then + state="check-error" + break + fi case "$state" in up\|*|failed\|*) break;; esac done - mapi DELETE "/v1/sandboxes/$id" >/dev/null 2>&1 + mapi DELETE "/v1/sandboxes/$id" >/dev/null \ + || die "could not delete probe sandbox $id" case "$state" in + check-error) die "could not check probe sandbox $id" ;; up\|*) ok "a sandbox ran: ${state#up|}" say "the fleet can actually do work" diff --git a/scripts/tests/test_fleet_http.py b/scripts/tests/test_fleet_http.py new file mode 100644 index 00000000..1d68237f --- /dev/null +++ b/scripts/tests/test_fleet_http.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Run fleet.sh against a loopback master without SSHing to any host.""" + +import http.server +import json +import os +import subprocess +import tempfile +import threading +import unittest +from pathlib import Path + + +FLEET = Path(__file__).resolve().parents[1] / "fleet.sh" +PROBE = "/v1/sandboxes/test-sandbox" +SSH = """#!/usr/bin/env python3 +import os +import shlex +import subprocess +import sys +from urllib.parse import urlsplit + +host, command = sys.argv[-2:] +if (host, command) == ("test-node", "sudo bash -s"): + # Accept the rootfs install script without executing it. + sys.stdin.read() + sys.exit(0) +if host != "test-master": + sys.exit(f"unexpected SSH target: {host}") + +args = shlex.split(command) +if args[:2] != ["sudo", "curl"]: + sys.exit("unexpected SSH command") +args = args[2:] +url = urlsplit(args.pop()) +if url.scheme != "https" or url.hostname != "127.0.0.1" or url.port != int(os.environ["FLEET_TEST_PORT"]): + sys.exit("unexpected master URL") + +# Adapt only TLS and authentication for the fixture. Let real curl process +# the production status-handling flags, headers and body. +cert = args.index("--cacert") +del args[cert:cert + 2] +auth = args.index("authorization: Bearer $(sudo cat /var/lib/sandstorm/pki/api-token)") +args[auth] = "authorization: Bearer test-token" +url = url._replace(scheme="http").geturl() +sys.exit(subprocess.run(["curl", "--disable", *args, url]).returncode) +""" + + +class FleetHttpTest(unittest.TestCase): + def run_fleet(self, command, responses): + observed = [] + + class Master(http.server.BaseHTTPRequestHandler): + def respond(self): + request = (self.command, self.path) + length = int(self.headers.get("Content-Length", 0)) + self.rfile.read(length) + observed.append(request) + status, body = responses.get(request, (500, {"error": "unexpected request"})) + self.send_response(status) + if body is not None: + self.send_header("Content-Type", "application/json") + self.end_headers() + if body is not None: + self.wfile.write(json.dumps(body).encode()) + + do_POST = do_GET = do_DELETE = respond + + def log_message(self, *_args): + pass + + with tempfile.TemporaryDirectory() as tmp, http.server.ThreadingHTTPServer( + ("127.0.0.1", 0), Master + ) as master: + directory = Path(tmp) + ssh = directory / "ssh" + ssh.write_text(SSH) + ssh.chmod(0o755) + sleep = directory / "sleep" + sleep.write_text("#!/bin/sh\nexit 0\n") + sleep.chmod(0o755) + (directory / "fleet.env").write_text( + "MASTER=test-master\n" + "MASTER_ADDR=test-master\n" + "BIND_ADDR=127.0.0.1\n" + f"PORT={master.server_port}\n" + "NODES=test-node\n" + "POOL=default\n" + "CA=test-ca.pem\n" + ) + thread = threading.Thread( + target=master.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + thread.start() + try: + result = subprocess.run( + ["bash", str(FLEET), command], + env={ + **os.environ, + "PATH": f"{directory}{os.pathsep}{os.environ['PATH']}", + "no_proxy": "127.0.0.1", + "NO_PROXY": "127.0.0.1", + "SANDSTORM_FLEET_STATE": tmp, + "FLEET_TEST_PORT": str(master.server_port), + }, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=10, + ) + finally: + master.shutdown() + thread.join() + + self.assertEqual(observed, list(responses)) + return result + + def test_seed_reports_template_registration_conflict(self): + result = self.run_fleet( + "seed", {("POST", "/v1/templates"): (409, {"error": "template already exists"})} + ) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("the master refused the template", result.stdout) + self.assertIn("HTTP 409", result.stdout) + self.assertIn("template already exists", result.stdout) + self.assertNotIn("template 'py' registered", result.stdout) + + def test_seed_reports_successful_registration(self): + result = self.run_fleet("seed", {("POST", "/v1/templates"): (201, None)}) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertIn("template 'py' registered", result.stdout) + + def test_verify_reports_probe_cleanup_failure(self): + result = self.run_verify(500, {"error": "probe deletion unavailable"}) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("could not delete probe sandbox test-sandbox", result.stdout) + self.assertIn("HTTP 500", result.stdout) + self.assertIn("probe deletion unavailable", result.stdout) + + def test_verify_cleans_up_after_probe_status_failure(self): + result = self.run_verify( + 204, None, status_reply=(500, {"error": "probe status unavailable"}) + ) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("could not check probe sandbox test-sandbox", result.stdout) + self.assertIn("HTTP 500", result.stdout) + self.assertIn("probe status unavailable", result.stdout) + + def test_verify_reports_probe_creation_refusal(self): + result = self.run_fleet( + "verify", {("POST", "/v1/sandboxes"): (409, {"error": "node cannot admit work"})} + ) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("the master would not create a probe sandbox", result.stdout) + self.assertIn("HTTP 409", result.stdout) + self.assertIn("node cannot admit work", result.stdout) + + def test_verify_reports_success_after_probe_cleanup(self): + result = self.run_verify(204, None) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertIn("the fleet can actually do work", result.stdout) + + def run_verify(self, status, body, *, status_reply=(200, {"ip": "192.0.2.2"})): + return self.run_fleet( + "verify", + { + ("POST", "/v1/sandboxes"): (201, {"id": "test-sandbox"}), + ("GET", PROBE): status_reply, + ("DELETE", PROBE): (status, body), + }, + ) + + +if __name__ == "__main__": + unittest.main()