Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ jobs:
run: npm test
- name: test the installer's backend choice
run: sh packaging/tests/install_backend.sh
- name: test fleet script HTTP errors
run: python3 scripts/tests/test_fleet_http.py
- name: cargo fmt
run: cargo fmt --all -- --check
- name: cargo clippy
Expand Down
51 changes: 38 additions & 13 deletions scripts/fleet.sh
Original file line number Diff line number Diff line change
Expand Up @@ -155,24 +155,42 @@ snapshot_host() {

getval() { sed -n "s/^$2=//p" "$1" | head -1; }

# curl the master, from the master, over loopback. The workstation may
# curl the master, from the master. The workstation may
# have no route to the control plane -- a fleet behind NAT is the normal
# case -- and nothing here needs one.
mapi() {
local method="$1" path="$2" body="${3:-}"
# Built as one string for the REMOTE shell to parse, with the quotes
# the remote needs. An array expanded with ${a[*]} would arrive as
# bare words and "Bearer <token>" would become two arguments.
local cmd
cmd="sudo curl -sS --cacert /var/lib/sandstorm/pki/ca.pem"
local cmd response code
cmd="sudo curl -sS --write-out '\n%{http_code}' --cacert /var/lib/sandstorm/pki/ca.pem"
cmd="$cmd -H \"authorization: Bearer \$(sudo cat /var/lib/sandstorm/pki/api-token)\""
cmd="$cmd -H 'content-type: application/json' -X $method"
[ -n "$body" ] && cmd="$cmd --data-raw '$body'"
# The bind address, not loopback: the master listens on one interface
# by default now, so 127.0.0.1 answers nothing. Its certificate carries
# the bind address as a SAN, so this verifies.
cmd="$cmd 'https://${MASTER_BIND:-127.0.0.1}:$MASTER_PORT$path'"
sshq "$MASTER_HOST" "$cmd"
if ! response=$(sshq "$MASTER_HOST" "$cmd"); then
printf '%s %s: could not complete the request\n' "$method" "$path" >&2
return 1
fi
# Strip only the status curl appended. Leave the body on stdout for
# successful callers, including an empty body after HTTP 204.
code=${response##*$'\n'}
case "$response" in
*$'\n'[0-9][0-9][0-9]) response=${response%$'\n'*} ;;
*) printf '%s %s: missing HTTP status\n' "$method" "$path" >&2; return 1 ;;
esac
case "$code" in
2[0-9][0-9]) printf '%s' "$response" ;;
*)
printf '%s %s: HTTP %s\n' "$method" "$path" "$code" >&2
[ -z "$response" ] || printf '%.200s\n' "$response" >&2
return 1
;;
esac
}

preflight() {
Expand Down Expand Up @@ -557,7 +575,7 @@ EOF
say "registering it on the master"
local body
body="{\"name\":\"$NAME\",\"rootfs\":\"rootfs\",\"spec\":{\"args\":[\"/usr/bin/python3\",\"/app/s.py\"],\"memory_max_mib\":$MEM}}"
if mapi POST /v1/templates "$body" >/dev/null 2>&1; then
if mapi POST /v1/templates "$body" >/dev/null; then
ok "template '$NAME' registered"
say "now: python3 bench/fleet/loadtest.py --master https://$MASTER_ADDR:$PORT --ca $CA \\"
echo " --token-file <(printf %s \"\$(sed -n s/^TOKEN=//p $STATE)\") --template $NAME --total 50"
Expand Down Expand Up @@ -587,25 +605,32 @@ cmd_verify() {

say "creating one '$NAME' sandbox and waiting for the node to start it"
local made id
made=$(mapi POST /v1/sandboxes "{\"template\":\"$NAME\",\"pool\":\"$POOL\"}")
made=$(mapi POST /v1/sandboxes "{\"template\":\"$NAME\",\"pool\":\"$POOL\"}") \
|| die "the master would not create a probe sandbox"
id=$(printf '%s' "$made" | python3 -c 'import json,sys
try: print(json.load(sys.stdin)["id"])
except Exception: pass' 2>/dev/null)
[ -n "$id" ] || die "the master would not create one: $(printf '%s' "$made" | head -c 200)"
sid = json.load(sys.stdin)["id"]
if isinstance(sid, str): print(sid)' 2>/dev/null)
[ -n "$id" ] || die "the master did not return a probe sandbox ID"

local i state
local i state reply
for i in $(seq 1 30); do
sleep 2
state=$(mapi GET "/v1/sandboxes/$id" | python3 -c 'import json,sys
if ! reply=$(mapi GET "/v1/sandboxes/$id") ||
! state=$(printf '%s' "$reply" | python3 -c 'import json,sys
s = json.load(sys.stdin)
if s.get("ip"): print("up|" + s["ip"])
elif s.get("last_error"): print("failed|" + s["last_error"].split(chr(10))[0])
else: print("pending|")' 2>/dev/null)
else: print("pending|")' 2>/dev/null); then
state="check-error"
break
fi
case "$state" in up\|*|failed\|*) break;; esac
done

mapi DELETE "/v1/sandboxes/$id" >/dev/null 2>&1
mapi DELETE "/v1/sandboxes/$id" >/dev/null \
|| die "could not delete probe sandbox $id"
case "$state" in
check-error) die "could not check probe sandbox $id" ;;
up\|*)
ok "a sandbox ran: ${state#up|}"
say "the fleet can actually do work"
Expand Down
176 changes: 176 additions & 0 deletions scripts/tests/test_fleet_http.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
#!/usr/bin/env python3
"""Run fleet.sh against a loopback master without SSHing to any host."""

import http.server
import json
import os
import subprocess
import tempfile
import threading
import unittest
from pathlib import Path


FLEET = Path(__file__).resolve().parents[1] / "fleet.sh"
PROBE = "/v1/sandboxes/test-sandbox"
SSH = """#!/usr/bin/env python3
import os
import shlex
import subprocess
import sys
from urllib.parse import urlsplit

host, command = sys.argv[-2:]
if (host, command) == ("test-node", "sudo bash -s"):
# Accept the rootfs install script without executing it.
sys.stdin.read()
sys.exit(0)
if host != "test-master":
sys.exit(f"unexpected SSH target: {host}")

args = shlex.split(command)
if args[:2] != ["sudo", "curl"]:
sys.exit("unexpected SSH command")
args = args[2:]
url = urlsplit(args.pop())
if url.scheme != "https" or url.hostname != "127.0.0.1" or url.port != int(os.environ["FLEET_TEST_PORT"]):
sys.exit("unexpected master URL")

# Adapt only TLS and authentication for the fixture. Let real curl process
# the production status-handling flags, headers and body.
cert = args.index("--cacert")
del args[cert:cert + 2]
auth = args.index("authorization: Bearer $(sudo cat /var/lib/sandstorm/pki/api-token)")
args[auth] = "authorization: Bearer test-token"
url = url._replace(scheme="http").geturl()
sys.exit(subprocess.run(["curl", "--disable", *args, url]).returncode)
"""


class FleetHttpTest(unittest.TestCase):
def run_fleet(self, command, responses):
observed = []

class Master(http.server.BaseHTTPRequestHandler):
def respond(self):
request = (self.command, self.path)
length = int(self.headers.get("Content-Length", 0))
self.rfile.read(length)
observed.append(request)
status, body = responses.get(request, (500, {"error": "unexpected request"}))
self.send_response(status)
if body is not None:
self.send_header("Content-Type", "application/json")
self.end_headers()
if body is not None:
self.wfile.write(json.dumps(body).encode())

do_POST = do_GET = do_DELETE = respond

def log_message(self, *_args):
pass

with tempfile.TemporaryDirectory() as tmp, http.server.ThreadingHTTPServer(
("127.0.0.1", 0), Master
) as master:
directory = Path(tmp)
ssh = directory / "ssh"
ssh.write_text(SSH)
ssh.chmod(0o755)
sleep = directory / "sleep"
sleep.write_text("#!/bin/sh\nexit 0\n")
sleep.chmod(0o755)
(directory / "fleet.env").write_text(
"MASTER=test-master\n"
"MASTER_ADDR=test-master\n"
"BIND_ADDR=127.0.0.1\n"
f"PORT={master.server_port}\n"
"NODES=test-node\n"
"POOL=default\n"
"CA=test-ca.pem\n"
)
thread = threading.Thread(
target=master.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True
)
thread.start()
try:
result = subprocess.run(
["bash", str(FLEET), command],
env={
**os.environ,
"PATH": f"{directory}{os.pathsep}{os.environ['PATH']}",
"no_proxy": "127.0.0.1",
"NO_PROXY": "127.0.0.1",
"SANDSTORM_FLEET_STATE": tmp,
"FLEET_TEST_PORT": str(master.server_port),
},
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=10,
)
finally:
master.shutdown()
thread.join()

self.assertEqual(observed, list(responses))
return result

def test_seed_reports_template_registration_conflict(self):
result = self.run_fleet(
"seed", {("POST", "/v1/templates"): (409, {"error": "template already exists"})}
)
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("the master refused the template", result.stdout)
self.assertIn("HTTP 409", result.stdout)
self.assertIn("template already exists", result.stdout)
self.assertNotIn("template 'py' registered", result.stdout)

def test_seed_reports_successful_registration(self):
result = self.run_fleet("seed", {("POST", "/v1/templates"): (201, None)})
self.assertEqual(result.returncode, 0, result.stdout)
self.assertIn("template 'py' registered", result.stdout)

def test_verify_reports_probe_cleanup_failure(self):
result = self.run_verify(500, {"error": "probe deletion unavailable"})
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("could not delete probe sandbox test-sandbox", result.stdout)
self.assertIn("HTTP 500", result.stdout)
self.assertIn("probe deletion unavailable", result.stdout)

def test_verify_cleans_up_after_probe_status_failure(self):
result = self.run_verify(
204, None, status_reply=(500, {"error": "probe status unavailable"})
)
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("could not check probe sandbox test-sandbox", result.stdout)
self.assertIn("HTTP 500", result.stdout)
self.assertIn("probe status unavailable", result.stdout)

def test_verify_reports_probe_creation_refusal(self):
result = self.run_fleet(
"verify", {("POST", "/v1/sandboxes"): (409, {"error": "node cannot admit work"})}
)
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("the master would not create a probe sandbox", result.stdout)
self.assertIn("HTTP 409", result.stdout)
self.assertIn("node cannot admit work", result.stdout)

def test_verify_reports_success_after_probe_cleanup(self):
result = self.run_verify(204, None)
self.assertEqual(result.returncode, 0, result.stdout)
self.assertIn("the fleet can actually do work", result.stdout)

def run_verify(self, status, body, *, status_reply=(200, {"ip": "192.0.2.2"})):
return self.run_fleet(
"verify",
{
("POST", "/v1/sandboxes"): (201, {"id": "test-sandbox"}),
("GET", PROBE): status_reply,
("DELETE", PROBE): (status, body),
},
)


if __name__ == "__main__":
unittest.main()
Loading