diff --git a/scripts/agent.mjs b/scripts/agent.mjs index ca4e7bda731c..4071b6c06a3e 100755 --- a/scripts/agent.mjs +++ b/scripts/agent.mjs @@ -4,6 +4,7 @@ import { copyFileSync, existsSync, readFileSync, realpathSync } from "node:fs"; import { join } from "node:path"; +import { setTimeout as setTimeoutPromise } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { @@ -21,11 +22,13 @@ import { getKernel, getOs, homedir, + isLinux, isMacOS, isPosix, isWindows, mkdir, spawnSafe, + spawnSync, which, writeFile, } from "./utils.mjs"; @@ -105,6 +108,15 @@ async function doBuildkiteAgent(action, cliOptions = {}) { } if (isOpenRc()) { + // openrc starts a runlevel's unrelated services one at a time in name + // order, so without `after docker` this service ("b") starts, and the + // ephemeral agent acquires its job, before the boot has reached "docker"; + // the shard then begins with no daemon socket at all (routine on the + // alpine test agents, occasionally for minutes when a service in between + // is slow). `after` is ordering only: images without docker still boot + // the agent. Alpine's docker service returns as soon as dockerd is + // spawned, so start() below additionally waits for the daemon to answer. + // Both take effect when the images are next published. const servicePath = "/etc/init.d/buildkite-agent"; const service = `#!/sbin/openrc-run name="buildkite-agent" @@ -123,6 +135,7 @@ async function doBuildkiteAgent(action, cliOptions = {}) { depend() { need net use dns logger + after docker } `; writeFile(servicePath, service, { mode: 0o755 }); @@ -390,6 +403,19 @@ async function doBuildkiteAgent(action, cliOptions = {}) { .map(([key, value]) => `${key}=${value}`) .join(","); + // The job this machine was booted for starts the moment the agent + // registers (acquire-job above), and the Linux test jobs need dockerd. The + // init system only orders this service after docker's start script (see + // install()), which on alpine returns as soon as dockerd is spawned, so + // give the daemon a bounded head start here: registering later only + // delays the job, registering before the daemon listens starts the job + // against a dead socket (test/docker/index.ts has the history). Past the + // budget the agent registers anyway and the job's "--- Docker" section + // reports the daemon's state. + if (isLinux && which("docker")) { + await waitForDockerDaemon(2 * 60_000); + } + await spawnSafe( [ command, @@ -410,6 +436,29 @@ async function doBuildkiteAgent(action, cliOptions = {}) { } } +/** + * Polls `docker version` (which fails until the daemon is listening) once a + * second for up to `budgetMs`. Returns either way; the outcome is logged. + * @param {number} budgetMs + */ +async function waitForDockerDaemon(budgetMs) { + const startedAt = Date.now(); + const elapsed = () => `${Math.round((Date.now() - startedAt) / 1000)}s`; + const reachable = () => spawnSync(["docker", "version"], { timeout: 10_000 }).exitCode === 0; + if (reachable()) { + return; + } + console.log(`Docker daemon is not reachable yet; waiting up to ${Math.round(budgetMs / 1000)}s for it...`); + while (Date.now() - startedAt < budgetMs) { + await setTimeoutPromise(1_000); + if (reachable()) { + console.log(`Docker daemon became reachable after ${elapsed()}`); + return; + } + } + console.warn(`Docker daemon still unreachable after ${elapsed()}; starting the agent without it`); +} + /** * @returns {boolean} */ diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 335c181c8987..f3c795796e39 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -3160,7 +3160,29 @@ export function printEnvironment() { startGroup("Docker", () => { const shell = which(["sh", "bash"]); if (shell) { - spawnSync([shell, "-c", "docker ps"], { stdio: "inherit" }); + const { exitCode } = spawnSync([shell, "-c", "docker ps"], { stdio: "inherit" }); + if (exitCode !== 0) { + // The agent can start the job before the boot has brought dockerd + // up (scripts/agent.mjs orders and waits for it on images published + // after that change; test/docker/coordinator.ts waits for it on + // every image, so this is not fatal). On openrc the service table + // shows which boot step the daemon was still behind. + if (which("rc-status")) { + spawnSync([shell, "-c", "rc-status"], { stdio: "inherit" }); + } + // One line per affected job under a single per-build annotation, so + // how often this still happens can be read off builds (and compared + // across image publishes) without grepping job logs. Info style: + // nothing has failed, and `bun run ci:errors` lists only error-style + // annotations. + reportAnnotationToBuildKite({ + context: "docker-daemon-not-up-at-job-start", + label: "docker daemon not up at job start", + content: `- dockerd was not up when [${getBuildLabel()}](${getBuildUrl()}) started on \`${getHostname()}\`\n`, + style: "info", + priority: 1, + }); + } } }); } diff --git a/test/cli/install/bun-install-proxy.test.ts b/test/cli/install/bun-install-proxy.test.ts index 40672217c0ab..5642b30ec38e 100644 --- a/test/cli/install/bun-install-proxy.test.ts +++ b/test/cli/install/bun-install-proxy.test.ts @@ -1,43 +1,38 @@ import { beforeAll, it } from "bun:test"; -import { exec } from "child_process"; import { rm } from "fs/promises"; -import { bunEnv, bunExe, dockerExe, isDockerEnabled, tempDirWithFiles } from "harness"; +import { bunEnv, bunExe, isDockerEnabled, tempDirWithFiles } from "harness"; import { join } from "path"; -import { promisify } from "util"; -const execAsync = promisify(exec); -const dockerCLI = dockerExe() as string; -const SQUID_URL = "http://127.0.0.1:3128"; +import { ensure } from "../../docker/index.ts"; + +let SQUID_URL: string; if (isDockerEnabled()) { + // The compose `squid` service is the same ubuntu/squid image this file used + // to `docker run` itself on a fixed host port. Going through ensure() means + // the shard's coordinator starts it (and waits for the docker daemon if the + // job began before it was up) and it gets a free port like every other + // test service. beforeAll(async () => { - async function isSquidRunning() { - const text = await fetch(SQUID_URL) - .then(res => res.text()) - .catch(() => {}); - return text?.includes("squid") ?? false; - } - if (!(await isSquidRunning())) { - // try to create or error if is already created - await execAsync( - `${dockerCLI} run -d --name squid-container -e TZ=UTC -p 3128:3128 ubuntu/squid:5.2-22.04_beta`, - ).catch(() => {}); - - async function waitForSquid(max_wait = 60_000) { - const start = Date.now(); - while (true) { - if (await isSquidRunning()) { - return; - } - if (Date.now() - start > max_wait) { - throw new Error("Squid did not start in time"); - } - - await Bun.sleep(1000); - } - } - // wait for squid to start - await waitForSquid(); + const squid = await ensure("squid"); + SQUID_URL = `http://${squid.host}:${squid.ports[3128]}`; + // The container counts as healthy once the squid process exists. Like the + // isSquidRunning() loop this replaces, wait until it answers HTTP (a plain + // GET of the proxy itself gets squid's own error page) before installing + // through it. + const squidAnswers = async () => { + const body = await fetch(SQUID_URL, { signal: AbortSignal.timeout(5_000) }).then( + res => res.text(), + () => "", + ); + return body.includes("squid"); + }; + const deadline = Date.now() + 30_000; + while (!(await squidAnswers())) { + if (Date.now() > deadline) throw new Error(`squid at ${SQUID_URL} did not answer within 30s`); + await Bun.sleep(250); } - }); + // Same hook budget as describeWithContainer(): a cold start is a `compose + // build` plus `up --wait` with a 180s wait-timeout. + }, 240_000); it("bun install with proxy with big packages", async () => { const files = { diff --git a/test/docker/coordinator.ts b/test/docker/coordinator.ts index d189ccfa269c..5667b87cf380 100644 --- a/test/docker/coordinator.ts +++ b/test/docker/coordinator.ts @@ -17,13 +17,18 @@ * predicts which services argv's tests need and starts them at launch, so * they're healthy by the time the first test asks. * + * It is also the one place that waits for the docker daemon itself: the agent + * can start the shard while dockerd is still coming up, and every request + * (prestart or test) blocks on the shared wait below instead of failing on a + * one-shot probe. + * * Lifetime is tied to the runner: stdin is a pipe from it, and EOF means the * runner is gone. */ import { unlinkSync } from "node:fs"; import * as net from "node:net"; -import { ensure, type ServiceInfo, type ServiceName } from "./index.ts"; +import { ensure, waitForDockerDaemon, type ServiceInfo, type ServiceName } from "./index.ts"; import { prestartMap as prestartMapRaw } from "./prestart-map.mjs"; // Keys are paths relative to test/ — that's the shape runner.node.mjs passes @@ -44,6 +49,30 @@ if (!socketPath) { // environment. delete process.env.BUN_DOCKER_COORDINATOR; +// The shard may start before dockerd has finished coming up (see +// waitForDockerDaemon); start polling for it right away so the wait overlaps +// the non-container tests the runner schedules first. The window is well past +// the latest the daemon has been seen to appear in CI (up by ~7 minutes into +// the job); once it closes, requests fail immediately, so a host whose daemon +// never comes up costs the shard one window rather than a full wait per file +// attempt. The per-request cap is the budget scripts/runner.node.mjs gives the +// hook that is typically awaiting the request (--timeout = testTimeout / 2 on +// the release lanes; the ASAN lanes get three times that but run on systemd +// images, where the daemon has always been up at job start), so a request +// either gets the daemon or a clear error within the time its caller has +// anyway, and the 3-minute file budget keeps room for `compose up` and the +// tests after it. A file whose request hits the cap is retried by the runner, +// and the retry's request gets a fresh cap against the same shared wait, so a +// file's four attempts together cover ~6 minutes of daemon lateness from the +// file's first request (container-backed files are also scheduled late in the +// shard, see prestart-map.mjs). +const awaitDockerDaemon = waitForDockerDaemon({ + windowMs: 10 * 60_000, + pollMs: 1_000, + log: message => console.log(`coordinator: ${message}`), +}); +const DAEMON_WAIT_PER_REQUEST_MS = 90_000; + // Collapse concurrent requests for one service onto a single ensureServiceNow(), // and remember the last ServiceInfo a successful ensure produced. The full // ensure() path costs several serial `docker compose` spawns (build, ps -a, @@ -97,6 +126,7 @@ async function ensureServiceNow(service: ServiceName): Promise { console.log(`coordinator: ensuring ${service}`); try { + await awaitDockerDaemon(DAEMON_WAIT_PER_REQUEST_MS); const info = await ensure(service); console.log(`coordinator: ${service} ready`); lastGood.set(service, info); diff --git a/test/docker/index.ts b/test/docker/index.ts index 55a178f0f6ad..15361b35607b 100644 --- a/test/docker/index.ts +++ b/test/docker/index.ts @@ -125,6 +125,104 @@ function serviceFromEnv(service: ServiceName): ServiceInfo | null { return info; } +/** + * One `docker version` round trip. The CLI exits non-zero until the daemon + * accepts connections on its socket, so this is the daemon readiness signal. + * (`docker compose version` is not: compose is a CLI plugin and succeeds with + * no daemon at all, which is why scripts/runner.node.mjs can use it to decide + * whether to start the coordinator before the daemon is up.) A daemon that + * accepts the connection but never answers would otherwise hang the CLI, and + * with it every wait built on this probe, so the CLI is killed after + * `timeoutMs` and counted as unreachable; a healthy round trip takes well + * under a second. + */ +export async function isDockerDaemonReachable(timeoutMs = 10_000): Promise { + const proc = spawn({ + cmd: ["docker", "version"], + stdout: "ignore", + stderr: "ignore", + timeout: timeoutMs, + }); + return (await proc.exited) === 0; +} + +export interface DockerDaemonWaitOptions { + /** Stop polling, and fail every waiter, once the daemon has stayed unreachable this long. */ + windowMs: number; + /** Delay between probes. */ + pollMs: number; + /** Replaces the `docker version` probe; tests script it. */ + probe?: () => Promise; + /** Where the "waiting" / "reachable after Ns" / "giving up" lines go. */ + log?: (message: string) => void; +} + +/** + * Starts polling for the docker daemon and returns `waitFor(capMs)`, which + * resolves once a probe has succeeded and rejects if the daemon is still + * unreachable after `capMs`, or once `windowMs` has passed in total. Every + * caller shares the one poll loop: a rejected `waitFor` does not stop it, so a + * caller that retries later gets the daemon as soon as it is up, and callers + * arriving after the window has closed fail immediately. + * + * Linux CI agents can pick up a job before dockerd has finished starting. The + * openrc (alpine) images published before scripts/agent.mjs ordered the agent + * service after docker routinely start a shard with no /var/run/docker.sock + * yet; the daemon usually appears within ~10s, but in builds 97891, 98647, + * 98707 and 98746 it was still down 3-4.5 minutes into the job and up by 5-7 + * minutes, and the coordinator's one-shot probe failed every container-backed + * test file in those shards (all in-shard retries included, since they run + * within seconds of each other). The boot-ordering fix in agent.mjs is the + * primary fix once images are republished; this keeps shards green on the + * current images and covers whatever dockerd start-up time is left after it + * (the runner's `--- Docker` job section records how often that still + * happens). When a probe succeeds on the first try, which is the case on every + * systemd image, nothing is logged and `waitFor` costs nothing beyond that + * probe. + */ +export function waitForDockerDaemon({ + windowMs, + pollMs, + probe = isDockerDaemonReachable, + log = console.log, +}: DockerDaemonWaitOptions): (capMs: number) => Promise { + const startedAt = Date.now(); + const unreachableFor = () => `${Math.round((Date.now() - startedAt) / 1000)}s`; + + const ready: Promise = (async () => { + if (await probe()) return; + log(`docker daemon is not reachable yet; waiting up to ${Math.round(windowMs / 1000)}s for it to come up`); + while (Date.now() - startedAt < windowMs) { + await Bun.sleep(pollMs); + if (await probe()) { + log(`docker daemon became reachable after ${unreachableFor()}`); + return; + } + } + const message = `Docker daemon still unreachable after ${unreachableFor()} (\`docker version\` kept failing); not waiting for it any longer`; + log(message); + throw new Error(message); + })(); + // The window can close while no request is waiting; each waiter gets the + // rejection through the race below, so it must not also be reported as an + // unhandled rejection here. + ready.catch(() => {}); + + return function waitFor(capMs: number): Promise { + let timer: ReturnType | undefined; + const cap = new Promise((_, reject) => { + timer = setTimeout(() => { + reject( + new Error( + `Docker daemon not reachable (unreachable for ${unreachableFor()} so far); gave up waiting for it after ${Math.round(capMs / 1000)}s for this request`, + ), + ); + }, capMs); + }); + return Promise.race([ready, cap]).finally(() => clearTimeout(timer)); + }; +} + interface DockerComposeOptions { projectName?: string; composeFile?: string; @@ -174,15 +272,7 @@ class DockerComposeHelper { } async ensureDocker(): Promise { - // Check Docker is available - const dockerCheck = spawn({ - cmd: ["docker", "version"], - stdout: "pipe", - stderr: "pipe", - }); - - const exitCode = await dockerCheck.exited; - if (exitCode !== 0) { + if (!(await isDockerDaemonReachable())) { throw new Error("Docker is not available. Please ensure Docker is installed and running."); } diff --git a/test/docker/prestart-map.mjs b/test/docker/prestart-map.mjs index a8c0c0d9b5bd..b4956426be48 100644 --- a/test/docker/prestart-map.mjs +++ b/test/docker/prestart-map.mjs @@ -25,6 +25,7 @@ export const prestartMap = { "js/bun/s3/": ["minio"], "js/web/websocket/autobahn": ["autobahn"], "js/web/websocket/websocket-proxy": ["squid"], + "cli/install/bun-install-proxy": ["squid"], "integration/mysql2/": ["mysql_plain", "mysql_native_password"], "regression/issue/21311": ["postgres_plain"], "regression/issue/24850": ["mysql_plain"], diff --git a/test/harness.ts b/test/harness.ts index b3167919ec84..1886ca8d6be6 100644 --- a/test/harness.ts +++ b/test/harness.ts @@ -1110,6 +1110,16 @@ export function isDockerEnabled(): boolean { return false; } + // The CI runner sets this once test/docker/coordinator.ts is up; it owns the + // daemon from then on and waits for it if the agent started the shard before + // dockerd finished coming up (routine on the openrc images). Probing the + // daemon here as well would fail the whole file at import time during that + // window, and costs a `docker info` per file. describeWithContainer() and + // test/js/valkey/test-utils.ts already trust the coordinator the same way. + if (process.env.BUN_DOCKER_COORDINATOR) { + return true; + } + try { const info = execSync(`"${dockerCLI}" info`, { stdio: ["ignore", "pipe", "inherit"] }); return info.toString().indexOf("Server Version:") !== -1; diff --git a/test/internal/docker-coordinator.test.ts b/test/internal/docker-coordinator.test.ts new file mode 100644 index 000000000000..350302ffe4cb --- /dev/null +++ b/test/internal/docker-coordinator.test.ts @@ -0,0 +1,332 @@ +/** + * test/docker/coordinator.ts is spawned by scripts/runner.node.mjs at the start + * of every Linux CI shard and owns `docker compose` for the container-backed + * test files in it. The agent can start a shard before dockerd has finished + * coming up (on the openrc images the agent service is not ordered after + * docker), and until the coordinator waited for the daemon, its one-shot + * `docker version` answered every request, and so failed every container-backed + * test file in the shard including the in-shard retries, with "Docker is not + * available" (builds 97891, 98647, 98707, 98746: the daemon was still down 3-4.5 + * minutes into the job and came up a few minutes after that). + * + * The wait is driven here with a scripted probe and millisecond timings. The + * tests after that put a fake `docker` on PATH: one whose CLI hangs, to pin + * the probe's own bound (everything above assumes a probe returns), and one + * whose daemon is still starting, used both for isDockerEnabled() in + * test/harness.ts (which used to probe the daemon at import time of ~25 test + * files and so failed them the same way) and to run the real coordinator end + * to end. + */ +import { expect, test } from "bun:test"; +import { bunEnv, bunExe, isLinux, isWindows, tempDir } from "harness"; +import { chmodSync, existsSync, readFileSync } from "node:fs"; +import { connect } from "node:net"; +import { join } from "node:path"; + +import { waitForDockerDaemon } from "../docker/index.ts"; + +/** Answers `outcomes` in order, then `finalOutcome` forever. */ +function scriptedProbe(outcomes: boolean[], finalOutcome: boolean) { + const remaining = [...outcomes]; + const probe = async () => { + probe.calls++; + return remaining.length ? remaining.shift()! : finalOutcome; + }; + probe.calls = 0; + return probe; +} + +const rejection = (promise: Promise) => + promise.then( + () => { + throw new Error("expected the wait to fail"); + }, + (error: Error) => error, + ); + +test("a daemon that is already up costs one probe and logs nothing", async () => { + const probe = scriptedProbe([], true); + const log: string[] = []; + const waitFor = waitForDockerDaemon({ windowMs: 10_000, pollMs: 1, probe, log: m => log.push(m) }); + + await waitFor(1_000); + + expect(probe.calls).toBe(1); + expect(log).toEqual([]); +}); + +test("requests made while the daemon is starting wait for it on one shared poll loop", async () => { + const probe = scriptedProbe([false, false, false], true); + const log: string[] = []; + const waitFor = waitForDockerDaemon({ windowMs: 10_000, pollMs: 1, probe, log: m => log.push(m) }); + + // Two services requested at once, as the prestart does. + await Promise.all([waitFor(5_000), waitFor(5_000)]); + + expect(probe.calls).toBe(4); + expect(log).toEqual([ + "docker daemon is not reachable yet; waiting up to 10s for it to come up", + expect.stringMatching(/^docker daemon became reachable after \d+s$/), + ]); + + // From now on requests neither probe nor wait. + await waitFor(0); + expect(probe.calls).toBe(4); +}); + +test("a request that hits its cap fails by itself; polling continues so a retry gets the daemon", async () => { + let up = false; + let calls = 0; + const probe = async () => { + calls++; + return up; + }; + const waitFor = waitForDockerDaemon({ windowMs: 10_000, pollMs: 1, probe, log: () => {} }); + + const error = await rejection(waitFor(20)); + expect(error.message).toMatch( + /^Docker daemon not reachable \(unreachable for \d+s so far\); gave up waiting for it after 0s for this request$/, + ); + + // The runner retries the file whose request just failed; by then the daemon + // may be up, and the retry's request must see that. + const callsAtCap = calls; + up = true; + await waitFor(5_000); + expect(calls).toBeGreaterThan(callsAtCap); +}); + +test("a request pending when the window closes gets the window failure", async () => { + const probe = scriptedProbe([], false); + const log: string[] = []; + const waitFor = waitForDockerDaemon({ windowMs: 30, pollMs: 1, probe, log: m => log.push(m) }); + + const error = await rejection(waitFor(5_000)); + expect(error.message).toMatch( + /^Docker daemon still unreachable after \d+s \(`docker version` kept failing\); not waiting for it any longer$/, + ); + // The same line goes to the shard log, so the log explains the failures even + // when they surface in test files much later. + expect(log).toEqual(["docker daemon is not reachable yet; waiting up to 0s for it to come up", error.message]); +}); + +test("after the window closes with nothing waiting, later requests fail at once instead of waiting out their cap", async () => { + const probe = scriptedProbe([], false); + const { promise: windowClosed, resolve: onWindowClosed } = Promise.withResolvers(); + const waitFor = waitForDockerDaemon({ + windowMs: 30, + pollMs: 1, + probe, + log: message => { + if (message.startsWith("Docker daemon still unreachable")) onWindowClosed(message); + }, + }); + + // Nobody is waiting while the window closes. The log line is emitted just + // before the shared promise rejects, so let the event loop turn once: if the + // wait let that rejection escape, bun:test reports it as an unhandled error + // here, before any request attaches to it. + const message = await windowClosed; + await new Promise(resolve => setImmediate(resolve)); + const callsAtClose = probe.calls; + + const error = await rejection(waitFor(60_000)); + expect(error.message).toBe(message); + expect(probe.calls).toBe(callsAtClose); +}); + +// A `docker` CLI whose daemon is still starting: `docker info` fails, `docker +// version` fails the first time it is run and succeeds from then on, `docker +// compose ... port ` answers with a fixed host port, and the +// other compose subcommands (version, build, ps, up) succeed silently. Every +// invocation is appended to calls.log in the directory above bin/. +const fakeDocker = `#!/bin/sh +state=$(dirname "$0")/.. +echo "$*" >> "$state/calls.log" +case "$1" in + info) + exit 1 + ;; + version) + if [ -e "$state/probed-once" ]; then exit 0; fi + : > "$state/probed-once" + echo "Cannot connect to the Docker daemon at unix:///var/run/docker.sock" >&2 + exit 1 + ;; + compose) + for arg in "$@"; do + if [ "$arg" = port ]; then echo "0.0.0.0:54321"; exit 0; fi + done + ;; +esac +exit 0 +`; + +// A `docker` CLI stuck talking to a daemon that accepted the connection but +// never answers. +const hangingDocker = `#!/bin/sh +if [ "$1" = version ]; then exec sleep 15; fi +exit 0 +`; + +function fakeDockerDir(script = fakeDocker) { + // Short name: the coordinator socket created inside it has to fit in + // sun_path (104 bytes on macOS). + const dir = tempDir("coord", { "bin/docker": script }); + chmodSync(join(String(dir), "bin", "docker"), 0o755); + return dir; +} + +const calls = (dir: string) => (existsSync(join(dir, "calls.log")) ? readFileSync(join(dir, "calls.log"), "utf8") : ""); + +// The docker helpers look the CLI up in the environment their process started +// with (Bun.which and Bun.spawn both read that, not later edits to +// process.env), so the fake CLI has to be first on PATH of a fresh process. +// ensure() also answers from BUN_TEST_SERVICE_ without touching docker, +// and BUN_DOCKER_* / COMPOSE_* change what it runs, so none of those may leak +// in from the machine running this file. +function envWithFakeDocker(dir: string): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(bunEnv)) { + if (value !== undefined && !/^(BUN_TEST_SERVICE_|BUN_DOCKER_|COMPOSE_)/.test(key)) env[key] = value; + } + env.PATH = `${join(dir, "bin")}:${env.PATH}`; + return env; +} + +/** Runs `script` (which must print one JSON value) in a fresh bun with the fake docker on PATH. */ +async function evalWithFakeDocker(env: Record, script: string): Promise { + await using proc = Bun.spawn({ cmd: [bunExe(), "-e", script], env, stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(exitCode).toBe(0); + return JSON.parse(stdout); +} + +const harnessPath = JSON.stringify(join(import.meta.dir, "..", "harness.ts")); +const dockerIndexPath = JSON.stringify(join(import.meta.dir, "..", "docker", "index.ts")); + +function isDockerEnabledWith(dir: string, coordinator: string | undefined): Promise { + const env = envWithFakeDocker(dir); + if (coordinator !== undefined) env.BUN_DOCKER_COORDINATOR = coordinator; + return evalWithFakeDocker( + env, + `import { isDockerEnabled } from ${harnessPath}; + let result; + try { result = isDockerEnabled(); } catch (error) { result = "threw: " + error.message; } + console.log(JSON.stringify(result));`, + ); +} + +test.concurrent.skipIf(isWindows)( + "a probe whose docker CLI hangs counts as unreachable instead of hanging the wait", + async () => { + using dir = fakeDockerDir(hangingDocker); + const result = await evalWithFakeDocker( + envWithFakeDocker(String(dir)), + `import { isDockerDaemonReachable } from ${dockerIndexPath}; + const startedAt = Date.now(); + const reachable = await isDockerDaemonReachable(250); + console.log(JSON.stringify({ reachable, tookMs: Date.now() - startedAt }));`, + ); + // Without the bound the probe returns only when the fake CLI's 15s sleep ends. + expect(result).toEqual({ reachable: false, tookMs: expect.any(Number) }); + expect((result as { tookMs: number }).tookMs).toBeLessThan(15_000); + }, +); + +// On Linux arm64 isDockerEnabled() is false before it looks at anything else. +test.concurrent.skipIf(isWindows || (isLinux && process.arch === "arm64"))( + "isDockerEnabled() trusts the shard's coordinator instead of probing the daemon from every test file", + async () => { + using withCoordinator = fakeDockerDir(); + using withoutCoordinator = fakeDockerDir(); + const [enabled, enabledWithout] = await Promise.all([ + isDockerEnabledWith(String(withCoordinator), join(String(withCoordinator), "coordinator.sock")), + isDockerEnabledWith(String(withoutCoordinator), undefined), + ]); + + expect(enabled).toBe(true); + expect(calls(String(withCoordinator))).toBe(""); + + // Without a coordinator the daemon is probed as before, and this one is + // down: false, or on Linux CI the harness's own "docker is required" error. + expect(enabledWithout).not.toBe(true); + expect(calls(String(withoutCoordinator))).toBe("info\n"); + }, +); + +test.concurrent.skipIf(isWindows)( + "the coordinator answers a request made while the daemon is still starting", + async () => { + using dir = fakeDockerDir(); + const socketPath = join(String(dir), "coordinator.sock"); + + const env = envWithFakeDocker(String(dir)); + env.BUN_DOCKER_COORDINATOR_SOCKET = socketPath; + + await using coordinator = Bun.spawn({ + cmd: [bunExe(), join(import.meta.dir, "..", "docker", "coordinator.ts")], + env, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); + + // The runner connects tests to the socket only after this line. stdout has + // to be read incrementally for that, so it is collected by hand; a + // coordinator that exits before printing it fails the test right away. + let stdout = ""; + const stderr = coordinator.stderr.text(); + const { promise: listening, resolve: onListening, reject: onExitedFirst } = Promise.withResolvers(); + const stdoutDrained = (async () => { + for await (const chunk of coordinator.stdout) { + stdout += Buffer.from(chunk).toString(); + if (stdout.includes(`COORDINATOR_READY ${socketPath}\n`)) onListening(); + } + onExitedFirst( + new Error(`coordinator exited before COORDINATOR_READY\nstdout:\n${stdout}\nstderr:\n${await stderr}`), + ); + })(); + await listening; + + // What ensureViaCoordinator() in test/docker/index.ts sends and reads back. + const reply = await new Promise((resolve, reject) => { + const socket = connect(socketPath, () => + socket.write(JSON.stringify({ type: "ensure", service: "postgres_plain" }) + "\n"), + ); + let buffer = ""; + socket.setEncoding("utf8"); + socket.on("data", chunk => { + buffer += chunk; + const newline = buffer.indexOf("\n"); + if (newline !== -1) { + socket.end(); + resolve(buffer.slice(0, newline)); + } + }); + socket.on("error", reject); + socket.on("close", () => reject(new Error(`coordinator closed the socket without replying; got: ${buffer}`))); + }); + expect(JSON.parse(reply)).toEqual({ ok: true, info: { host: "127.0.0.1", ports: { 5432: 54321 } } }); + + // The probe that failed, the poll that found the daemon, and only then any + // compose invocation. + const invocations = calls(String(dir)).trim().split("\n"); + expect(invocations.slice(0, 2)).toEqual(["version", "version"]); + expect(invocations.findIndex(call => call.startsWith("compose "))).toBeGreaterThanOrEqual(2); + expect(invocations).toContainEqual( + expect.stringMatching(/^compose .* up -d --wait --wait-timeout 180 postgres_plain$/), + ); + + // EOF on stdin is how the coordinator learns the shard is over. + coordinator.stdin.end(); + const exitCode = await coordinator.exited; + await stdoutDrained; + expect(stdout).toMatch(/^coordinator: docker daemon became reachable after \d+s$/m); + expect(stdout).toContain("coordinator: postgres_plain ready\n"); + expect(await stderr).toBe(""); + expect(exitCode).toBe(0); + expect(existsSync(socketPath)).toBeFalse(); + }, +);