Certify a restarting deploy in a canary worker before it rolls out

.github/workflows/unit-test.yml

@@ -94,8 +94,10 @@ jobs:
       # from the Actions step API: a cap-killed sample is right-censored and reads far too low.
       # Derived 2026-09-18 over the 30 `Unit Test` pushes since 2026-09-14, and only `resources`
       # was re-budgeted — capped at 7m off a ~4m30-5m25 estimate nobody re-measured, it reached
-      # 6m58 completed, went red on main, and needed ~7m40 at its worst. `main` (3m19) sits at ~1.2x
-      # its own worst, not 1.5x. `lmdb` was re-derived 2026-09-28 over the pushes since 2026-09-26:
+      # 6m58 completed, went red on main, and needed ~7m40 at its worst. `main` was re-derived
+      # 2026-10-02: the eight pushes since 2026-10-01 took 3m12-3m59 against its 4m cap, and
+      # harper#2981's legs reached mocha's epilogue at 4m02-4m12 before the cap killed them, so 6m.
+      # `lmdb` was re-derived 2026-09-28 over the pushes since 2026-09-26:
       # slowest completed 3m38, ~3m20 once `subscriptionSuperseded` stopped replaying the shared
       # `test` database's whole audit log; 1.5x is 5m, and the legs killed at 4m ran longer than any
       # that completed, so 6m. Its LMDB-`resources` phase stretches on a slow runner in tests that
@@ -112,7 +114,7 @@ jobs:
       # core 4m50 -> 8m, indexes 2m50 -> 5m.
       - name: 'Unit tests: main'
         if: "!cancelled() && steps.setup-harper.outcome == 'success'"
-        timeout-minutes: 4
+        timeout-minutes: 6
         run: npm run test:unit:main
 
       - name: 'Unit tests: apitests'

DESIGN.md

@@ -139,8 +139,8 @@ Index of the design notes for the harper core: one line per note, grouped by the
 
 ## components/ — deploys and the load lifecycle
 
-- [A deploy builds off to the side, is validated, and only then goes live](components/DESIGN.md#a-deploy-builds-off-to-the-side-is-validated-and-only-then-goes-live) — A deploy builds in `.deploy-staging`, validates that tree, then swaps live to `.deploy-aside` and the candidate in, waiting out handle holders with the old version in place; the component's root-config entry is journaled with the activation and published durably, under one lock, only after the swap commits; the release it displaces goes back under its own deployment id, which `deployment_id` activates again, and activating the id already live answers without a swap.
-- [Component preparation is serialized across worker threads](components/DESIGN.md#component-preparation-is-serialized-across-worker-threads) — One component transaction at a time across workers; the lifecycle broadcast sits outside the lock; a deploy's own validation load never waits on a deploy; the load lock is keyed by application and plugin.
+- [A deploy builds off to the side, is certified, and only then goes live](components/DESIGN.md#a-deploy-builds-off-to-the-side-is-certified-and-only-then-goes-live) — A deploy builds in `.deploy-staging`, certifies that tree, then swaps live to `.deploy-aside` and the candidate in, waiting out handle holders with the old version in place; the component's root-config entry is journaled with the activation and published durably, under one lock, only after the swap commits; a deploy that restarts workers is decided by a canary worker held out of traffic until it reports its load, and a rejection restores the release it replaced or fails the component closed; the release a swap displaces goes back under its own deployment id, which `deployment_id` activates again, and activating the id already live answers without a swap.
+- [Component preparation is serialized across worker threads](components/DESIGN.md#component-preparation-is-serialized-across-worker-threads) — One component transaction at a time across workers; the lifecycle broadcast sits outside the lock; the load lock is keyed by application and plugin.
 - [Peer-side deploy_component payload read: retryable blob stalls and `Readable.from()` cancellation](components/DESIGN.md#peer-side-deploy_component-payload-read-retryable-blob-stalls-and-readablefrom-cancellation) — Retry a `BLOB_UNAVAILABLE` read only before any byte reached the consumer; use `Readable.from()` for cancellation, not a hand-rolled `ReadableStream`.
 - [An origin waits for a peer's deploy answer only as long as the peer may take](components/DESIGN.md#an-origin-waits-for-a-peers-deploy-answer-only-as-long-as-the-peer-may-take) — The replicated deploy's per-peer deadline sums the peer's own allowances, so it never undercuts a healthy peer; a missed deadline is a failure with an unknown outcome, not cancellation.
 - [A replicated deploy reports each node's install fingerprint, and never acts on it](components/DESIGN.md#a-replicated-deploy-reports-each-nodes-install-fingerprint-and-never-acts-on-it) — Each node records the source identity its resolver already produced and the sha256 of its root lockfiles; the origin compares peers once on the aggregate and reports differences in a warning, the message and `peer_results`, without ever failing the deploy.

bin/restart.ts

@@ -32,7 +32,7 @@ const ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS = 5000;
 
 let calledFromCli;
 
-export { restart, restartService };
+export { restart, restartService, activateDeploymentOnPeers };
 
 // Add ITC event listener to main thread which will be called from child that receives restart request.
 if (isMainThread) {
@@ -183,6 +183,72 @@ async function restart(req: any) {
 	return RESTART_RESPONSE;
 }
 
+const DEPLOYMENT_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+/**
+ * A certified rolling deploy's second half: the release is staged on every peer, and this activates it on one peer
+ * at a time, each certifying it with its own restart. Every peer is visited — each decides for itself — and the job
+ * fails at the end naming each one that did not take it.
+ */
+async function activateDeploymentOnPeers(activation: any) {
+	const { project, deployment_id: deploymentId, deployment_row: deploymentRow, nodes } = activation ?? {};
+	if (
+		typeof project !== 'string' ||
+		!DEPLOYMENT_ID_PATTERN.test(deploymentId) ||
+		!DEPLOYMENT_ID_PATTERN.test(deploymentRow) ||
+		(nodes !== undefined && (!Array.isArray(nodes) || nodes.some((node) => typeof node !== 'string')))
+	) {
+		throw handleHDBError(
+			new Error(),
+			'Invalid activate_deployment: expected project, deployment_id, deployment_row and an optional list of nodes',
+			HTTP_STATUS_CODES.BAD_REQUEST,
+			undefined,
+			undefined,
+			true
+		);
+	}
+	const peers = ((global as any).server.nodes ?? []).filter(
+		(node) => node.name !== getThisNodeName() && (!nodes || nodes.includes(node.name))
+	);
+	const results = [];
+	if (peers.length === 0) return { activated: results };
+	const replication = (global as any).server.replication;
+	replication.monitorNodeCAs();
+	const { peerDeployAnswerTimeoutMs } = await import('../components/operations.js');
+	const timeoutMs = peerDeployAnswerTimeoutMs({ restart: true });
+	for (const node of peers) {
+		try {
+			const response = await replication.sendOperationToNode(
+				node,
+				{
+					operation: 'deploy_component',
+					project,
+					deployment_id: deploymentId,
+					_deploymentId: deploymentRow,
+					restart: true,
+					replicated: false,
+				},
+				{ timeoutMs }
+			);
+			const answer = response?.value ?? response?.body ?? response;
+			results.push({ node: node.name, certification: answer?.certification, message: answer?.message });
+		} catch (error) {
+			results.push({ node: node.name, error: error?.message ?? String(error) });
+		}
+	}
+	const failed = results.filter((result) => result.error);
+	if (failed.length > 0) {
+		const error: any = new Error(
+			`Deployment ${deploymentId} of ${project} was not activated on ${failed.length} of ${results.length} peer ` +
+				`node(s): ${failed.map((result) => `${result.node} (${result.error})`).join('; ')}`
+		);
+		// What a failed job records as its message, so get_job keeps every peer's outcome.
+		error.http_resp_msg = { error: error.message, activated: results };
+		throw error;
+	}
+	return { activated: results };
+}
+
 /**
  * Used to restart a particular service, services includes - httpWorkers
  * @param req
@@ -236,6 +302,19 @@ async function restartService(req: any) {
 			);
 		}
 	}
+	if (req.activate_deployment !== undefined) {
+		if (isMainThread) {
+			throw handleHDBError(
+				new Error(),
+				'activate_deployment runs as a job',
+				HTTP_STATUS_CODES.BAD_REQUEST,
+				undefined,
+				undefined,
+				true
+			);
+		}
+		return activateDeploymentOnPeers(req.activate_deployment);
+	}
 	processMan.expectedRestartOfChildren();
 	if (!isMainThread) {
 		if (req.replicated) {