From 73f45a1a10998b78edcd00ebbe852deecd60fc3f Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 8 Aug 2026 18:33:15 -0500 Subject: [PATCH 01/90] feat: add deployer host installer --- .../ci-fleet-deployer-cleanup.service | 18 + .../deployer/ci-fleet-deployer-cleanup.timer | 11 + .../deployer/ci-fleet-deployer-drain.service | 17 + .../deployer/ci-fleet-deployer-health.service | 18 + .../deployer/ci-fleet-deployer-health.timer | 12 + deploy/deployer/ci-fleet-deployer.service | 24 + docs/DEPLOYER-HOST.md | 295 ++++++++ docs/README.md | 3 + scripts/deployer-runtime.sh | 111 +++ scripts/install-deployer.sh | 652 ++++++++++++++++++ scripts/test-deployer-units.sh | 20 + scripts/test-install-deployer.sh | 430 ++++++++++++ scripts/validate.sh | 2 + 13 files changed, 1613 insertions(+) create mode 100644 deploy/deployer/ci-fleet-deployer-cleanup.service create mode 100644 deploy/deployer/ci-fleet-deployer-cleanup.timer create mode 100644 deploy/deployer/ci-fleet-deployer-drain.service create mode 100644 deploy/deployer/ci-fleet-deployer-health.service create mode 100644 deploy/deployer/ci-fleet-deployer-health.timer create mode 100644 deploy/deployer/ci-fleet-deployer.service create mode 100644 docs/DEPLOYER-HOST.md create mode 100755 scripts/deployer-runtime.sh create mode 100755 scripts/install-deployer.sh create mode 100755 scripts/test-deployer-units.sh create mode 100755 scripts/test-install-deployer.sh diff --git a/deploy/deployer/ci-fleet-deployer-cleanup.service b/deploy/deployer/ci-fleet-deployer-cleanup.service new file mode 100644 index 0000000..76ecb11 --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-cleanup.service @@ -0,0 +1,18 @@ +[Unit] +Description=Run application-owned scoped deployer cleanup +After=docker.service + +[Service] +Type=oneshot +User=root +ExecStart=/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh cleanup +Environment=CI_FLEET_DEPLOYER_CONFIG=/var/lib/ci-fleet-deployer/active-policy.conf +TimeoutStartSec=15min +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadOnlyPaths=/etc/ci-fleet-deployer /opt/ci-fleet-deployer +ReadWritePaths=/var/lib/ci-fleet-deployer /var/log/ci-fleet-deployer /var/lock/ci-fleet-deployer /run/docker.sock +RestrictSUIDSGID=yes diff --git a/deploy/deployer/ci-fleet-deployer-cleanup.timer b/deploy/deployer/ci-fleet-deployer-cleanup.timer new file mode 100644 index 0000000..0ac89aa --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-cleanup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run scoped ci-fleet deployer cleanup daily + +[Timer] +OnCalendar=daily +RandomizedDelaySec=15min +Persistent=true +Unit=ci-fleet-deployer-cleanup.service + +[Install] +WantedBy=timers.target diff --git a/deploy/deployer/ci-fleet-deployer-drain.service b/deploy/deployer/ci-fleet-deployer-drain.service new file mode 100644 index 0000000..eac308a --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-drain.service @@ -0,0 +1,17 @@ +[Unit] +Description=Drain the ci-fleet deployer before maintenance + +[Service] +Type=oneshot +User=root +ExecStart=/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh drain +Environment=CI_FLEET_DEPLOYER_CONFIG=/var/lib/ci-fleet-deployer/active-policy.conf +TimeoutStartSec=2min +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadOnlyPaths=/etc/ci-fleet-deployer /opt/ci-fleet-deployer +ReadWritePaths=/var/lib/ci-fleet-deployer /var/log/ci-fleet-deployer /var/lock/ci-fleet-deployer +RestrictSUIDSGID=yes diff --git a/deploy/deployer/ci-fleet-deployer-health.service b/deploy/deployer/ci-fleet-deployer-health.service new file mode 100644 index 0000000..8d87b06 --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-health.service @@ -0,0 +1,18 @@ +[Unit] +Description=Check ci-fleet deployer health +After=docker.service + +[Service] +Type=oneshot +User=root +ExecStart=/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh health +Environment=CI_FLEET_DEPLOYER_CONFIG=/var/lib/ci-fleet-deployer/active-policy.conf +TimeoutStartSec=2min +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadOnlyPaths=/etc/ci-fleet-deployer /opt/ci-fleet-deployer +ReadWritePaths=/var/lib/ci-fleet-deployer /var/log/ci-fleet-deployer /var/lock/ci-fleet-deployer /run/docker.sock +RestrictSUIDSGID=yes diff --git a/deploy/deployer/ci-fleet-deployer-health.timer b/deploy/deployer/ci-fleet-deployer-health.timer new file mode 100644 index 0000000..637cfb9 --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer-health.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Check ci-fleet deployer health every five minutes + +[Timer] +OnActiveSec=2min +OnUnitActiveSec=5min +RandomizedDelaySec=30s +Persistent=true +Unit=ci-fleet-deployer-health.service + +[Install] +WantedBy=timers.target diff --git a/deploy/deployer/ci-fleet-deployer.service b/deploy/deployer/ci-fleet-deployer.service new file mode 100644 index 0000000..786eaad --- /dev/null +++ b/deploy/deployer/ci-fleet-deployer.service @@ -0,0 +1,24 @@ +[Unit] +Description=Run one approved ci-fleet deployment +After=docker.service network-online.target +Wants=docker.service network-online.target +ConditionPathExists=!/var/lib/ci-fleet-deployer/drained + +[Service] +Type=oneshot +User=root +Group=root +ExecStart=/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh deploy +Environment=CI_FLEET_DEPLOYER_CONFIG=/var/lib/ci-fleet-deployer/active-policy.conf +Environment=CI_FLEET_DEPLOYER_REQUEST=/var/lib/ci-fleet-deployer/request.conf +TimeoutStartSec=45min +TimeoutStopSec=45min +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadOnlyPaths=/etc/ci-fleet-deployer /opt/ci-fleet-deployer +ReadWritePaths=/var/lib/ci-fleet-deployer /var/log/ci-fleet-deployer /var/lock/ci-fleet-deployer /run/docker.sock +RestrictSUIDSGID=yes +LockPersonality=yes diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md new file mode 100644 index 0000000..ae80499 --- /dev/null +++ b/docs/DEPLOYER-HOST.md @@ -0,0 +1,295 @@ +# Deployer host installation and operation + +This runbook installs the generic ci-fleet deployer runtime on a dedicated Linux host. It does not deploy an application by itself. Application repositories retain their own deployment logic through the narrow adapter contract below. + +A deployer is a separate trust boundary. It must never register as an ordinary GitHub Actions runner, accept pull-request jobs, share a Docker daemon with CI workers, or run on the ordinary controller host. Staging and production require different hosts or equally isolated security boundaries, identities, credential scopes, state, approval evidence, and target policy. + +## Capability and security model + +The runtime accepts only a full source commit, an image reference ending in `@sha256:<64 lowercase hex>`, exact matching approval evidence, explicit environment and target identities, a protected credential reference, and checkpoint evidence. It never follows `latest`, a branch, a moving tag, or an implicit production default. + +GitHub-native Environment protection is optional, not assumed. In particular, do not assume a private repository on GitHub Free has protected Environments, Environment secrets, branch protection, or rulesets. Use one of these fail-closed approval providers: + +- `manual-exact-head`: a protected host-local evidence file records an approval outside the secret store; +- `external-exact-head`: an external policy system writes the same bounded evidence contract; +- `github-environment`: accepted only with separate host-local capability evidence proving exact-head Environment protection was verified for this installation. + +A secret store proves only that a credential exists. It is not approval evidence. Every provider still binds the environment, target, source commit, artifact digest, approving identity, policy identity, approval ID, and UTC time. + +The systemd services run as root because access to the Docker socket is root-equivalent and protected credential references may be root-only. `NoNewPrivileges`, a read-only host filesystem, explicit writable paths, private temporary storage, and no supplementary service account reduce accidental reach, but they do not turn Docker access into a low-privilege boundary. Put nothing else on this host. + +## Supported host and prerequisites + +The installer supports Debian 12/13 and Ubuntu 22.04/24.04 with systemd. It checks Docker Engine, synchronized time, disk capacity, DNS, HTTPS reachability, and Compose v2 when `REQUIRE_COMPOSE=1`. The configured network host is supplied to curl over standard input so a private endpoint is not placed in the process argument list. + +It rejects: + +- ordinary ci-fleet controller state or runner units; +- any unrelated running or stopped Docker container; +- any unrelated custom Docker network or volume; +- unsafe owners, modes, symlinks, traversal, malformed configuration, or ambiguous identities. + +`--check` is read-only: it creates no user, directory, lock, release, service, timer, image, container, registration, or GitHub state. Mutating modes require root. + +## Filesystem contract + +| Path | Owner/mode | Purpose | Uninstall | +| --- | --- | --- | --- | +| `/etc/ci-fleet-deployer/` | root `0700` | host-local policy boundary | retained | +| `/etc/ci-fleet-deployer/deployer.conf` | root `0600` | bounded non-secret policy and references | retained | +| `/etc/ci-fleet-deployer/adapters/` | root `0700` | application-owned adapter | retained | +| `/etc/ci-fleet-deployer/credentials/` | root `0700` | credential files | retained | +| `/etc/ci-fleet-deployer/evidence/` | root `0700` | approval/capability/checkpoint evidence | retained | +| `/opt/ci-fleet-deployer/releases//` | root `0755` | immutable core runtime release | retained | +| `/opt/ci-fleet-deployer/current` | root symlink | atomically selected core release | removed | +| `/var/lib/ci-fleet-deployer/` | root `0700` | active policy, request, drain, transaction, LKG state | retained except transient drain/active state | +| `/var/lock/ci-fleet-deployer/` | root `0700` | flock serialization boundary | retained | +| `/var/log/ci-fleet-deployer/` | root `0700` | secret-free audit log | retained | +| `/etc/systemd/system/ci-fleet-deployer*` | root `0644` | deploy, health, cleanup, and drain units/timers | removed | + +Releases, last-known-good state, audit records, configuration, credentials, and evidence are retained deliberately. Their retention or destruction is a separate operator decision. + +## Application adapter contract + +The adapter is an application-owned root-only executable under `/etc/ci-fleet-deployer/adapters`. Its SHA-256 is pinned in `deployer.conf`. Core invokes exactly one positional operation: + +```text +adapter validate +adapter health +adapter cleanup +adapter deploy +adapter rollback +``` + +`validate` must be non-mutating and must prove that the candidate policy is usable before core changes the active release. `health` returns zero only when the deployer and application-owned contract are healthy. `cleanup` may remove only resources carrying the application's exact deployer ownership identity; it must never run global prune or touch unrelated resources. `deploy` reads the active policy and request paths from the documented environment variables and owns application-specific staging, rollout, health, and rollback. `rollback` restores application state compatible with the recorded last-known-good core policy. + +Operations have no interactive input. Zero means success; nonzero means failure. The adapter must honor the systemd timeout, avoid child processes that outlive it, redact logs, and never print credential contents, authorization headers, cookies, private endpoints, or secret-manager responses. Core validates immutable identifiers and approval evidence; it cannot validate application-specific correctness. + +## Prepare host-local files + +All commands in this section run on the dedicated deployer host and change host-local state. They do not contact GitHub or deploy an application. + +```bash +sudo install -d -o root -g root -m 0700 \ + /etc/ci-fleet-deployer \ + /etc/ci-fleet-deployer/adapters \ + /etc/ci-fleet-deployer/credentials \ + /etc/ci-fleet-deployer/evidence +sudo install -o root -g root -m 0700 ./application-adapter \ + /etc/ci-fleet-deployer/adapters/application-adapter +sudo install -o root -g root -m 0600 /dev/null \ + /etc/ci-fleet-deployer/credentials/application.credential +sudoedit /etc/ci-fleet-deployer/credentials/application.credential +``` + +`sudoedit` writes the credential without displaying it. Do not use a secret value in a command argument, shell history, environment variable, fixture, or log. An approved external secret-manager reference may replace the regular file; the adapter retrieves the value without core seeing it. + +Create checkpoint evidence only after an operator or approved external backup adapter has produced a recoverable checkpoint. Core cannot see or operate a hypervisor and does not claim to verify one. This example records evidence, not a secret: + +```bash +sudo install -o root -g root -m 0600 /dev/null \ + /etc/ci-fleet-deployer/evidence/checkpoint.conf +sudoedit /etc/ci-fleet-deployer/evidence/checkpoint.conf +``` + +Fictional checkpoint evidence: + +```text +SCHEMA_VERSION=1 +ENVIRONMENT=staging +TARGET_ID=example-staging +CHECKPOINT_ID=checkpoint-20260808-1 +RECORDED_AT=2026-08-08T19:55:00Z +``` + +Create independent exact-head approval evidence: + +```bash +sudo install -o root -g root -m 0600 /dev/null \ + /etc/ci-fleet-deployer/evidence/approval.conf +sudoedit /etc/ci-fleet-deployer/evidence/approval.conf +``` + +Fictional manual/external evidence: + +```text +SCHEMA_VERSION=1 +ENVIRONMENT=staging +TARGET_ID=example-staging +SOURCE_COMMIT=1111111111111111111111111111111111111111 +ARTIFACT_IMAGE=registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +APPROVAL_IDENTITY=example-reviewer +POLICY_IDENTITY=example-staging-policy-v1 +APPROVAL_ID=approval-20260808-1 +APPROVED_AT=2026-08-08T20:00:00Z +``` + +For `github-environment`, also create mode-`0600` capability evidence. This file must be produced by an authorized capability check; copying the fictional text is not proof: + +```text +SCHEMA_VERSION=1 +ENVIRONMENT_PROTECTION=verified +EXACT_HEAD=1111111111111111111111111111111111111111 +CAPABILITY_ID=example-capability-check +CHECKED_AT=2026-08-08T20:00:00Z +``` + +Create the bounded configuration. Values cannot contain shell expressions; the installer parses `KEY=VALUE` without sourcing it. Unknown, duplicate, empty, malformed, or missing keys fail closed. + +```bash +core_ref=$(git rev-parse HEAD) +adapter_sha=$(sudo sha256sum /etc/ci-fleet-deployer/adapters/application-adapter | cut -d' ' -f1) +sudo install -o root -g root -m 0600 /dev/null \ + /etc/ci-fleet-deployer/deployer.conf +{ + printf 'SCHEMA_VERSION=1\n' + printf 'CORE_REF=%s\n' "$core_ref" + printf 'ENVIRONMENT=staging\n' + printf 'TARGET_ID=example-staging\n' + printf 'DEPLOYER_IDENTITY=staging-deployer-01\n' + printf 'ADAPTER_PATH=/etc/ci-fleet-deployer/adapters/application-adapter\n' + printf 'ADAPTER_SHA256=%s\n' "$adapter_sha" + printf 'CREDENTIAL_PROVIDER=file\n' + printf 'CREDENTIAL_REF=/etc/ci-fleet-deployer/credentials/application.credential\n' + printf 'CREDENTIAL_SCOPE=staging\n' + printf 'APPROVAL_PROVIDER=manual-exact-head\n' + printf 'APPROVAL_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/approval.conf\n' + printf 'CHECKPOINT_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/checkpoint.conf\n' + printf 'SOURCE_COMMIT=1111111111111111111111111111111111111111\n' + printf 'ARTIFACT_IMAGE=registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n' + printf 'NETWORK_HOST=registry.example.invalid\n' + printf 'MIN_DISK_GIB=10\n' + printf 'REQUIRE_COMPOSE=1\n' +} | sudo tee /etc/ci-fleet-deployer/deployer.conf >/dev/null +sudo chmod 0600 /etc/ci-fleet-deployer/deployer.conf +``` + +For an external secret manager, use `CREDENTIAL_PROVIDER=external` and a non-secret reference shaped like `external:example-vault:staging/deployer`. For GitHub Environment approval, use `APPROVAL_PROVIDER=github-environment` and add `APPROVAL_CAPABILITY_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/github-capability.conf`. + +## Install, check, repair, upgrade, drain, and rollback + +These commands run from a clean checkout at the exact `CORE_REF` on the dedicated deployer host. + +Fresh install changes the host: + +```bash +sudo ./scripts/install-deployer.sh \ + --install --config /etc/ci-fleet-deployer/deployer.conf +``` + +Read-only validation: + +```bash +sudo ./scripts/install-deployer.sh \ + --check --config /etc/ci-fleet-deployer/deployer.conf +``` + +Repair owned drift without selecting a different environment or target; changes the host only when drift exists: + +```bash +sudo ./scripts/install-deployer.sh \ + --repair --config /etc/ci-fleet-deployer/deployer.conf +``` + +Upgrade after updating the exact core/source/artifact/approval/checkpoint fields; changes the host transactionally: + +```bash +sudo ./scripts/install-deployer.sh \ + --upgrade --config /etc/ci-fleet-deployer/deployer.conf +``` + +The adapter candidate passes `validate` before activation. Core stages a complete commit-pinned release, atomically switches it, writes active policy/state with mode `0600`, and keeps the prior state and policy as last-known-good. A failed candidate does not replace the current release. + +Drain before reboot or maintenance; changes drain state and refuses while a deployment is active: + +```bash +sudo ./scripts/install-deployer.sh \ + --drain --config /etc/ci-fleet-deployer/deployer.conf +sudo systemctl start ci-fleet-deployer-drain.service +``` + +Rollback changes the active core/application state but deliberately does not overwrite the operator-owned desired configuration or evidence. It refuses while a deployment is active: + +```bash +sudo ./scripts/install-deployer.sh \ + --rollback --config /etc/ci-fleet-deployer/deployer.conf +``` + +The report's next action is `restore-host-policy-evidence-then-check`. Restore the prior reviewed `deployer.conf`, exact-head approval evidence, and checkpoint evidence through the same protected operator process that created them; do not copy secrets through the shell. Then verify read-only: + +```bash +sudo ./scripts/install-deployer.sh \ + --check --config /etc/ci-fleet-deployer/deployer.conf +``` + +A cross-environment or cross-target in-place upgrade is rejected. Build a separately isolated host with its own identity and credential scope instead. + +## Submit an approved deployment + +Place one root-owned mode-`0600` request at `/var/lib/ci-fleet-deployer/request.conf`. It uses the same exact-head approval fields shown above. No secret values belong in the request. After a successful adapter call, the runtime atomically moves it to `last-request.conf`; submitting the identical completed request again fails closed. + +Then run on the deployer host; this changes the application target through its adapter: + +```bash +sudo systemctl start ci-fleet-deployer.service +sudo systemctl status --no-pager ci-fleet-deployer.service +``` + +The runtime serializes operations with flock, writes a mode-`0600` active-operation marker, and removes it on completion. While the adapter runs, `systemd-inhibit` blocks shutdown and sleep; the deploy service has an explicit 45-minute start/stop bound. Upgrade, rollback, drain, and uninstall refuse while that marker belongs to a live or bounded recent process. A root-owned stale marker older than the fixed one-hour recovery bound is removed only by a serialized mutating installer run. Unsafe or malformed stale state fails closed. + +Health runs every five minutes and cleanup daily. Cleanup is delegated to the application adapter because only application-owned code knows its exact resources. Core itself issues no Docker delete or prune command. + +## Verification and reports + +Verify unit definitions and state without reading protected content: + +```bash +sudo systemd-analyze verify /etc/systemd/system/ci-fleet-deployer*.service \ + /etc/systemd/system/ci-fleet-deployer*.timer +sudo systemctl is-enabled ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer +sudo systemctl is-active ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer +sudo stat -c '%U:%G %a %n' \ + /etc/ci-fleet-deployer \ + /var/lib/ci-fleet-deployer \ + /var/lock/ci-fleet-deployer \ + /var/log/ci-fleet-deployer +``` + +Every installer exit emits one stable secret-free report. Example: + +```text +REPORT action=install result=CHANGED environment=staging target=example-staging version=2222222222222222222222222222222222222222 digest=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa health=healthy changed=yes rollback_available=no next=run-check +``` + +`result` is one of `CHANGED`, `NO_CHANGE`, `BLOCKED`, or `FAILED`. `BLOCKED` means an operator prerequisite or safety gate must be resolved. `FAILED` means attempted candidate work failed. The report includes identifiers, never credential contents. + +## Failure recovery + +1. Do not delete the lock, state, or current release blindly. +2. Confirm whether `/var/lib/ci-fleet-deployer/active-operation` names a live process. +3. If live, wait or follow the application adapter's documented bounded recovery. Never interrupt an unknown deployment to force an upgrade. +4. If the installer reports safe stale state, rerun the same mutating command; it removes only a protected stale marker and current-core staging directory while holding the lock. +5. If candidate validation or activation failed, run `--check`; the previous active release remains selected. +6. If application health failed after an authorized deployment, run `--rollback`, then `--check`. +7. If machine recovery is required, use the checkpoint named in the protected evidence or rebuild the isolated host from reviewed inputs. Core does not operate the hypervisor. + +## Uninstall + +Uninstall changes the host, drains first by refusing any active deployment, disables timers, removes only ci-fleet deployer units and the current activation pointer, and is idempotent: + +```bash +sudo ./scripts/install-deployer.sh \ + --uninstall --config /etc/ci-fleet-deployer/deployer.conf +``` + +It retains configuration, credential references and files, evidence, immutable releases, audit log, and last-known-good state. Review retention policy and destroy those items separately only after access revocation and audit requirements are satisfied. It never unregisters a runner, modifies GitHub, deletes an application repository, or removes unrelated Docker resources. + +## Repository-only validation + +The deterministic test uses an explicit alternate-root test mode and mocked Docker/systemd/network commands. It never changes the test machine's Docker daemon, systemd, GitHub state, or a deployment target: + +```bash +scripts/test-install-deployer.sh +``` + +Prepared-host proof is still required before production use: exercise the reviewed adapter, Docker/systemd behavior, network policy, checkpoint recovery, immutable artifact verification, staging and production isolation, active-deployment drain, health, rollback, and scoped cleanup on a separately authorized isolated deployment host. diff --git a/docs/README.md b/docs/README.md index 72c197c..2ab2cc5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ New operator? Follow the [Quickstart](QUICKSTART.md): what ci-fleet does, instal | Make a project compliant | [Project CI standard](PROJECT-STANDARD.md) and [compliance checklist](COMPLIANCE-CHECKLIST.md) | | Split tests across parallel workers | [Project CI standard](PROJECT-STANDARD.md) and the [parallel workflow example](../examples/workflows/parallel-ci.yml.example) | | Configure automatic updates and cleanup | [Host maintenance](HOST-MAINTENANCE.md) | +| Install or operate a dedicated deployment host | [Deployer host installation](DEPLOYER-HOST.md) | | Monitor hosts and detect missed reports | [Fleet health monitoring](HEALTH-MONITORING.md) and [authenticated status reporting](STATUS-REPORTING.md) | | Handle GitHub App, workflow, or deployment secrets | [Secrets model](SECRETS.md) and [security policy](../SECURITY.md) | | Review accepted implementation scope | [Design decisions](DESIGN-DECISIONS.md) | @@ -42,6 +43,7 @@ New operator? Follow the [Quickstart](QUICKSTART.md): what ci-fleet does, instal | Test shard | One bounded slice of a larger test suite, designed to run independently and usually finish within five minutes. | | Private delivery configuration | Repository allowlists, logical controllers, capacity budgets, environment policy, and required secret names kept in a private but secret-free repository. | | Desired state | A reviewed schema-v3 declaration of runner pools and logical controllers stored in a private, secret-free Git repository. | +| Deployer host | A dedicated non-runner Linux host that applies exact-head approved immutable artifacts through application-owned deployment logic. | ## Design and boundaries @@ -72,6 +74,7 @@ These pages are normative for compatible projects and hosts: - [Run the live pilot](LIVE-PILOT.md) - [Add a host](ADDING-A-HOST.md) - [Install, adopt, check, upgrade, roll back, or remove a controller](DESIRED-STATE.md) +- [Install, check, repair, upgrade, drain, roll back, or remove a deployer](DEPLOYER-HOST.md) - [Add a project](ADDING-A-PROJECT.md) - [Deploy the current experimental prototype](DEPLOYMENT-PROTOTYPE.md) - [Maintain, drain, clean, update, and reboot hosts](HOST-MAINTENANCE.md) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh new file mode 100755 index 0000000..11543cf --- /dev/null +++ b/scripts/deployer-runtime.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set +x + +operation=${1:-} +case "$operation" in health|cleanup|deploy|rollback|drain) ;; *) printf 'ERROR: usage: deployer-runtime.sh health|cleanup|deploy|rollback|drain\n' >&2; exit 2 ;; esac + +root=${CI_FLEET_DEPLOYER_ROOT:-} +testing=${CI_FLEET_DEPLOYER_TESTING:-0} +[[ -z "$root" || "$testing" == 1 ]] || { printf 'ERROR: alternate root is test-only\n' >&2; exit 2; } +if [[ "$testing" == 1 && -z "$root" ]]; then printf 'ERROR: test mode requires an alternate root\n' >&2; exit 2; fi +root_path() { printf '%s%s' "$root" "$1"; } +config=${CI_FLEET_DEPLOYER_CONFIG:-$(root_path /var/lib/ci-fleet-deployer/active-policy.conf)} +request=${CI_FLEET_DEPLOYER_REQUEST:-$(root_path /var/lib/ci-fleet-deployer/request.conf)} +state_root=$(root_path /var/lib/ci-fleet-deployer) +log_root=$(root_path /var/log/ci-fleet-deployer) +lock_dir=$(root_path /var/lock/ci-fleet-deployer) +active=$state_root/active-operation +drained=$state_root/drained +last_request=$state_root/last-request.conf +audit_log=$log_root/audit.log + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 2; } +expected_uid=0 +[[ "$testing" != 1 ]] || expected_uid=$(id -u) +secure_file() { + local path=$1 description=$2 mode=${3:-600} + [[ ! -L "$path" && -f "$path" ]] || die "$description must be a regular file, not a symlink" + [[ $(realpath -e -- "$path") == $(realpath -m -- "$path") ]] || die "$description path contains a symlink" + [[ $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || die "$description has unsafe owner or mode" +} +secure_directory() { + local path=$1 description=$2 + [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:700" ]] || die "$description has unsafe owner, mode, or type" +} +parse_file() { + local path=$1 prefix=$2 kind=$3 allowed=$4 line key value + declare -gA "$prefix=()" + local -n output=$prefix + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -n "$line" && ${line:0:1} != '#' ]] || continue + [[ "$line" == *=* ]] || die "malformed $kind line" + key=${line%%=*}; value=${line#*=} + [[ "$key" =~ ^[A-Z][A-Z0-9_]*$ && -n "$value" && "$value" != *$'\r'* ]] || die "malformed $kind line" + [[ " $allowed " == *" $key "* ]] || die "unknown $kind key" + [[ ! -v "output[$key]" ]] || die "duplicate $kind key" + # key indexes a nameref to an associative array. + # shellcheck disable=SC2004 + output[$key]=$value + done <"$path" +} + +secure_file "$config" 'deployer configuration' +config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' +parse_file "$config" cfg configuration "$config_keys" +for key in ENVIRONMENT TARGET_ID ADAPTER_PATH ADAPTER_SHA256 SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done +[[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' +[[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' +secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 +[[ $(sha256sum "${cfg[ADAPTER_PATH]}" | cut -d' ' -f1) == "${cfg[ADAPTER_SHA256]}" ]] || die 'application adapter digest mismatch' + +secure_directory "$state_root" 'deployer state directory' +secure_directory "$log_root" 'deployer log directory' +secure_directory "$lock_dir" 'deployer lock directory' +exec 9<"$lock_dir" +flock -n 9 || die 'another deployer operation is running' + +case "$operation" in + drain) + [[ ! -e "$active" ]] || die 'active deployment prevents drain' + temporary=$(mktemp "$state_root/.drained.XXXXXX") + chmod 0600 "$temporary" + mv -Tf "$temporary" "$drained" + ;; + health|cleanup|rollback) + "${cfg[ADAPTER_PATH]}" "$operation" + ;; + deploy) + [[ ! -e "$drained" ]] || die 'deployer is drained' + secure_file "$request" 'deployment request' + if [[ -e "$last_request" || -L "$last_request" ]]; then + secure_file "$last_request" 'last completed deployment request' + cmp -s "$request" "$last_request" && die 'deployment request was already completed' + fi + request_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' + parse_file "$request" req request "$request_keys" + for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT; do + [[ -v "req[$key]" ]] || die "deployment request is missing $key" + done + [[ ${req[SCHEMA_VERSION]} == 1 ]] || die 'unsupported deployment request schema' + for key in ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do + [[ ${req[$key]} == "${cfg[$key]}" ]] || die "deployment request $key does not match installed policy" + done + [[ ${req[SOURCE_COMMIT]} =~ ^[0-9a-f]{40}$ && ${req[ARTIFACT_IMAGE]} =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?/[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || die 'deployment request is not immutable and qualified' + for key in APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID; do [[ ${req[$key]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ ]] || die "deployment request has an unsafe $key"; done + [[ ${req[APPROVED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'deployment request has an invalid approval time' + umask 077 + temporary=$(mktemp "$state_root/.active.XXXXXX") + printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" + mv -Tf "$temporary" "$active" + trap 'rm -f "$active"' EXIT INT TERM + systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ + --why='approved deployment is active' -- "${cfg[ADAPTER_PATH]}" deploy + mv -Tf "$request" "$last_request" + if [[ -e "$audit_log" || -L "$audit_log" ]]; then secure_file "$audit_log" 'deployer audit log'; else install -m 0600 /dev/null "$audit_log"; fi + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=success\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" \ + >>"$audit_log" + ;; +esac diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh new file mode 100755 index 0000000..1b33853 --- /dev/null +++ b/scripts/install-deployer.sh @@ -0,0 +1,652 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +set +x +export PYTHONDONTWRITEBYTECODE=1 + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +mode= +config= +error_reported=0 +root=${CI_FLEET_DEPLOYER_ROOT:-} +testing=${CI_FLEET_DEPLOYER_TESTING:-0} +effective_uid=${EUID:-$(id -u)} +action=unknown +environment=unknown +target=unknown +core_ref=unknown +artifact=unknown +health=unknown +staging_path= +transaction_dir= +transaction_committed=0 +on_exit() { + local status=$? + if [[ -n ${transaction_dir:-} && ${transaction_committed:-0} != 1 ]]; then restore_transaction || true; fi + [[ -z ${staging_path:-} ]] || rm -rf -- "$staging_path" + if ((status != 0 && error_reported == 0)); then report FAILED no inspect-and-retry "$(rollback_available)" >&2; fi + return "$status" +} +trap on_exit EXIT + +usage() { + cat >&2 <<'EOF' +usage: install-deployer.sh --check|--install|--upgrade|--repair|--rollback|--drain|--uninstall --config /etc/ci-fleet-deployer/deployer.conf + +Modes are explicit and mutually exclusive. Configuration and credential references +are host-local; secret values are never accepted as arguments. +EOF +} +report() { + local result=$1 changed=$2 next=$3 rollback=${4:-no} + printf 'REPORT action=%s result=%s environment=%s target=%s version=%s digest=%s health=%s changed=%s rollback_available=%s next=%s\n' \ + "$action" "$result" "$environment" "$target" "$core_ref" "${artifact#*@}" "$health" "$changed" "$rollback" "$next" +} +rollback_available() { [[ -n ${previous_state:-} && -f ${previous_state:-/nonexistent} && -n ${previous_policy:-} && -f ${previous_policy:-/nonexistent} ]] && printf yes || printf no; } +die() { error_reported=1; printf 'ERROR: %s\n' "$*" >&2; report FAILED no inspect-and-retry "$(rollback_available)" >&2; exit 2; } +block() { error_reported=1; printf 'BLOCKED: %s\n' "$*" >&2; report BLOCKED no resolve-precondition "$(rollback_available)" >&2; exit 3; } + +while (($#)); do + case "$1" in + --check|--install|--upgrade|--repair|--rollback|--drain|--uninstall) + [[ -z "$mode" ]] || die 'select exactly one operating mode' + mode=${1#--}; action=$mode; shift ;; + --config) (($# >= 2)) || die '--config requires a value'; config=$2; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown argument: $1" ;; + esac +done +[[ -n "$mode" ]] || { usage; die 'an explicit operating mode is required'; } +[[ -n "$config" ]] || die '--config is required' +[[ -z "$root" || "$testing" == 1 ]] || die 'CI_FLEET_DEPLOYER_ROOT is test-only' +if [[ "$testing" == 1 ]]; then + [[ -n "$root" ]] || die 'test mode requires an alternate root' + effective_uid=${CI_FLEET_DEPLOYER_EUID_OVERRIDE:-$effective_uid} +else + [[ -z "$root" ]] || die 'alternate root is forbidden' + [[ -z ${CI_FLEET_DEPLOYER_EUID_OVERRIDE:-} ]] || die 'effective UID override is test-only' +fi +if [[ "$mode" != check && "$effective_uid" != 0 ]]; then die 'run this mode as root'; fi + +root_path() { printf '%s%s' "$root" "$1"; } +etc_root=$(root_path /etc/ci-fleet-deployer) +install_root=$(root_path /opt/ci-fleet-deployer) +releases=$install_root/releases +current=$install_root/current +state_root=$(root_path /var/lib/ci-fleet-deployer) +state_file=$state_root/install-state.json +active_policy=$state_root/active-policy.conf +previous_state=$state_root/last-known-good.json +previous_policy=$state_root/last-known-good-policy.conf +drained=$state_root/drained +active_operation=$state_root/active-operation +lock_root=$(root_path /var/lock/ci-fleet-deployer) +log_root=$(root_path /var/log/ci-fleet-deployer) +systemd_root=$(root_path /etc/systemd/system) +unit_source=$repo_root/deploy/deployer +unit_names=( + ci-fleet-deployer.service + ci-fleet-deployer-health.service ci-fleet-deployer-health.timer + ci-fleet-deployer-cleanup.service ci-fleet-deployer-cleanup.timer + ci-fleet-deployer-drain.service +) +timer_names=(ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer) +config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' + +expected_uid=0 +[[ "$testing" != 1 ]] || expected_uid=$(id -u) +inside() { + local path=$1 base=$2 normalized normalized_base + normalized=$(realpath -m -- "$path") + normalized_base=$(realpath -m -- "$base") + [[ "$normalized" == "$normalized_base/"* ]] +} +secure_file() { + local path=$1 description=$2 mode=${3:-600} + [[ ! -L "$path" && -f "$path" ]] || block "$description must be a regular file, not a symlink" + [[ $(realpath -e -- "$path") == $(realpath -m -- "$path") ]] || block "$description path contains a symlink" + [[ $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || block "$description must be owned by root with mode 0$mode" +} +secure_directory() { + local path=$1 mode=$2 create=${3:-0} + [[ ! -L "$path" ]] || die "unsafe symlinked managed directory: $path" + if [[ ! -e "$path" ]]; then + [[ "$create" == 1 ]] || return 1 + install -d -m "$mode" "$path" + fi + if [[ "$create" == 1 && -d "$path" && $(stat -c %u "$path") == "$expected_uid" ]]; then chmod "$mode" "$path"; fi + [[ -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || die "unsafe managed directory: $path" +} + +parse_file() { + local path=$1 prefix=$2 kind=$3 allowed=$4 line key value + declare -gA "$prefix=()" + local -n output=$prefix + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -n "$line" && ${line:0:1} != '#' ]] || continue + [[ "$line" == *=* ]] || block "malformed $kind line" + key=${line%%=*}; value=${line#*=} + [[ "$key" =~ ^[A-Z][A-Z0-9_]*$ && -n "$value" && "$value" != *$'\r'* && "$value" != *$'\n'* ]] || block "malformed $kind line" + [[ ! -v "output[$key]" ]] || block "duplicate $kind key: $key" + [[ " $allowed " == *" $key "* ]] || block "unknown $kind key: $key" + # key indexes a nameref to an associative array. + # shellcheck disable=SC2004 + output[$key]=$value + done <"$path" +} + +validate_config() { + inside "$config" "$etc_root" || block "configuration path must be inside $etc_root" + secure_directory "$etc_root" 700 0 || block 'configuration directory is missing' + secure_file "$config" 'configuration file' + parse_file "$config" cfg configuration "$config_keys" + local key + for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID; do + [[ -v "cfg[$key]" ]] || block "configuration is missing required key: $key" + done + [[ ${cfg[SCHEMA_VERSION]} == 1 ]] || block 'unsupported configuration schema' + environment=${cfg[ENVIRONMENT]}; target=${cfg[TARGET_ID]}; core_ref=${cfg[CORE_REF]:-unknown}; artifact=${cfg[ARTIFACT_IMAGE]:-unknown} + [[ "$environment" =~ ^[a-z][a-z0-9-]{0,31}$ ]] || block 'invalid explicit environment' + [[ "$target" =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'invalid explicit target identity' + if [[ "$mode" == drain || "$mode" == uninstall ]]; then return; fi + secure_directory "$etc_root/adapters" 700 0 || block 'adapter directory is missing' + secure_directory "$etc_root/credentials" 700 0 || block 'credential directory is missing' + secure_directory "$etc_root/evidence" 700 0 || block 'evidence directory is missing' + for key in SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE; do + [[ -v "cfg[$key]" ]] || block "configuration is missing required key: $key" + done + [[ ${cfg[DEPLOYER_IDENTITY]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'invalid deployer identity' + [[ ${cfg[CREDENTIAL_SCOPE]} == "$environment" ]] || block 'credential scope must exactly match the explicit environment' + [[ "$core_ref" =~ ^[0-9a-f]{40}$ && ${cfg[SOURCE_COMMIT]} =~ ^[0-9a-f]{40}$ ]] || block 'core and source revisions must be full lowercase commit SHAs' + [[ ${cfg[ARTIFACT_IMAGE]} =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?/[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || block 'artifact image must be an immutable qualified digest reference' + [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || block 'adapter digest must be lowercase SHA-256' + [[ ${cfg[NETWORK_HOST]} =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$ ]] || block 'invalid network prerequisite host' + [[ ${cfg[MIN_DISK_GIB]} =~ ^[1-9][0-9]{0,3}$ ]] || block 'MIN_DISK_GIB must be a positive integer' + [[ ${cfg[REQUIRE_COMPOSE]} == 0 || ${cfg[REQUIRE_COMPOSE]} == 1 ]] || block 'REQUIRE_COMPOSE must be 0 or 1' + inside "${cfg[ADAPTER_PATH]}" "$etc_root/adapters" || block 'adapter path is outside the approved adapter directory' + secure_file "${cfg[ADAPTER_PATH]}" 'adapter file' 700 + [[ $(sha256sum "${cfg[ADAPTER_PATH]}" | cut -d' ' -f1) == "${cfg[ADAPTER_SHA256]}" ]] || block 'adapter digest does not match the protected regular file' + case ${cfg[CREDENTIAL_PROVIDER]} in + file) + inside "${cfg[CREDENTIAL_REF]}" "$etc_root/credentials" || block 'credential reference is outside the approved credential directory' + [[ ! -L ${cfg[CREDENTIAL_REF]} && -f ${cfg[CREDENTIAL_REF]} ]] || block 'credential reference must be a regular file, not a symlink' + [[ $(stat -c '%u:%a' "${cfg[CREDENTIAL_REF]}") == "$expected_uid:600" ]] || block 'credential file must be owner-only mode 0600' + ;; + external) + [[ ${cfg[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || block 'invalid external secret-manager adapter reference' + ;; + *) block 'CREDENTIAL_PROVIDER must be file or external' ;; + esac + validate_evidence +} + +validate_evidence() { + local allowed='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' + inside "${cfg[APPROVAL_EVIDENCE_PATH]}" "$etc_root/evidence" || block 'approval evidence is outside the approved evidence directory' + secure_file "${cfg[APPROVAL_EVIDENCE_PATH]}" 'approval evidence' + parse_file "${cfg[APPROVAL_EVIDENCE_PATH]}" approval 'approval evidence' "$allowed" + local key + for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT; do + [[ -v "approval[$key]" ]] || block "approval evidence is missing $key" + done + [[ ${approval[SCHEMA_VERSION]} == 1 ]] || block 'unsupported approval evidence schema' + for key in ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do + [[ ${approval[$key]} == "${cfg[$key]}" ]] || block "approval evidence does not match exact $key" + done + [[ ${approval[APPROVAL_IDENTITY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${approval[POLICY_IDENTITY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${approval[APPROVAL_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || block 'approval identity is malformed' + [[ ${approval[APPROVED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || block 'approval timestamp must be UTC RFC3339' + case ${cfg[APPROVAL_PROVIDER]} in + manual-exact-head|external-exact-head) ;; + github-environment) + [[ -v 'cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]' ]] || block 'GitHub Environment approval requires capability evidence' + inside "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" "$etc_root/evidence" || block 'capability evidence is outside the approved evidence directory' + secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' + parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' 'SCHEMA_VERSION ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' + [[ ${capability[SCHEMA_VERSION]:-} == 1 && ${capability[ENVIRONMENT_PROTECTION]:-} == verified && ${capability[EXACT_HEAD]:-} == "${cfg[SOURCE_COMMIT]}" ]] || block 'GitHub Environment capability evidence is not exact-head verified' + ;; + *) block 'unsupported approval provider' ;; + esac + inside "${cfg[CHECKPOINT_EVIDENCE_PATH]}" "$etc_root/evidence" || block 'checkpoint evidence is outside the approved evidence directory' + secure_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" 'checkpoint evidence' + parse_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" checkpoint 'checkpoint evidence' 'SCHEMA_VERSION ENVIRONMENT TARGET_ID CHECKPOINT_ID RECORDED_AT' + [[ ${checkpoint[SCHEMA_VERSION]:-} == 1 && ${checkpoint[ENVIRONMENT]:-} == "$environment" && ${checkpoint[TARGET_ID]:-} == "$target" ]] || block 'checkpoint evidence does not match the explicit environment and target' + [[ ${checkpoint[CHECKPOINT_ID]:-} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${checkpoint[RECORDED_AT]:-} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || block 'checkpoint evidence is malformed' +} + +require_host() { + local command os_id os_version available required + for command in bash awk cut sort stat sha256sum readlink realpath install cmp mv cp rm mkdir mktemp chmod ln flock kill git python3 docker systemctl systemd-analyze systemd-inhibit timedatectl curl df date; do + command -v "$command" >/dev/null || block "$command is required" + done + local os_release + os_release=$(root_path /etc/os-release) + [[ -f "$os_release" && ! -L "$os_release" ]] || block 'supported Linux os-release metadata is missing' + os_id=$(awk -F= '$1=="ID" {gsub(/"/,"",$2); print $2}' "$os_release") + os_version=$(awk -F= '$1=="VERSION_ID" {gsub(/"/,"",$2); print $2}' "$os_release") + [[ "$os_id" == debian && "$os_version" =~ ^(12|13)(\.|$) || "$os_id" == ubuntu && "$os_version" =~ ^(22\.04|24\.04)$ ]] || block 'unsupported Linux distribution or release' + [[ -d $(root_path /run/systemd/system) ]] || block 'systemd is not the active init system' + systemctl is-system-running >/dev/null 2>&1 || block 'systemd is unavailable' + docker info >/dev/null 2>&1 || block 'Docker Engine is unavailable' + [[ ${cfg[REQUIRE_COMPOSE]} != 1 ]] || docker compose version >/dev/null 2>&1 || block 'Docker Compose v2 is required but unavailable' + [[ $(timedatectl show -p NTPSynchronized --value 2>/dev/null) == yes ]] || block 'host time is not synchronized' + available=$(df -Pk "$state_root" 2>/dev/null | awk 'NR==2 {print $4}') + [[ "$available" =~ ^[0-9]+$ ]] || available=$(df -Pk "$(dirname "$state_root")" | awk 'NR==2 {print $4}') + required=$((cfg[MIN_DISK_GIB] * 1024 * 1024)) + ((available >= required)) || block 'insufficient deployer disk capacity' + if [[ "$testing" != 1 || ${CI_FLEET_DEPLOYER_TEST_NETWORK:-} != ok ]]; then + printf '%s\n' "${cfg[NETWORK_HOST]}" | python3 -c 'import socket,sys; socket.getaddrinfo(sys.stdin.readline().strip(), 443)' >/dev/null 2>&1 || block 'network prerequisite DNS lookup failed' + fi + printf 'url = "https://%s/"\nconnect-timeout = 5\nmax-time = 10\nhead\nsilent\n' "${cfg[NETWORK_HOST]}" | curl --config - >/dev/null 2>&1 || block 'network prerequisite HTTPS check failed' + reject_mixed_role +} + +require_maintenance_host() { + local command + for command in bash awk stat readlink realpath install cp rm mkdir chmod mv flock kill systemctl date; do + command -v "$command" >/dev/null || block "$command is required for maintenance" + done + [[ -d "$systemd_root" && ! -L "$systemd_root" ]] || block 'systemd unit directory is unavailable' +} + +reject_mixed_role() { + local unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" + for unit in ci-fleet-health.service ci-fleet-reconcile.service ci-fleet-cleanup.service actions.runner.service; do + [[ ! -e "$systemd_root/$unit" ]] || block 'ordinary CI controller or runner state is present' + done + for path in "$(root_path /etc/ci-fleet/ci-fleet.env)" "$(root_path /opt/ci-fleet/current)" "$(root_path /var/lib/ci-fleet/install-state.json)"; do + [[ ! -e "$path" && ! -L "$path" ]] || block 'ordinary CI controller or runner state is present' + done + output=$(docker ps -a --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || block 'Docker workload inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || block 'unrelated Docker workload is present'; done <<<"$output" + output=$(docker network ls --filter type=custom --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || block 'Docker network inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || block 'incompatible custom Docker network is present'; done <<<"$output" + output=$(docker volume ls --format '{{.Name}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || block 'Docker volume inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || block 'incompatible Docker volume is present'; done <<<"$output" +} + +validate_checkout() { + local head + head=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') || block 'installer checkout is not Git-authored' + [[ "$head" == "$core_ref" ]] || block 'CORE_REF must equal the exact reviewed checkout HEAD' + if [[ "$testing" != 1 ]]; then + git -C "$repo_root" diff --quiet HEAD -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer || block 'reviewed deployer inputs differ from HEAD' + fi +} + +active_deployment() { + local pid started now + [[ -f "$active_operation" && ! -L "$active_operation" ]] || return 1 + pid=$(awk -F= '$1=="pid" {print $2}' "$active_operation") + started=$(awk -F= '$1=="started_at" {print $2}' "$active_operation") + [[ "$pid" =~ ^[1-9][0-9]*$ && "$started" =~ ^[0-9]+$ ]] || return 0 + if kill -0 "$pid" 2>/dev/null; then return 0; fi + now=$(date +%s) + ((now - started <= 3600)) +} + +release_complete() { + local release=$1 stored actual unit entry + [[ -d "$release" && ! -L "$release" && $(stat -c '%u:%a' "$release") == "$expected_uid:755" && -x "$release/scripts/install-deployer.sh" && -x "$release/scripts/deployer-runtime.sh" ]] || return 1 + for entry in "$release/scripts/install-deployer.sh" "$release/scripts/deployer-runtime.sh"; do [[ ! -L "$entry" && $(stat -c '%u:%a' "$entry") == "$expected_uid:755" ]] || return 1; done + [[ -f "$release/.ci-fleet-tree-sha256" ]] || return 1 + stored=$(<"$release/.ci-fleet-tree-sha256") + actual=$(cd "$release" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1) + [[ "$stored" == "$actual" ]] || return 1 + for unit in "${unit_names[@]}"; do [[ -f "$release/deploy/deployer/$unit" && ! -L "$release/deploy/deployer/$unit" && $(stat -c '%u:%a' "$release/deploy/deployer/$unit") == "$expected_uid:644" ]] || return 1; done +} + +state_matches() { + [[ -f "$state_file" && ! -L "$state_file" && $(stat -c '%u:%a' "$state_file") == "$expected_uid:600" ]] || return 1 + [[ -f "$active_policy" && ! -L "$active_policy" && $(stat -c '%u:%a' "$active_policy") == "$expected_uid:600" && $(cmp -s "$config" "$active_policy"; echo $?) == 0 ]] || return 1 + printf '%s\n' "$core_ref" "$environment" "$target" "${cfg[DEPLOYER_IDENTITY]}" "${cfg[SOURCE_COMMIT]}" "$artifact" "${approval[APPROVAL_ID]}" "${approval[POLICY_IDENTITY]}" "${cfg[APPROVAL_PROVIDER]}" "${checkpoint[CHECKPOINT_ID]}" | python3 -c ' +import json, sys +try: value=json.load(open(sys.argv[1], encoding="utf-8")) +except (OSError, ValueError): raise SystemExit(1) +keys=("core_ref","environment","target","deployer_identity","source_commit","artifact","approval_id","policy_identity","approval_provider","checkpoint_id") +expected=[line.rstrip("\n") for line in sys.stdin] +raise SystemExit(0 if all(value.get(k)==v for k,v in zip(keys,expected)) else 1) +' "$state_file" +} + +units_match() { + local unit + for unit in "${unit_names[@]}"; do + [[ -f "$systemd_root/$unit" && ! -L "$systemd_root/$unit" ]] || return 1 + cmp -s "$unit_source/$unit" "$systemd_root/$unit" || return 1 + done + for unit in "${timer_names[@]}"; do systemctl is-enabled "$unit" >/dev/null 2>&1 && systemctl is-active "$unit" >/dev/null 2>&1 || return 1; done +} + +current_matches() { + local target_path=$releases/$core_ref + [[ -L "$current" && $(readlink -f "$current") == $(readlink -f "$target_path") ]] || return 1 + release_complete "$target_path" +} + +managed_boundaries_match() { + local path mode + for path in "$install_root:755" "$releases:755" "$state_root:700" "$lock_root:700" "$log_root:700"; do + mode=${path##*:}; path=${path%:*} + [[ -d "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || return 1 + done +} + +converged() { managed_boundaries_match && current_matches && state_matches && units_match; } + +acquire_lock() { + local path + secure_directory "$lock_root" 700 1 + exec 9<"$lock_root" + flock -n 9 || block 'another deployer installer operation is running' + if [[ -e "$active_operation" ]] && ! active_deployment; then + [[ ! -L "$active_operation" && -f "$active_operation" && $(stat -c '%u:%a' "$active_operation") == "$expected_uid:600" ]] || block 'stale operation state is unsafe' + rm -f "$active_operation" + fi + if [[ -d "$releases" ]]; then + shopt -s nullglob + for path in "$releases"/."$core_ref".staging.*; do + [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:755" ]] || block 'interrupted release staging state is unsafe' + rm -rf -- "$path" + done + shopt -u nullglob + fi + recover_interrupted_transaction +} + +begin_transaction() { + local name path current_target + for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + path=$state_root/$name + [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:600" ]] || block 'managed transaction state has an unsafe type, owner, or mode' + done + if [[ -L "$current" ]]; then + current_target=$(readlink "$current") + [[ "$current_target" =~ ^releases/[0-9a-f]{40}$ ]] || block 'current release pointer is unsafe' + elif [[ -e "$current" ]]; then block 'current release pointer has an unsafe type' + fi + for name in "${unit_names[@]}"; do + path=$systemd_root/$name + [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c %u "$path") == "$expected_uid" ]] || block 'managed systemd unit has an unsafe owner or type' + done + transaction_dir=$(mktemp -d "$state_root/.transaction.XXXXXX") + chmod 0700 "$transaction_dir" + install -d -m 0700 "$transaction_dir/units" "$transaction_dir/state" + for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + path=$state_root/$name + if [[ -f "$path" ]]; then + install -m 0600 "$path" "$transaction_dir/state/$name" + printf '%s\n' "$name" >>"$transaction_dir/state-present" + fi + done + [[ -z ${current_target:-} ]] || printf '%s\n' "$current_target" >"$transaction_dir/current-target" + for name in "${unit_names[@]}"; do + path=$systemd_root/$name + if [[ -f "$path" ]]; then + install -m 0644 "$path" "$transaction_dir/units/$name" + printf '%s\n' "$name" >>"$transaction_dir/units-present" + fi + done + for name in "${timer_names[@]}"; do + if systemctl is-enabled "$name" >/dev/null 2>&1; then printf '%s\n' "$name" >>"$transaction_dir/timers-enabled"; fi + done +} + +restore_transaction() { + local name target_value + [[ -n ${transaction_dir:-} && -d $transaction_dir ]] || return 0 + transaction_committed=1 + systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true + for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name"; done + if [[ -f "$transaction_dir/units-present" ]]; then + while IFS= read -r name; do + [[ " ${unit_names[*]} " == *" $name "* && -f "$transaction_dir/units/$name" && ! -L "$transaction_dir/units/$name" ]] || block 'transaction unit manifest is unsafe' + install -m 0644 "$transaction_dir/units/$name" "$systemd_root/$name" + done <"$transaction_dir/units-present" + fi + for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do rm -f -- "$state_root/$name"; done + if [[ -f "$transaction_dir/state-present" ]]; then + while IFS= read -r name; do + [[ " install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf " == *" $name "* && -f "$transaction_dir/state/$name" && ! -L "$transaction_dir/state/$name" ]] || block 'transaction state manifest is unsafe' + install -m 0600 "$transaction_dir/state/$name" "$state_root/$name" + done <"$transaction_dir/state-present" + fi + rm -f -- "$current" "$install_root/.current.new" "$state_root/.install-state.new" "$active_policy.new" "$state_file.new" + if [[ -f "$transaction_dir/current-target" ]]; then + target_value=$(<"$transaction_dir/current-target") + [[ "$target_value" =~ ^releases/[0-9a-f]{40}$ ]] || block 'transaction current pointer is unsafe' + ln -s "$target_value" "$current" + fi + systemctl daemon-reload >/dev/null 2>&1 || true + if [[ -f "$transaction_dir/timers-enabled" ]]; then + while IFS= read -r name; do + [[ " ${timer_names[*]} " == *" $name "* ]] || block 'transaction timer manifest is unsafe' + systemctl enable --now "$name" >/dev/null 2>&1 || true + done <"$transaction_dir/timers-enabled" + fi + rm -rf -- "$transaction_dir" + transaction_dir= +} + +recover_interrupted_transaction() { + local candidates=() candidate + [[ -d "$state_root" ]] || return 0 + shopt -s nullglob + candidates=("$state_root"/.transaction.*) + shopt -u nullglob + ((${#candidates[@]} <= 1)) || block 'multiple interrupted installer transactions require operator recovery' + ((${#candidates[@]} == 1)) || return 0 + candidate=${candidates[0]} + [[ ! -L "$candidate" && -d "$candidate" && $(stat -c '%u:%a' "$candidate") == "$expected_uid:700" ]] || block 'interrupted installer transaction is unsafe' + transaction_dir=$candidate + restore_transaction + transaction_committed=0 +} + +commit_transaction() { + transaction_committed=1 + rm -rf -- "$transaction_dir" + transaction_dir= +} + +atomic_replace_directory() { + local replacement=$1 target_path=$2 + if [[ ! -e "$target_path" && ! -L "$target_path" ]]; then mv "$replacement" "$target_path"; return; fi + [[ ! -L "$target_path" && -d "$target_path" ]] || block 'managed release target has an unsafe type' + python3 - "$replacement" "$target_path" <<'PY' +import ctypes, os, sys +source, target = map(os.fsencode, sys.argv[1:]) +libc = ctypes.CDLL(None, use_errno=True) +renameat2 = getattr(libc, 'renameat2', None) +if renameat2 is None: + raise OSError('atomic directory exchange is unavailable') +renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] +if renameat2(-100, source, -100, target, 2) != 0: + error = ctypes.get_errno(); raise OSError(error, os.strerror(error)) +fd = os.open(os.path.dirname(target), os.O_RDONLY | os.O_DIRECTORY) +try: os.fsync(fd) +finally: os.close(fd) +PY +} + +install_release() { + local release=$releases/$core_ref staging + secure_directory "$install_root" 755 1 + secure_directory "$releases" 755 1 + if release_complete "$release"; then return; fi + staging=$(mktemp -d "$releases/.${core_ref}.staging.XXXXXX") + chmod 0755 "$staging" + staging_path=$staging + install -d -m 0755 "$staging/scripts" "$staging/deploy/deployer" + install -m 0755 "$repo_root/scripts/install-deployer.sh" "$repo_root/scripts/deployer-runtime.sh" "$staging/scripts/" + install -m 0644 "$unit_source"/* "$staging/deploy/deployer/" + (cd "$staging" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1) >"$staging/.ci-fleet-tree-sha256" + chmod 0644 "$staging/.ci-fleet-tree-sha256" + release_complete "$staging" || die 'staged deployer release is incomplete' + atomic_replace_directory "$staging" "$release" + [[ ! -e "$staging" ]] || rm -rf -- "$staging" + staging_path= +} + +write_state() { + local destination=$1 temporary + temporary=$(mktemp "$state_root/.state.XXXXXX") + printf '%s\n' "$core_ref" "$environment" "$target" "${cfg[DEPLOYER_IDENTITY]}" "${cfg[SOURCE_COMMIT]}" "$artifact" "${approval[APPROVAL_ID]}" "${approval[APPROVAL_IDENTITY]}" "${approval[POLICY_IDENTITY]}" "${cfg[APPROVAL_PROVIDER]}" "${checkpoint[CHECKPOINT_ID]}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" | python3 -c ' +import json,sys +keys=("core_ref","environment","target","deployer_identity","source_commit","artifact","approval_id","approval_identity","policy_identity","approval_provider","checkpoint_id","installed_at") +values=[line.rstrip("\n") for line in sys.stdin] +with open(sys.argv[1],"w",encoding="utf-8") as f: json.dump(dict(zip(keys,values)),f,indent=2,sort_keys=True); f.write("\n") +' "$temporary" + chmod 0600 "$temporary" + mv -Tf "$temporary" "$destination" +} + +install_units() { + local unit systemd_mode + [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' + systemd_mode=$(stat -c %a "$systemd_root") + (((8#$systemd_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' + for unit in "${unit_names[@]}"; do + if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then + [[ ! -L "$systemd_root/$unit" && -f "$systemd_root/$unit" && $(stat -c %u "$systemd_root/$unit") == "$expected_uid" ]] || block 'managed systemd unit has an unsafe owner or type' + fi + install -m 0644 "$unit_source/$unit" "$systemd_root/$unit" + done + systemd-analyze verify "${unit_names[@]/#/$systemd_root/}" >/dev/null || die 'systemd unit verification failed' + systemctl daemon-reload + systemctl enable --now "${timer_names[@]}" >/dev/null +} + +policy_adapter_operation() { + local policy=$1 operation_name=$2 description=$3 key + local -A policy_cfg=() + secure_file "$policy" "$description" + parse_file "$policy" policy_cfg "$description" "$config_keys" + for key in ADAPTER_PATH ADAPTER_SHA256; do [[ -v "policy_cfg[$key]" ]] || die "$description is missing $key"; done + [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" + secure_file "${policy_cfg[ADAPTER_PATH]}" "$description adapter" 700 + [[ $(sha256sum "${policy_cfg[ADAPTER_PATH]}" | cut -d' ' -f1) == "${policy_cfg[ADAPTER_SHA256]}" ]] || die "$description adapter digest mismatch" + CI_FLEET_DEPLOYER_CONFIG="$policy" "${policy_cfg[ADAPTER_PATH]}" "$operation_name" >/dev/null 2>&1 +} + +perform_check() { + if active_deployment; then block 'active deployment prevents a consistent check'; fi + converged || block 'installed deployer state is absent or drifted' + policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer health check failed' + health=healthy + report NO_CHANGE no none "$(rollback_available)" +} + +perform_converge() { + local had_state=0 old_environment old_target candidate_changed=1 old_release + [[ -f "$state_file" ]] && had_state=1 + if ((had_state)); then + read -r old_environment old_target < <(python3 - "$state_file" <<'PY' +import json,sys +v=json.load(open(sys.argv[1])); print(v.get('environment',''),v.get('target','')) +PY +) + [[ "$old_environment" == "$environment" && "$old_target" == "$target" ]] || block 'installed environment and target identity cannot change in place' + if [[ "$mode" == install && -L "$current" ]] && ! state_matches; then block 'install cannot select a new candidate; use --upgrade or --repair'; fi + elif [[ "$mode" == upgrade ]]; then + block '--upgrade requires an existing installation' + fi + if state_matches; then candidate_changed=0; fi + if [[ "$mode" == upgrade && ! -L "$current" ]]; then block '--upgrade requires an active installation; use --install after uninstall'; fi + if active_deployment; then block 'active deployment prevents this operation'; fi + if converged; then + policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer health check failed' + health=healthy; report NO_CHANGE no none "$(rollback_available)"; return + fi + if [[ -L "$current" ]]; then + old_release=$(readlink -f "$current") + release_complete "$old_release" || block 'active deployer release is incomplete' + policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement' + fi + CI_FLEET_DEPLOYER_CONFIG="$config" "${cfg[ADAPTER_PATH]}" validate >/dev/null 2>&1 || die 'candidate adapter validation failed' + install_release + secure_directory "$state_root" 700 1 + secure_directory "$log_root" 700 1 + begin_transaction + if ((had_state && candidate_changed)); then + install -m 0600 "$state_file" "$previous_state" + install -m 0600 "$active_policy" "$previous_policy" + fi + ln -sfn "releases/$core_ref" "$install_root/.current.new" + install_units + install -m 0600 "$config" "$active_policy.new" + write_state "$state_root/.install-state.new" + mv -Tf "$active_policy.new" "$active_policy" + mv -Tf "$state_root/.install-state.new" "$state_file" + mv -Tf "$install_root/.current.new" "$current" + rm -f "$drained" + policy_adapter_operation "$active_policy" health 'candidate policy' || die 'candidate health check failed after activation' + commit_transaction + health=healthy + report CHANGED yes run-check "$(rollback_available)" +} + +perform_rollback() { + active_deployment && block 'active deployment prevents rollback' + [[ -f "$previous_state" && -f "$previous_policy" ]] || block 'no last-known-good release is available' + secure_directory "$state_root" 700 0 + begin_transaction + install -m 0600 "$previous_state" "$state_file.new" + install -m 0600 "$previous_policy" "$active_policy.new" + core_ref=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["core_ref"])' "$previous_state") + environment=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["environment"])' "$previous_state") + target=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["target"])' "$previous_state") + artifact=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["artifact"])' "$previous_state") + release_complete "$releases/$core_ref" || die 'last-known-good release is incomplete' + policy_adapter_operation "$previous_policy" rollback 'last-known-good policy' || die 'application adapter rollback failed' + ln -sfn "releases/$core_ref" "$install_root/.current.new" + install_units + mv -Tf "$active_policy.new" "$active_policy" + mv -Tf "$state_file.new" "$state_file" + mv -Tf "$install_root/.current.new" "$current" + policy_adapter_operation "$active_policy" health 'rolled-back policy' || die 'rolled-back deployer health check failed' + commit_transaction + health=healthy + report CHANGED yes restore-host-policy-evidence-then-check yes +} + +perform_drain() { + if active_deployment; then block 'active deployment prevents drain'; fi + secure_directory "$state_root" 700 1 + if [[ -f "$drained" ]]; then report NO_CHANGE no safe-to-maintain "$(rollback_available)"; return; fi + : >"$drained" + chmod 0600 "$drained" + report CHANGED yes safe-to-maintain "$(rollback_available)" +} + +perform_uninstall() { + local changed=no unit managed_present=no + if [[ -e "$state_root" || -L "$state_root" || -e "$lock_root" || -L "$lock_root" || -e "$current" || -L "$current" ]]; then managed_present=yes; fi + for unit in "${unit_names[@]}"; do [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/$unit" ]] || managed_present=yes; done + if [[ "$managed_present" == no ]]; then report NO_CHANGE no retained-state "$(rollback_available)"; return; fi + if active_deployment; then block 'active deployment prevents this operation'; fi + acquire_lock + secure_directory "$state_root" 700 1 + : >"$drained" + chmod 0600 "$drained" + if active_deployment; then block 'active deployment started while draining'; fi + if [[ -L "$current" ]]; then rm -f "$current"; changed=yes; fi + systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true + for unit in "${unit_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then rm -f "$systemd_root/$unit"; changed=yes; fi; done + systemctl daemon-reload >/dev/null 2>&1 || true + rm -f "$drained" "$active_operation" + if [[ "$changed" == yes ]]; then report CHANGED yes retained-state "$(rollback_available)"; else report NO_CHANGE no retained-state "$(rollback_available)"; fi +} + +validate_config +if [[ "$mode" == drain || "$mode" == uninstall ]]; then + require_maintenance_host +else + validate_checkout + require_host +fi +case "$mode" in + check) perform_check ;; + install|upgrade|repair) acquire_lock; perform_converge ;; + rollback) acquire_lock; perform_rollback ;; + drain) acquire_lock; perform_drain ;; + uninstall) perform_uninstall ;; +esac diff --git a/scripts/test-deployer-units.sh b/scripts/test-deployer-units.sh new file mode 100755 index 0000000..0ec0cfd --- /dev/null +++ b/scripts/test-deployer-units.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +command -v systemd-analyze >/dev/null || { printf 'systemd-analyze is required\n' >&2; exit 1; } +[[ -d /usr/lib/systemd/system ]] || { printf 'systemd unit library is missing\n' >&2; exit 1; } +root=$(mktemp -d) +trap 'rm -rf "$root"' EXIT +install -d -m 0755 "$root/etc/systemd/system" \ + "$root/opt/ci-fleet-deployer/current/scripts" "$root/usr/lib/systemd" +cp -a /usr/lib/systemd/system "$root/usr/lib/systemd/" +install -m 0755 "$repo_root/scripts/deployer-runtime.sh" \ + "$root/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh" +install -m 0644 "$repo_root"/deploy/deployer/* "$root/etc/systemd/system/" +systemd-analyze verify --root="$root" \ + ci-fleet-deployer.service \ + ci-fleet-deployer-health.service ci-fleet-deployer-health.timer \ + ci-fleet-deployer-cleanup.service ci-fleet-deployer-cleanup.timer \ + ci-fleet-deployer-drain.service +printf 'DEPLOYER_UNIT_TESTS_OK\n' diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh new file mode 100755 index 0000000..36626e1 --- /dev/null +++ b/scripts/test-install-deployer.sh @@ -0,0 +1,430 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +installer=$repo_root/scripts/install-deployer.sh +runtime=$repo_root/scripts/deployer-runtime.sh + +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +expect_success() { + local output + output=$("$@" 2>&1) || fail "expected success: $*; output=$output" + printf '%s\n' "$output" +} +expect_failure() { + local expected=$1 output + shift + if output=$("$@" 2>&1); then fail "expected failure: $*"; fi + grep -Fq -- "$expected" <<<"$output" || fail "missing [$expected]: $output" + printf '%s\n' "$output" +} + +[[ -x "$installer" && -x "$runtime" ]] || fail 'deployer installer/runtime is missing' +expect_failure 'an explicit operating mode is required' "$installer" + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +root=$tmp/root +fake_bin=$tmp/bin +mkdir -p "$fake_bin" "$root/etc/ci-fleet-deployer/adapters" \ + "$root/etc/ci-fleet-deployer/credentials" "$root/etc/ci-fleet-deployer/evidence" \ + "$root/etc/systemd/system" "$root/run/systemd/system" +printf 'ID=debian\nVERSION_ID="12"\n' >"$root/etc/os-release" +chmod 0700 "$root/etc/ci-fleet-deployer" "$root/etc/ci-fleet-deployer/adapters" \ + "$root/etc/ci-fleet-deployer/credentials" "$root/etc/ci-fleet-deployer/evidence" + +cat >"$fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +set -u +case "${1:-}" in + info) exit "${FAKE_DOCKER_INFO_EXIT:-0}" ;; + compose) [[ "${2:-}" == version ]] && exit "${FAKE_COMPOSE_EXIT:-0}" ;; + ps) [[ -z "${FAKE_DOCKER_PS:-}" ]] || printf '%s\n' "$FAKE_DOCKER_PS" ;; + network) [[ "${2:-}" == ls ]] && { [[ -z "${FAKE_DOCKER_NETWORKS:-}" ]] || printf '%s\n' "$FAKE_DOCKER_NETWORKS"; exit 0; } ;; + volume) [[ "${2:-}" == ls ]] && { [[ -z "${FAKE_DOCKER_VOLUMES:-}" ]] || printf '%s\n' "$FAKE_DOCKER_VOLUMES"; exit 0; } ;; +esac +exit 0 +EOF +cat >"$fake_bin/systemctl" <<'EOF' +#!/usr/bin/env bash +set -u +root=${CI_FLEET_DEPLOYER_ROOT:-} +log=${FAKE_SYSTEMCTL_LOG:-/dev/null} +printf '%s\n' "$*" >>"$log" +case "${1:-}" in + is-system-running) [[ -z "${FAKE_SYSTEMD_FAIL:-}" ]] || exit 1; printf 'running\n' ;; + is-enabled|is-active) [[ -e "$root/var/lib/ci-fleet-deployer/unit-${2:-}" ]] ;; + enable) + shift + [[ "${1:-}" != --now ]] || shift + for unit in "$@"; do : >"$root/var/lib/ci-fleet-deployer/unit-$unit"; done ;; + disable) + shift + [[ "${1:-}" != --now ]] || shift + for unit in "$@"; do rm -f "$root/var/lib/ci-fleet-deployer/unit-$unit"; done ;; +esac +exit 0 +EOF +cat >"$fake_bin/systemd-analyze" <<'EOF' +#!/usr/bin/env bash +[[ "${1:-}" == verify ]] || exit 1 +exit "${FAKE_SYSTEMD_VERIFY_EXIT:-0}" +EOF +cat >"$fake_bin/timedatectl" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "${FAKE_TIME_SYNC:-yes}" +EOF +cat >"$fake_bin/curl" <<'EOF' +#!/usr/bin/env bash +cat >/dev/null +exit "${FAKE_CURL_EXIT:-0}" +EOF +cat >"$fake_bin/systemd-inhibit" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +while (($#)); do + [[ "$1" != -- ]] || { shift; exec "$@"; } + shift +done +exit 2 +EOF +cat >"$fake_bin/df" <<'EOF' +#!/usr/bin/env bash +printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\nfixture 104857600 1 %s 1%% /\n' "${FAKE_DISK_AVAILABLE:-104857599}" +EOF +chmod 0755 "$fake_bin"/* +export PATH="$fake_bin:$PATH" +export FAKE_SYSTEMCTL_LOG=$tmp/systemctl.log +export CI_FLEET_DEPLOYER_TESTING=1 CI_FLEET_DEPLOYER_ROOT=$root +export CI_FLEET_DEPLOYER_EUID_OVERRIDE=0 CI_FLEET_DEPLOYER_TEST_NETWORK=ok + +adapter=$root/etc/ci-fleet-deployer/adapters/application-adapter +cat >"$adapter" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail +printf '%s\n' "$1" >>"${FAKE_ADAPTER_LOG:?}" +if [[ -e "${FAKE_ADAPTER_FAIL:-/nonexistent}" ]]; then + fail_operation=$(<"$FAKE_ADAPTER_FAIL") + [[ "$fail_operation" != all && "$fail_operation" != "$1" ]] || exit 42 +fi +case "$1" in validate|health|cleanup|deploy|rollback) ;; *) exit 2 ;; esac +EOF +chmod 0700 "$adapter" +export FAKE_ADAPTER_LOG=$tmp/adapter.log +credential=$root/etc/ci-fleet-deployer/credentials/application.credential +printf 'CANARY_SECRET_VALUE_DO_NOT_PRINT\n' >"$credential" +chmod 0600 "$credential" +approval=$root/etc/ci-fleet-deployer/evidence/approval.conf +checkpoint=$root/etc/ci-fleet-deployer/evidence/checkpoint.conf +core_ref=$(git -C "$repo_root" rev-parse HEAD) +source_ref=1111111111111111111111111111111111111111 +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence() { + local evidence_environment=${1:-staging} evidence_target=${2:-example-staging} + cat >"$approval" <"$checkpoint" <"$root/etc/ci-fleet-deployer/deployer.conf" <"$root/etc/os-release" +expect_failure 'unsupported Linux distribution or release' "$installer" --check --config "$config" >/dev/null +cp "$tmp/os-release" "$root/etc/os-release" +FAKE_TIME_SYNC=no expect_failure 'host time is not synchronized' "$installer" --check --config "$config" >/dev/null +FAKE_SYSTEMD_FAIL=1 expect_failure 'systemd is unavailable' "$installer" --check --config "$config" >/dev/null +FAKE_DOCKER_INFO_EXIT=1 expect_failure 'Docker Engine is unavailable' "$installer" --check --config "$config" >/dev/null +CI_FLEET_DEPLOYER_TEST_NETWORK=fail expect_failure 'network prerequisite DNS lookup failed' "$installer" --check --config "$config" >/dev/null +FAKE_CURL_EXIT=1 expect_failure 'network prerequisite HTTPS check failed' "$installer" --check --config "$config" >/dev/null +FAKE_DISK_AVAILABLE=1 expect_failure 'insufficient deployer disk capacity' "$installer" --check --config "$config" >/dev/null +FAKE_COMPOSE_EXIT=1 expect_failure 'Docker Compose v2 is required but unavailable' "$installer" --check --config "$config" >/dev/null + +chmod 0755 "$root/etc/ci-fleet-deployer" +expect_failure 'unsafe managed directory' "$installer" --check --config "$config" >/dev/null +chmod 0700 "$root/etc/ci-fleet-deployer" +mv "$root/etc/ci-fleet-deployer/adapters" "$root/etc/ci-fleet-deployer/adapters.real" +ln -s adapters.real "$root/etc/ci-fleet-deployer/adapters" +expect_failure 'unsafe symlinked managed directory' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/ci-fleet-deployer/adapters" +mv "$root/etc/ci-fleet-deployer/adapters.real" "$root/etc/ci-fleet-deployer/adapters" + +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ARTIFACT_IMAGE=registry.example.invalid/example/app@sha256:', 'ARTIFACT_IMAGE=registry.example.invalid/example/app:latest#')) +PY +expect_failure 'artifact image must be an immutable' "$installer" --check --config "$config" >/dev/null +write_config + +capability=$root/etc/ci-fleet-deployer/evidence/github-capability.conf +python3 - "$config" "$capability" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=manual-exact-head', 'APPROVAL_PROVIDER=github-environment') + 'APPROVAL_CAPABILITY_EVIDENCE_PATH='+sys.argv[2]+'\n') +PY +expect_failure 'GitHub capability evidence must be a regular file' "$installer" --check --config "$config" >/dev/null +cat >"$capability" </dev/null +write_config +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external').replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=external:example-vault:staging/deployer')) +PY +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +write_config + +printf 'UNKNOWN=value\n' >"$tmp/unknown.conf"; chmod 0600 "$tmp/unknown.conf" +expect_failure 'configuration path must be inside' "$installer" --check --config "$tmp/unknown.conf" >/dev/null +printf 'UNKNOWN=value\n' >"$config" +expect_failure 'unknown configuration key: UNKNOWN' "$installer" --check --config "$config" >/dev/null +write_config +printf 'SCHEMA_VERSION=1\n' >>"$config" +expect_failure 'duplicate configuration key: SCHEMA_VERSION' "$installer" --check --config "$config" >/dev/null +write_config + +export CI_FLEET_DEPLOYER_EUID_OVERRIDE=1000 +expect_failure 'run this mode as root' "$installer" --install --config "$config" >/dev/null +export CI_FLEET_DEPLOYER_EUID_OVERRIDE=0 + +before=$(find "$root" -printf '%P %y %m\n' | sort | sha256sum) +expect_failure 'result=BLOCKED' "$installer" --check --config "$config" >/dev/null +after=$(find "$root" -printf '%P %y %m\n' | sort | sha256sum) +[[ "$before" == "$after" ]] || fail '--check changed the test host' +fresh_uninstall=$(expect_success "$installer" --uninstall --config "$config") +[[ "$after" == "$(find "$root" -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'fresh uninstall mutated an unmanaged host' +grep -Fq 'result=NO_CHANGE' <<<"$fresh_uninstall" || fail 'fresh uninstall did not report NO_CHANGE' + +first=$(expect_success "$installer" --install --config "$config") +grep -Fq 'REPORT action=install result=CHANGED environment=staging' <<<"$first" || fail 'fresh install report is incomplete' +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'fresh install lacks atomic current release' +[[ $(stat -c %a "$root/var/lib/ci-fleet-deployer/install-state.json") == 600 ]] || fail 'install state mode is not 0600' +for unit in ci-fleet-deployer.service ci-fleet-deployer-health.service ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.service ci-fleet-deployer-cleanup.timer ci-fleet-deployer-drain.service; do + [[ -f "$root/etc/systemd/system/$unit" ]] || fail "missing unit $unit" +done +second=$(expect_success "$installer" --install --config "$config") +grep -Fq 'result=NO_CHANGE' <<<"$second" || fail 'second install was not idempotent' +check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'REPORT action=check result=NO_CHANGE' <<<"$check" || fail 'check did not report convergence' + +interrupted=$root/var/lib/ci-fleet-deployer/.transaction.interrupted +mkdir -m 0700 "$interrupted" "$interrupted/units" "$interrupted/state" +for name in install-state.json active-policy.conf; do + cp "$root/var/lib/ci-fleet-deployer/$name" "$interrupted/state/$name" + printf '%s\n' "$name" >>"$interrupted/state-present" +done +for path in "$root"/etc/systemd/system/ci-fleet-deployer*; do + name=${path##*/}; cp "$path" "$interrupted/units/$name"; printf '%s\n' "$name" >>"$interrupted/units-present" +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$interrupted/current-target" +printf '%s\n' ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer >"$interrupted/timers-enabled" +printf 'interrupted\n' >>"$root/var/lib/ci-fleet-deployer/install-state.json" +printf 'interrupted\n' >>"$root/etc/systemd/system/ci-fleet-deployer.service" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$interrupted" ]] || fail 'interrupted transaction was not recovered' +expect_success "$installer" --check --config "$config" >/dev/null + +rm "$root/etc/systemd/system/ci-fleet-deployer-health.timer" +expect_failure 'result=BLOCKED' "$installer" --check --config "$config" >/dev/null +repair=$(expect_success "$installer" --repair --config "$config") +grep -Fq 'result=CHANGED' <<<"$repair" || fail 'repair did not report a change' +[[ -f "$root/etc/systemd/system/ci-fleet-deployer-health.timer" ]] || fail 'repair did not restore unit drift' + +transaction_state=$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json") +printf '# transaction-checkpoint\n' >>"$root/etc/systemd/system/ci-fleet-deployer.service" +transaction_unit=$(sha256sum "$root/etc/systemd/system/ci-fleet-deployer.service") +FAKE_SYSTEMD_VERIFY_EXIT=1 expect_failure 'systemd unit verification failed' "$installer" --repair --config "$config" >/dev/null +[[ "$transaction_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'failed transaction replaced healthy state' +[[ "$transaction_unit" == "$(sha256sum "$root/etc/systemd/system/ci-fleet-deployer.service")" ]] || fail 'failed transaction did not restore units' +if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'failed transaction left recovery residue'; fi + +stale_stage="$root/opt/ci-fleet-deployer/releases/.${core_ref}.staging.interrupted" +mkdir "$stale_stage"; chmod 0755 "$stale_stage" +printf 'pid=999999\nstarted_at=1\n' >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$stale_stage" && ! -e "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'bounded stale transaction recovery did not converge' + +exec 8<"$root/var/lock/ci-fleet-deployer" +flock -n 8 || fail 'fixture could not acquire installer lock' +expect_failure 'another deployer installer operation is running' "$installer" --repair --config "$config" >/dev/null +flock -u 8; exec 8>&- + +mv "$root/var/lock/ci-fleet-deployer" "$root/var/lock/ci-fleet-deployer.real" +ln -s "$root/var/lock/ci-fleet-deployer.real" "$root/var/lock/ci-fleet-deployer" +expect_failure 'unsafe symlinked managed directory' "$installer" --repair --config "$config" >/dev/null +rm "$root/var/lock/ci-fleet-deployer" +mv "$root/var/lock/ci-fleet-deployer.real" "$root/var/lock/ci-fleet-deployer" + +unit_path=$root/etc/systemd/system/ci-fleet-deployer.service +mv "$unit_path" "$unit_path.real" +printf 'unrelated-unit\n' >"$tmp/unrelated-unit" +ln -s "$tmp/unrelated-unit" "$unit_path" +expect_failure 'managed systemd unit has an unsafe owner or type' "$installer" --repair --config "$config" >/dev/null +[[ $(<"$tmp/unrelated-unit") == unrelated-unit ]] || fail 'systemd unit symlink attack changed an unrelated file' +rm "$unit_path"; mv "$unit_path.real" "$unit_path" + +printf 'mixed-role\n' >"$root/etc/systemd/system/ci-fleet-health.service" +expect_failure 'ordinary CI controller or runner state is present' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/systemd/system/ci-fleet-health.service" +export FAKE_DOCKER_PS='unrelated workload' +expect_failure 'unrelated Docker workload is present' "$installer" --check --config "$config" >/dev/null +unset FAKE_DOCKER_PS +export FAKE_DOCKER_PS='owned|deployer|staging-deployer-01' +expect_success "$installer" --check --config "$config" >/dev/null +export FAKE_DOCKER_PS='wrong|deployer|other-identity' +expect_failure 'unrelated Docker workload is present' "$installer" --check --config "$config" >/dev/null +unset FAKE_DOCKER_PS + +chmod 0644 "$credential" +secret_error=$(expect_failure 'credential file must be owner-only mode 0600' "$installer" --check --config "$config") +[[ "$secret_error" != *CANARY_SECRET_VALUE_DO_NOT_PRINT* ]] || fail 'credential content leaked in error output' +chmod 0600 "$credential" + +ln -s "$credential" "$root/etc/ci-fleet-deployer/credentials/symlinked" +python3 - "$config" "$root/etc/ci-fleet-deployer/credentials/symlinked" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_REF='+str(p.parent/'credentials/application.credential'), 'CREDENTIAL_REF='+sys.argv[2])) +PY +expect_failure 'credential reference must be a regular file, not a symlink' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/ci-fleet-deployer/credentials/symlinked" +write_config + +printf 'bad\n' >>"$approval" +approval_error=$(expect_failure 'malformed approval evidence line' "$installer" --check --config "$config") +[[ "$approval_error" != *CANARY_SECRET_VALUE_DO_NOT_PRINT* ]] || fail 'secret content leaked beside evidence failure' +write_evidence + +old_state=$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json") +new_image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +image=$new_image +write_evidence +write_config +export FAKE_ADAPTER_FAIL=$tmp/fail-adapter +printf 'validate\n' >"$FAKE_ADAPTER_FAIL" +expect_failure 'candidate adapter validation failed' "$installer" --upgrade --config "$config" >/dev/null +[[ "$old_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'failed candidate replaced healthy state' +rm "$FAKE_ADAPTER_FAIL"; unset FAKE_ADAPTER_FAIL + +upgrade=$(expect_success "$installer" --upgrade --config "$config") +grep -Fq 'result=CHANGED' <<<"$upgrade" || fail 'upgrade did not activate new immutable artifact' +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'rollback did not restore last-known-good state' +grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'rollback state lacks prior artifact' +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$rollback" || fail 'rollback report lacks the exact operator reconciliation action' +grep -Fq 'sha256:bbbbbbbb' "$config" || fail 'rollback unexpectedly rewrote operator-owned desired policy' + +write_evidence production example-production +write_config production example-production +expect_failure 'installed environment and target identity cannot change in place' "$installer" --upgrade --config "$config" >/dev/null +write_evidence +write_config staging example-staging + +active=$root/var/lib/ci-fleet-deployer/active-operation +printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$active" +chmod 0600 "$active" +expect_failure 'active deployment prevents this operation' "$installer" --upgrade --config "$config" >/dev/null +expect_failure 'active deployment prevents drain' "$installer" --drain --config "$config" >/dev/null +rm "$active" +drain=$(expect_success "$installer" --drain --config "$config") +grep -Fq 'result=CHANGED' <<<"$drain" || fail 'drain marker was not created' +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'drain state is absent' + +unrelated=$root/var/lib/ci-fleet-deployer/operator-note +printf 'preserve\n' >"$unrelated" +mv "$root/etc/ci-fleet-deployer/adapters" "$root/etc/ci-fleet-deployer/adapters.retained" +mv "$root/etc/ci-fleet-deployer/credentials" "$root/etc/ci-fleet-deployer/credentials.retained" +mv "$root/etc/ci-fleet-deployer/evidence" "$root/etc/ci-fleet-deployer/evidence.retained" +export FAKE_DOCKER_INFO_EXIT=1 +uninstall=$(expect_success "$installer" --uninstall --config "$config") +unset FAKE_DOCKER_INFO_EXIT +mv "$root/etc/ci-fleet-deployer/adapters.retained" "$root/etc/ci-fleet-deployer/adapters" +mv "$root/etc/ci-fleet-deployer/credentials.retained" "$root/etc/ci-fleet-deployer/credentials" +mv "$root/etc/ci-fleet-deployer/evidence.retained" "$root/etc/ci-fleet-deployer/evidence" +grep -Fq 'result=CHANGED' <<<"$uninstall" || fail 'uninstall did not report change' +[[ -f "$config" && -f "$credential" && -f "$unrelated" ]] || fail 'uninstall removed retained operator state or credentials' +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'uninstall retained activation pointer' +repeat_uninstall=$(expect_success "$installer" --uninstall --config "$config") +grep -Fq 'result=NO_CHANGE' <<<"$repeat_uninstall" || fail 'repeated uninstall was not idempotent' + +# Runtime contract: exact-head request/evidence, drain and scoped adapter calls. +expect_success "$installer" --install --config "$config" >/dev/null +request=$root/var/lib/ci-fleet-deployer/request.conf +cp "$approval" "$request" +chmod 0600 "$request" +export CI_FLEET_DEPLOYER_CONFIG=$config CI_FLEET_DEPLOYER_REQUEST=$request +expect_success "$runtime" health >/dev/null +expect_success "$runtime" cleanup >/dev/null +expect_success "$runtime" deploy >/dev/null +[[ ! -e "$request" && -f "$root/var/lib/ci-fleet-deployer/last-request.conf" ]] || fail 'completed request was not consumed atomically' +cp "$root/var/lib/ci-fleet-deployer/last-request.conf" "$request"; chmod 0600 "$request" +expect_failure 'deployment request was already completed' "$runtime" deploy >/dev/null +grep -Fq 'sha256:bbbbbbbb' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'audit log omitted the immutable digest' +if grep -Fq 'registry.example.invalid' "$root/var/log/ci-fleet-deployer/audit.log"; then fail 'audit log exposed a private-capable endpoint'; fi +for operation in health cleanup deploy; do grep -Fxq "$operation" "$FAKE_ADAPTER_LOG" || fail "runtime did not invoke adapter $operation"; done +: >"$root/var/lib/ci-fleet-deployer/drained" +expect_failure 'deployer is drained' "$runtime" deploy >/dev/null + +grep -Fq 'DEPLOYER-HOST.md' "$repo_root/docs/README.md" || fail 'operator index does not link the deployer runbook' +[[ -x "$repo_root/scripts/test-deployer-units.sh" ]] || fail 'real systemd unit verification is not wired' +grep -Fq 'scripts/test-deployer-units.sh' "$repo_root/scripts/validate.sh" || fail 'repository validation omits systemd unit verification' +for phrase in '--check' '--install' '--upgrade' '--repair' '--drain' '--rollback' '--uninstall' 'manual-exact-head' 'github-environment' 'GitHub Free' 'application-owned' 'REPORT action='; do + grep -Fq -- "$phrase" "$repo_root/docs/DEPLOYER-HOST.md" || fail "deployer runbook omits $phrase" +done +for unit in "$repo_root"/deploy/deployer/*; do + grep -Fq 'ci-fleet-deployer' "$unit" || fail "unit is not deployer-scoped: $unit" +done + +printf 'DEPLOYER_INSTALL_TESTS_OK\n' diff --git a/scripts/validate.sh b/scripts/validate.sh index 9660aaa..e2b86cf 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -28,6 +28,8 @@ python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.jso python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null scripts/test-capacity-preflight.sh scripts/test-install-worker-controller.sh +scripts/test-install-deployer.sh +scripts/test-deployer-units.sh scripts/test-install-status-receiver.sh tmp=$(mktemp) From ed8c5e6351955a875609ae8f172a4fc4ac7b7020 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 8 Aug 2026 21:54:02 -0500 Subject: [PATCH 02/90] fix: address deployer safety review findings --- docs/DEPLOYER-HOST.md | 25 ++++++-- scripts/deployer-runtime.sh | 37 ++++++++++- scripts/install-deployer.sh | 101 ++++++++++++++++++++++++++----- scripts/test-install-deployer.sh | 73 ++++++++++++++++++++-- 4 files changed, 211 insertions(+), 25 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index ae80499..521a948 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -16,6 +16,8 @@ GitHub-native Environment protection is optional, not assumed. In particular, do A secret store proves only that a credential exists. It is not approval evidence. Every provider still binds the environment, target, source commit, artifact digest, approving identity, policy identity, approval ID, and UTC time. +Production remains separately gated during the controlled-migration phase. `ENVIRONMENT=production` is rejected unless `PRODUCTION_AUTHORIZATION_EVIDENCE_PATH` names an additional protected exact-target, exact-head, exact-artifact authorization record. Ordinary approval or credential access does not satisfy this gate, and the field is rejected outside production. + The systemd services run as root because access to the Docker socket is root-equivalent and protected credential references may be root-only. `NoNewPrivileges`, a read-only host filesystem, explicit writable paths, private temporary storage, and no supplementary service account reduce accidental reach, but they do not turn Docker access into a low-privilege boundary. Put nothing else on this host. ## Supported host and prerequisites @@ -63,7 +65,9 @@ adapter rollback `validate` must be non-mutating and must prove that the candidate policy is usable before core changes the active release. `health` returns zero only when the deployer and application-owned contract are healthy. `cleanup` may remove only resources carrying the application's exact deployer ownership identity; it must never run global prune or touch unrelated resources. `deploy` reads the active policy and request paths from the documented environment variables and owns application-specific staging, rollout, health, and rollback. `rollback` restores application state compatible with the recorded last-known-good core policy. -Operations have no interactive input. Zero means success; nonzero means failure. The adapter must honor the systemd timeout, avoid child processes that outlive it, redact logs, and never print credential contents, authorization headers, cookies, private endpoints, or secret-manager responses. Core validates immutable identifiers and approval evidence; it cannot validate application-specific correctness. +Operations have no interactive input. Zero means success; nonzero means failure. Direct installer validation and health calls are limited to two minutes; rollback is limited to 45 minutes. The adapter must avoid child processes that outlive those bounds, redact logs, and never print credential contents, authorization headers, cookies, private endpoints, or secret-manager responses. Core validates immutable identifiers and approval evidence; it cannot validate application-specific correctness. + +Rollback must be atomic from the adapter's perspective: nonzero restores the pre-call application state; zero means rollback health is already verified. For rollback only, core exports `CI_FLEET_DEPLOYER_ROLLBACK_COMMIT`; the adapter atomically creates that root-owned mode-`0600` file as its final successful step. Core stages and selects the last-known-good core before invoking the adapter, restores the newer core on an uncommitted failure, and consumes committed last-known-good state only after that marker exists. ## Prepare host-local files @@ -134,6 +138,19 @@ CAPABILITY_ID=example-capability-check CHECKED_AT=2026-08-08T20:00:00Z ``` +For production, a separate authorized process must also create evidence such as: + +```text +SCHEMA_VERSION=1 +ENVIRONMENT=production +TARGET_ID=example-production +SOURCE_COMMIT=1111111111111111111111111111111111111111 +ARTIFACT_IMAGE=registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +AUTHORIZED_BY=example-production-authorizer +GATE_ID=production-gate-20260808-1 +AUTHORIZED_AT=2026-08-08T20:00:00Z +``` + Create the bounded configuration. Values cannot contain shell expressions; the installer parses `KEY=VALUE` without sourcing it. Unknown, duplicate, empty, malformed, or missing keys fail closed. ```bash @@ -164,7 +181,7 @@ sudo install -o root -g root -m 0600 /dev/null \ sudo chmod 0600 /etc/ci-fleet-deployer/deployer.conf ``` -For an external secret manager, use `CREDENTIAL_PROVIDER=external` and a non-secret reference shaped like `external:example-vault:staging/deployer`. For GitHub Environment approval, use `APPROVAL_PROVIDER=github-environment` and add `APPROVAL_CAPABILITY_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/github-capability.conf`. +For an external secret manager, use `CREDENTIAL_PROVIDER=external` and a non-secret reference shaped like `external:example-vault:staging/deployer`. For GitHub Environment approval, use `APPROVAL_PROVIDER=github-environment` and add `APPROVAL_CAPABILITY_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/github-capability.conf`. Production additionally requires `PRODUCTION_AUTHORIZATION_EVIDENCE_PATH=/etc/ci-fleet-deployer/evidence/production-authorization.conf`. ## Install, check, repair, upgrade, drain, and rollback @@ -235,9 +252,9 @@ sudo systemctl start ci-fleet-deployer.service sudo systemctl status --no-pager ci-fleet-deployer.service ``` -The runtime serializes operations with flock, writes a mode-`0600` active-operation marker, and removes it on completion. While the adapter runs, `systemd-inhibit` blocks shutdown and sleep; the deploy service has an explicit 45-minute start/stop bound. Upgrade, rollback, drain, and uninstall refuse while that marker belongs to a live or bounded recent process. A root-owned stale marker older than the fixed one-hour recovery bound is removed only by a serialized mutating installer run. Unsafe or malformed stale state fails closed. +The runtime and read-only checks serialize on the same flock. Deploy writes a mode-`0600` active-operation marker and removes it on completion. While the adapter runs, `systemd-inhibit` blocks shutdown and sleep; the deploy service has an explicit 45-minute start/stop bound. Upgrade, rollback, drain, and uninstall refuse while that marker belongs to a live or bounded recent process. A root-owned stale marker older than the fixed one-hour recovery bound is removed only by a serialized mutating installer run. Unsafe or malformed stale state fails closed. -Health runs every five minutes and cleanup daily. Cleanup is delegated to the application adapter because only application-owned code knows its exact resources. Core itself issues no Docker delete or prune command. +Health runs every five minutes and cleanup daily. Cleanup refuses while drained and is delegated to the application adapter because only application-owned code knows its exact resources. Core itself issues no Docker delete or prune command. ## Verification and reports diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 11543cf..c80ec2b 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -15,6 +15,7 @@ request=${CI_FLEET_DEPLOYER_REQUEST:-$(root_path /var/lib/ci-fleet-deployer/requ state_root=$(root_path /var/lib/ci-fleet-deployer) log_root=$(root_path /var/log/ci-fleet-deployer) lock_dir=$(root_path /var/lock/ci-fleet-deployer) +evidence_dir=$(root_path /etc/ci-fleet-deployer/evidence) active=$state_root/active-operation drained=$state_root/drained last_request=$state_root/last-request.conf @@ -33,6 +34,10 @@ secure_directory() { local path=$1 description=$2 [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:700" ]] || die "$description has unsafe owner, mode, or type" } +inside() { + local path=$1 base=$2 + [[ $(realpath -m -- "$path") == "$(realpath -m -- "$base")/"* ]] +} parse_file() { local path=$1 prefix=$2 kind=$3 allowed=$4 line key value declare -gA "$prefix=()" @@ -51,7 +56,7 @@ parse_file() { } secure_file "$config" 'deployer configuration' -config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' +config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' parse_file "$config" cfg configuration "$config_keys" for key in ENVIRONMENT TARGET_ID ADAPTER_PATH ADAPTER_SHA256 SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done [[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' @@ -72,9 +77,13 @@ case "$operation" in chmod 0600 "$temporary" mv -Tf "$temporary" "$drained" ;; - health|cleanup|rollback) + health|rollback) "${cfg[ADAPTER_PATH]}" "$operation" ;; + cleanup) + [[ ! -e "$drained" ]] || die 'deployer is drained' + "${cfg[ADAPTER_PATH]}" cleanup + ;; deploy) [[ ! -e "$drained" ]] || die 'deployer is drained' secure_file "$request" 'deployment request' @@ -94,6 +103,29 @@ case "$operation" in [[ ${req[SOURCE_COMMIT]} =~ ^[0-9a-f]{40}$ && ${req[ARTIFACT_IMAGE]} =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?/[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || die 'deployment request is not immutable and qualified' for key in APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID; do [[ ${req[$key]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ ]] || die "deployment request has an unsafe $key"; done [[ ${req[APPROVED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'deployment request has an invalid approval time' + [[ -v 'cfg[APPROVAL_EVIDENCE_PATH]' ]] || die 'installed policy is missing approval evidence' + inside "${cfg[APPROVAL_EVIDENCE_PATH]}" "$evidence_dir" || die 'approval evidence is outside the protected evidence directory' + secure_file "${cfg[APPROVAL_EVIDENCE_PATH]}" 'approval evidence' + approval_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' + parse_file "${cfg[APPROVAL_EVIDENCE_PATH]}" approved 'approval evidence' "$approval_keys" + for key in $approval_keys; do + [[ -v "approved[$key]" && ${req[$key]} == "${approved[$key]}" ]] || die "deployment request does not match protected approval $key" + done + if [[ ${cfg[ENVIRONMENT]} == production ]]; then + [[ -v 'cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]' ]] || die 'production policy is missing separate authorization evidence' + inside "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" "$evidence_dir" || die 'production authorization evidence is outside the protected evidence directory' + secure_file "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" 'production authorization evidence' + production_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE AUTHORIZED_BY GATE_ID AUTHORIZED_AT' + parse_file "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" production 'production authorization evidence' "$production_keys" + for key in $production_keys; do [[ -v "production[$key]" ]] || die "production authorization evidence is missing $key"; done + [[ ${production[SCHEMA_VERSION]} == 1 && ${production[ENVIRONMENT]} == production ]] || die 'production authorization evidence has the wrong scope' + [[ ${production[AUTHORIZED_BY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${production[GATE_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${production[AUTHORIZED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'production authorization evidence is malformed' + for key in ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do + [[ -v "production[$key]" && ${req[$key]} == "${production[$key]}" ]] || die "deployment request does not match production authorization $key" + done + fi + if [[ -e "$audit_log" || -L "$audit_log" ]]; then secure_file "$audit_log" 'deployer audit log'; else install -m 0600 /dev/null "$audit_log"; fi + : >>"$audit_log" umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" @@ -102,7 +134,6 @@ case "$operation" in systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ --why='approved deployment is active' -- "${cfg[ADAPTER_PATH]}" deploy mv -Tf "$request" "$last_request" - if [[ -e "$audit_log" || -L "$audit_log" ]]; then secure_file "$audit_log" 'deployer audit log'; else install -m 0600 /dev/null "$audit_log"; fi printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=success\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" \ diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 1b33853..89ea1d5 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -19,9 +19,12 @@ health=unknown staging_path= transaction_dir= transaction_committed=0 +recovered_rollback=0 on_exit() { local status=$? - if [[ -n ${transaction_dir:-} && ${transaction_committed:-0} != 1 ]]; then restore_transaction || true; fi + if [[ -n ${transaction_dir:-} && ${transaction_committed:-0} != 1 ]]; then + if [[ -f "$transaction_dir/application-rollback-committed" ]]; then finalize_committed_rollback || true; else restore_transaction || true; fi + fi [[ -z ${staging_path:-} ]] || rm -rf -- "$staging_path" if ((status != 0 && error_reported == 0)); then report FAILED no inspect-and-retry "$(rollback_available)" >&2; fi return "$status" @@ -90,7 +93,7 @@ unit_names=( ci-fleet-deployer-drain.service ) timer_names=(ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer) -config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' +config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' expected_uid=0 [[ "$testing" != 1 ]] || expected_uid=$(id -u) @@ -177,6 +180,26 @@ validate_config() { *) block 'CREDENTIAL_PROVIDER must be file or external' ;; esac validate_evidence + validate_production_gate +} + +validate_production_gate() { + local key + if [[ "$environment" != production ]]; then + [[ ! -v 'cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]' ]] || block 'production authorization evidence is forbidden outside production' + return + fi + [[ -v 'cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]' ]] || block 'production requires separate authorization evidence' + inside "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" "$etc_root/evidence" || block 'production authorization evidence is outside the approved evidence directory' + secure_file "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" 'production authorization evidence' + parse_file "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" production_gate 'production authorization evidence' 'SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE AUTHORIZED_BY GATE_ID AUTHORIZED_AT' + for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE AUTHORIZED_BY GATE_ID AUTHORIZED_AT; do + [[ -v "production_gate[$key]" ]] || block "production authorization evidence is missing $key" + done + [[ ${production_gate[SCHEMA_VERSION]} == 1 && ${production_gate[ENVIRONMENT]} == production ]] || block 'production authorization evidence has the wrong scope' + for key in TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do [[ ${production_gate[$key]} == "${cfg[$key]}" ]] || block "production authorization evidence does not match exact $key"; done + [[ ${production_gate[AUTHORIZED_BY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${production_gate[GATE_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || block 'production authorization identity is malformed' + [[ ${production_gate[AUTHORIZED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || block 'production authorization timestamp must be UTC RFC3339' } validate_evidence() { @@ -213,8 +236,8 @@ validate_evidence() { } require_host() { - local command os_id os_version available required - for command in bash awk cut sort stat sha256sum readlink realpath install cmp mv cp rm mkdir mktemp chmod ln flock kill git python3 docker systemctl systemd-analyze systemd-inhibit timedatectl curl df date; do + local command os_id os_version available required systemd_state + for command in bash awk cut sort stat sha256sum readlink realpath install cmp mv cp rm mkdir mktemp chmod ln flock kill timeout env git python3 docker systemctl systemd-analyze systemd-inhibit timedatectl curl df date; do command -v "$command" >/dev/null || block "$command is required" done local os_release @@ -224,7 +247,8 @@ require_host() { os_version=$(awk -F= '$1=="VERSION_ID" {gsub(/"/,"",$2); print $2}' "$os_release") [[ "$os_id" == debian && "$os_version" =~ ^(12|13)(\.|$) || "$os_id" == ubuntu && "$os_version" =~ ^(22\.04|24\.04)$ ]] || block 'unsupported Linux distribution or release' [[ -d $(root_path /run/systemd/system) ]] || block 'systemd is not the active init system' - systemctl is-system-running >/dev/null 2>&1 || block 'systemd is unavailable' + systemd_state=$(systemctl is-system-running 2>/dev/null || true) + [[ "$systemd_state" == running || "$systemd_state" == degraded ]] || block 'systemd is unavailable' docker info >/dev/null 2>&1 || block 'Docker Engine is unavailable' [[ ${cfg[REQUIRE_COMPOSE]} != 1 ]] || docker compose version >/dev/null 2>&1 || block 'Docker Compose v2 is required but unavailable' [[ $(timedatectl show -p NTPSynchronized --value 2>/dev/null) == yes ]] || block 'host time is not synchronized' @@ -248,10 +272,16 @@ require_maintenance_host() { } reject_mixed_role() { - local unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" + local unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" runner_unit for unit in ci-fleet-health.service ci-fleet-reconcile.service ci-fleet-cleanup.service actions.runner.service; do [[ ! -e "$systemd_root/$unit" ]] || block 'ordinary CI controller or runner state is present' done + shopt -s nullglob + for runner_unit in "$systemd_root"/actions.runner.*.service "$systemd_root"/multi-user.target.wants/actions.runner.*.service; do + shopt -u nullglob + [[ -n "$runner_unit" ]] && block 'ordinary GitHub Actions runner service is present' + done + shopt -u nullglob for path in "$(root_path /etc/ci-fleet/ci-fleet.env)" "$(root_path /opt/ci-fleet/current)" "$(root_path /var/lib/ci-fleet/install-state.json)"; do [[ ! -e "$path" && ! -L "$path" ]] || block 'ordinary CI controller or runner state is present' done @@ -352,6 +382,12 @@ acquire_lock() { recover_interrupted_transaction } +acquire_check_lock() { + [[ -d "$lock_root" && ! -L "$lock_root" && $(stat -c '%u:%a' "$lock_root") == "$expected_uid:700" ]] || block 'installed deployer lock boundary is absent or unsafe' + exec 9<"$lock_root" + flock -n 9 || block 'another deployer operation is running' +} + begin_transaction() { local name path current_target for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do @@ -437,10 +473,25 @@ recover_interrupted_transaction() { candidate=${candidates[0]} [[ ! -L "$candidate" && -d "$candidate" && $(stat -c '%u:%a' "$candidate") == "$expected_uid:700" ]] || block 'interrupted installer transaction is unsafe' transaction_dir=$candidate + if [[ -f "$transaction_dir/application-rollback-committed" ]]; then + finalize_committed_rollback + recovered_rollback=1 + transaction_committed=0 + return + fi restore_transaction transaction_committed=0 } +finalize_committed_rollback() { + local marker=$transaction_dir/application-rollback-committed + [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' + rm -f "$previous_state" "$previous_policy" + transaction_committed=1 + rm -rf -- "$transaction_dir" + transaction_dir= +} + commit_transaction() { transaction_committed=1 rm -rf -- "$transaction_dir" @@ -515,8 +566,28 @@ install_units() { systemctl enable --now "${timer_names[@]}" >/dev/null } +adapter_deadline() { + local operation_name=$1 seconds=120 + [[ "$operation_name" != rollback ]] || seconds=2700 + if [[ "$testing" == 1 && -n ${CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS:-} ]]; then + [[ ${CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS} =~ ^[1-9][0-9]?$ ]] || die 'invalid test-only adapter timeout' + seconds=$CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS + fi + printf '%s' "$seconds" +} + +run_adapter() { + local policy=$1 adapter_path=$2 operation_name=$3 marker=${4:-} seconds + seconds=$(adapter_deadline "$operation_name") + if [[ -n "$marker" ]]; then + timeout --signal=TERM --kill-after=10s "${seconds}s" env CI_FLEET_DEPLOYER_CONFIG="$policy" CI_FLEET_DEPLOYER_ROLLBACK_COMMIT="$marker" "$adapter_path" "$operation_name" + else + timeout --signal=TERM --kill-after=10s "${seconds}s" env CI_FLEET_DEPLOYER_CONFIG="$policy" "$adapter_path" "$operation_name" + fi +} + policy_adapter_operation() { - local policy=$1 operation_name=$2 description=$3 key + local policy=$1 operation_name=$2 description=$3 marker=${4:-} key local -A policy_cfg=() secure_file "$policy" "$description" parse_file "$policy" policy_cfg "$description" "$config_keys" @@ -524,7 +595,7 @@ policy_adapter_operation() { [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" secure_file "${policy_cfg[ADAPTER_PATH]}" "$description adapter" 700 [[ $(sha256sum "${policy_cfg[ADAPTER_PATH]}" | cut -d' ' -f1) == "${policy_cfg[ADAPTER_SHA256]}" ]] || die "$description adapter digest mismatch" - CI_FLEET_DEPLOYER_CONFIG="$policy" "${policy_cfg[ADAPTER_PATH]}" "$operation_name" >/dev/null 2>&1 + run_adapter "$policy" "${policy_cfg[ADAPTER_PATH]}" "$operation_name" "$marker" >/dev/null 2>&1 } perform_check() { @@ -561,7 +632,7 @@ PY release_complete "$old_release" || block 'active deployer release is incomplete' policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement' fi - CI_FLEET_DEPLOYER_CONFIG="$config" "${cfg[ADAPTER_PATH]}" validate >/dev/null 2>&1 || die 'candidate adapter validation failed' + run_adapter "$config" "${cfg[ADAPTER_PATH]}" validate >/dev/null 2>&1 || die 'candidate adapter validation failed' install_release secure_directory "$state_root" 700 1 secure_directory "$log_root" 700 1 @@ -585,6 +656,7 @@ PY } perform_rollback() { + if ((recovered_rollback)); then health=healthy; report CHANGED yes restore-host-policy-evidence-then-check no; return; fi active_deployment && block 'active deployment prevents rollback' [[ -f "$previous_state" && -f "$previous_policy" ]] || block 'no last-known-good release is available' secure_directory "$state_root" 700 0 @@ -596,16 +668,17 @@ perform_rollback() { target=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["target"])' "$previous_state") artifact=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["artifact"])' "$previous_state") release_complete "$releases/$core_ref" || die 'last-known-good release is incomplete' - policy_adapter_operation "$previous_policy" rollback 'last-known-good policy' || die 'application adapter rollback failed' + unit_source=$releases/$core_ref/deploy/deployer ln -sfn "releases/$core_ref" "$install_root/.current.new" install_units mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_file.new" "$state_file" mv -Tf "$install_root/.current.new" "$current" - policy_adapter_operation "$active_policy" health 'rolled-back policy' || die 'rolled-back deployer health check failed' - commit_transaction + policy_adapter_operation "$active_policy" rollback 'last-known-good policy' "$transaction_dir/application-rollback-committed" || die 'application adapter rollback failed' + finalize_committed_rollback + recovered_rollback=0 health=healthy - report CHANGED yes restore-host-policy-evidence-then-check yes + report CHANGED yes restore-host-policy-evidence-then-check no } perform_drain() { @@ -644,7 +717,7 @@ else require_host fi case "$mode" in - check) perform_check ;; + check) acquire_check_lock; perform_check ;; install|upgrade|repair) acquire_lock; perform_converge ;; rollback) acquire_lock; perform_rollback ;; drain) acquire_lock; perform_drain ;; diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 36626e1..975d280 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -52,7 +52,11 @@ root=${CI_FLEET_DEPLOYER_ROOT:-} log=${FAKE_SYSTEMCTL_LOG:-/dev/null} printf '%s\n' "$*" >>"$log" case "${1:-}" in - is-system-running) [[ -z "${FAKE_SYSTEMD_FAIL:-}" ]] || exit 1; printf 'running\n' ;; + is-system-running) + [[ -z "${FAKE_SYSTEMD_FAIL:-}" ]] || exit 1 + printf '%s\n' "${FAKE_SYSTEMD_STATE:-running}" + [[ ${FAKE_SYSTEMD_STATE:-running} != degraded ]] + ;; is-enabled|is-active) [[ -e "$root/var/lib/ci-fleet-deployer/unit-${2:-}" ]] ;; enable) shift @@ -103,11 +107,15 @@ cat >"$adapter" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail printf '%s\n' "$1" >>"${FAKE_ADAPTER_LOG:?}" +if [[ ${FAKE_ADAPTER_SLEEP_OPERATION:-} == "$1" ]]; then sleep 2; fi if [[ -e "${FAKE_ADAPTER_FAIL:-/nonexistent}" ]]; then fail_operation=$(<"$FAKE_ADAPTER_FAIL") [[ "$fail_operation" != all && "$fail_operation" != "$1" ]] || exit 42 fi case "$1" in validate|health|cleanup|deploy|rollback) ;; *) exit 2 ;; esac +if [[ "$1" == rollback && -n ${CI_FLEET_DEPLOYER_ROLLBACK_COMMIT:-} ]]; then + install -m 0600 /dev/null "$CI_FLEET_DEPLOYER_ROLLBACK_COMMIT" +fi EOF chmod 0700 "$adapter" export FAKE_ADAPTER_LOG=$tmp/adapter.log @@ -116,6 +124,7 @@ printf 'CANARY_SECRET_VALUE_DO_NOT_PRINT\n' >"$credential" chmod 0600 "$credential" approval=$root/etc/ci-fleet-deployer/evidence/approval.conf checkpoint=$root/etc/ci-fleet-deployer/evidence/checkpoint.conf +production_gate=$root/etc/ci-fleet-deployer/evidence/production-authorization.conf core_ref=$(git -C "$repo_root" rev-parse HEAD) source_ref=1111111111111111111111111111111111111111 image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' @@ -141,6 +150,19 @@ RECORDED_AT=2026-08-08T19:55:00Z EOF chmod 0600 "$approval" "$checkpoint" } +write_production_gate() { + cat >"$production_gate" <"$root/etc/ci-fleet-deployer/deployer.conf" <>"$root/etc/ci-fleet-deployer/deployer.conf"; fi chmod 0600 "$root/etc/ci-fleet-deployer/deployer.conf" } write_evidence write_config config=$root/etc/ci-fleet-deployer/deployer.conf +write_evidence production example-production +write_config production example-production +expect_failure 'production authorization evidence must be a regular file' "$installer" --check --config "$config" >/dev/null +write_production_gate +expect_failure 'installed deployer lock boundary is absent or unsafe' "$installer" --check --config "$config" >/dev/null +rm "$production_gate" +write_evidence +write_config + # Read-only preflight and strict policy variants fail closed without reading secrets. cp "$root/etc/os-release" "$tmp/os-release" printf 'ID=alpine\nVERSION_ID=3.20\n' >"$root/etc/os-release" @@ -214,14 +246,14 @@ CAPABILITY_ID=example-capability-check CHECKED_AT=2026-08-08T20:00:00Z EOF chmod 0600 "$capability" -expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +expect_failure 'installed deployer lock boundary is absent or unsafe' "$installer" --check --config "$config" >/dev/null write_config python3 - "$config" <<'PY' from pathlib import Path import sys p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external').replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=external:example-vault:staging/deployer')) PY -expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +expect_failure 'installed deployer lock boundary is absent or unsafe' "$installer" --check --config "$config" >/dev/null write_config printf 'UNKNOWN=value\n' >"$tmp/unknown.conf"; chmod 0600 "$tmp/unknown.conf" @@ -256,6 +288,7 @@ second=$(expect_success "$installer" --install --config "$config") grep -Fq 'result=NO_CHANGE' <<<"$second" || fail 'second install was not idempotent' check=$(expect_success "$installer" --check --config "$config") grep -Fq 'REPORT action=check result=NO_CHANGE' <<<"$check" || fail 'check did not report convergence' +FAKE_SYSTEMD_STATE=degraded expect_success "$installer" --check --config "$config" >/dev/null interrupted=$root/var/lib/ci-fleet-deployer/.transaction.interrupted mkdir -m 0700 "$interrupted" "$interrupted/units" "$interrupted/state" @@ -299,6 +332,7 @@ expect_success "$installer" --repair --config "$config" >/dev/null exec 8<"$root/var/lock/ci-fleet-deployer" flock -n 8 || fail 'fixture could not acquire installer lock' expect_failure 'another deployer installer operation is running' "$installer" --repair --config "$config" >/dev/null +expect_failure 'another deployer operation is running' "$installer" --check --config "$config" >/dev/null flock -u 8; exec 8>&- mv "$root/var/lock/ci-fleet-deployer" "$root/var/lock/ci-fleet-deployer.real" @@ -318,6 +352,9 @@ rm "$unit_path"; mv "$unit_path.real" "$unit_path" printf 'mixed-role\n' >"$root/etc/systemd/system/ci-fleet-health.service" expect_failure 'ordinary CI controller or runner state is present' "$installer" --check --config "$config" >/dev/null rm "$root/etc/systemd/system/ci-fleet-health.service" +printf 'runner\n' >"$root/etc/systemd/system/actions.runner.example-org-example-repo.example-runner.service" +expect_failure 'ordinary GitHub Actions runner service is present' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/systemd/system/actions.runner.example-org-example-repo.example-runner.service" export FAKE_DOCKER_PS='unrelated workload' expect_failure 'unrelated Docker workload is present' "$installer" --check --config "$config" >/dev/null unset FAKE_DOCKER_PS @@ -352,6 +389,10 @@ new_image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbb image=$new_image write_evidence write_config +export FAKE_ADAPTER_SLEEP_OPERATION=validate CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS=1 +expect_failure 'candidate adapter validation failed' "$installer" --upgrade --config "$config" >/dev/null +unset FAKE_ADAPTER_SLEEP_OPERATION CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS +[[ "$old_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'timed-out candidate replaced healthy state' export FAKE_ADAPTER_FAIL=$tmp/fail-adapter printf 'validate\n' >"$FAKE_ADAPTER_FAIL" expect_failure 'candidate adapter validation failed' "$installer" --upgrade --config "$config" >/dev/null @@ -360,12 +401,22 @@ rm "$FAKE_ADAPTER_FAIL"; unset FAKE_ADAPTER_FAIL upgrade=$(expect_success "$installer" --upgrade --config "$config") grep -Fq 'result=CHANGED' <<<"$upgrade" || fail 'upgrade did not activate new immutable artifact' +printf 'rollback\n' >"$tmp/fail-rollback" +export FAKE_ADAPTER_FAIL=$tmp/fail-rollback +expect_failure 'application adapter rollback failed' "$installer" --rollback --config "$config" >/dev/null +unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-rollback" +grep -Fq 'sha256:bbbbbbbb' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'failed application rollback did not restore current core state' +rollback_calls_before=$(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) +FAKE_SYSTEMD_VERIFY_EXIT=1 expect_failure 'systemd unit verification failed' "$installer" --rollback --config "$config" >/dev/null +[[ $(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) == "$rollback_calls_before" ]] || fail 'application rollback ran before core rollback staging was proven' +grep -Fq 'sha256:bbbbbbbb' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'failed rollback did not preserve current core state' rollback=$(expect_success "$installer" --rollback --config "$config") grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'rollback did not restore last-known-good state' grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'rollback state lacks prior artifact' grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$rollback" || fail 'rollback report lacks the exact operator reconciliation action' grep -Fq 'sha256:bbbbbbbb' "$config" || fail 'rollback unexpectedly rewrote operator-owned desired policy' +write_production_gate write_evidence production example-production write_config production example-production expect_failure 'installed environment and target identity cannot change in place' "$installer" --upgrade --config "$config" >/dev/null @@ -407,6 +458,19 @@ chmod 0600 "$request" export CI_FLEET_DEPLOYER_CONFIG=$config CI_FLEET_DEPLOYER_REQUEST=$request expect_success "$runtime" health >/dev/null expect_success "$runtime" cleanup >/dev/null +python3 - "$request" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=forged-approval')) +PY +expect_failure 'deployment request does not match protected approval APPROVAL_ID' "$runtime" deploy >/dev/null +cp "$approval" "$request"; chmod 0600 "$request" +printf 'unrelated-audit\n' >"$tmp/unrelated-audit" +ln -s "$tmp/unrelated-audit" "$root/var/log/ci-fleet-deployer/audit.log" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" && $(<"$tmp/unrelated-audit") == unrelated-audit ]] || fail 'unsafe audit storage was touched after adapter execution' +rm "$root/var/log/ci-fleet-deployer/audit.log" expect_success "$runtime" deploy >/dev/null [[ ! -e "$request" && -f "$root/var/lib/ci-fleet-deployer/last-request.conf" ]] || fail 'completed request was not consumed atomically' cp "$root/var/lib/ci-fleet-deployer/last-request.conf" "$request"; chmod 0600 "$request" @@ -416,11 +480,12 @@ if grep -Fq 'registry.example.invalid' "$root/var/log/ci-fleet-deployer/audit.lo for operation in health cleanup deploy; do grep -Fxq "$operation" "$FAKE_ADAPTER_LOG" || fail "runtime did not invoke adapter $operation"; done : >"$root/var/lib/ci-fleet-deployer/drained" expect_failure 'deployer is drained' "$runtime" deploy >/dev/null +expect_failure 'deployer is drained' "$runtime" cleanup >/dev/null grep -Fq 'DEPLOYER-HOST.md' "$repo_root/docs/README.md" || fail 'operator index does not link the deployer runbook' [[ -x "$repo_root/scripts/test-deployer-units.sh" ]] || fail 'real systemd unit verification is not wired' grep -Fq 'scripts/test-deployer-units.sh' "$repo_root/scripts/validate.sh" || fail 'repository validation omits systemd unit verification' -for phrase in '--check' '--install' '--upgrade' '--repair' '--drain' '--rollback' '--uninstall' 'manual-exact-head' 'github-environment' 'GitHub Free' 'application-owned' 'REPORT action='; do +for phrase in '--check' '--install' '--upgrade' '--repair' '--drain' '--rollback' '--uninstall' 'manual-exact-head' 'github-environment' 'GitHub Free' 'PRODUCTION_AUTHORIZATION_EVIDENCE_PATH' 'CI_FLEET_DEPLOYER_ROLLBACK_COMMIT' 'application-owned' 'REPORT action='; do grep -Fq -- "$phrase" "$repo_root/docs/DEPLOYER-HOST.md" || fail "deployer runbook omits $phrase" done for unit in "$repo_root"/deploy/deployer/*; do From 777f8c534cf27a6d56454d3a1b20b9a32eba3df3 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 8 Aug 2026 22:24:54 -0500 Subject: [PATCH 03/90] fix: harden deployer retained-state validation --- docs/DEPLOYER-HOST.md | 2 +- scripts/deployer-runtime.sh | 22 ++++++++++++++----- scripts/install-deployer.sh | 29 +++++++++++++++++-------- scripts/test-install-deployer.sh | 36 +++++++++++++++++++++++++++++++- 4 files changed, 73 insertions(+), 16 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index 521a948..a1df26c 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -243,7 +243,7 @@ A cross-environment or cross-target in-place upgrade is rejected. Build a separa ## Submit an approved deployment -Place one root-owned mode-`0600` request at `/var/lib/ci-fleet-deployer/request.conf`. It uses the same exact-head approval fields shown above. No secret values belong in the request. After a successful adapter call, the runtime atomically moves it to `last-request.conf`; submitting the identical completed request again fails closed. +Place one root-owned mode-`0600` request at `/var/lib/ci-fleet-deployer/request.conf`. It uses the same exact-head approval fields shown above. No secret values belong in the request. Before invoking the adapter, runtime revalidates the protected approval and checkpoint evidence. After success it atomically moves the request to `last-request.conf`; resubmitting the same semantic request fails closed regardless of field order or comments. Then run on the deployer host; this changes the application target through its adapter: diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index c80ec2b..149b3ed 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -58,7 +58,7 @@ parse_file() { secure_file "$config" 'deployer configuration' config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' parse_file "$config" cfg configuration "$config_keys" -for key in ENVIRONMENT TARGET_ID ADAPTER_PATH ADAPTER_SHA256 SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done +for key in ENVIRONMENT TARGET_ID ADAPTER_PATH ADAPTER_SHA256 CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done [[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 @@ -87,10 +87,6 @@ case "$operation" in deploy) [[ ! -e "$drained" ]] || die 'deployer is drained' secure_file "$request" 'deployment request' - if [[ -e "$last_request" || -L "$last_request" ]]; then - secure_file "$last_request" 'last completed deployment request' - cmp -s "$request" "$last_request" && die 'deployment request was already completed' - fi request_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' parse_file "$request" req request "$request_keys" for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT; do @@ -103,6 +99,22 @@ case "$operation" in [[ ${req[SOURCE_COMMIT]} =~ ^[0-9a-f]{40}$ && ${req[ARTIFACT_IMAGE]} =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?/[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || die 'deployment request is not immutable and qualified' for key in APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID; do [[ ${req[$key]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ ]] || die "deployment request has an unsafe $key"; done [[ ${req[APPROVED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'deployment request has an invalid approval time' + if [[ -e "$last_request" || -L "$last_request" ]]; then + secure_file "$last_request" 'last completed deployment request' + parse_file "$last_request" completed 'last completed deployment request' "$request_keys" + replay=1 + for key in $request_keys; do + [[ -v "completed[$key]" && ${req[$key]} == "${completed[$key]}" ]] || replay=0 + done + ((replay == 0)) || die 'deployment request was already completed' + fi + inside "${cfg[CHECKPOINT_EVIDENCE_PATH]}" "$evidence_dir" || die 'checkpoint evidence is outside the protected evidence directory' + secure_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" 'checkpoint evidence' + checkpoint_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID CHECKPOINT_ID RECORDED_AT' + parse_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" checkpoint 'checkpoint evidence' "$checkpoint_keys" + for key in $checkpoint_keys; do [[ -v "checkpoint[$key]" ]] || die "checkpoint evidence is missing $key"; done + [[ ${checkpoint[SCHEMA_VERSION]} == 1 && ${checkpoint[ENVIRONMENT]} == "${req[ENVIRONMENT]}" && ${checkpoint[TARGET_ID]} == "${req[TARGET_ID]}" ]] || die 'checkpoint evidence does not match the deployment target' + [[ ${checkpoint[CHECKPOINT_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${checkpoint[RECORDED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'checkpoint evidence is malformed' [[ -v 'cfg[APPROVAL_EVIDENCE_PATH]' ]] || die 'installed policy is missing approval evidence' inside "${cfg[APPROVAL_EVIDENCE_PATH]}" "$evidence_dir" || die 'approval evidence is outside the protected evidence directory' secure_file "${cfg[APPROVAL_EVIDENCE_PATH]}" 'approval evidence' diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 89ea1d5..713c9df 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -225,6 +225,7 @@ validate_evidence() { secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' 'SCHEMA_VERSION ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' [[ ${capability[SCHEMA_VERSION]:-} == 1 && ${capability[ENVIRONMENT_PROTECTION]:-} == verified && ${capability[EXACT_HEAD]:-} == "${cfg[SOURCE_COMMIT]}" ]] || block 'GitHub Environment capability evidence is not exact-head verified' + [[ ${capability[CAPABILITY_ID]:-} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${capability[CHECKED_AT]:-} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || block 'GitHub Environment capability evidence is missing identity or UTC time' ;; *) block 'unsupported approval provider' ;; esac @@ -236,7 +237,7 @@ validate_evidence() { } require_host() { - local command os_id os_version available required systemd_state + local command os_id os_version available required systemd_state disk_path for command in bash awk cut sort stat sha256sum readlink realpath install cmp mv cp rm mkdir mktemp chmod ln flock kill timeout env git python3 docker systemctl systemd-analyze systemd-inhibit timedatectl curl df date; do command -v "$command" >/dev/null || block "$command is required" done @@ -252,8 +253,9 @@ require_host() { docker info >/dev/null 2>&1 || block 'Docker Engine is unavailable' [[ ${cfg[REQUIRE_COMPOSE]} != 1 ]] || docker compose version >/dev/null 2>&1 || block 'Docker Compose v2 is required but unavailable' [[ $(timedatectl show -p NTPSynchronized --value 2>/dev/null) == yes ]] || block 'host time is not synchronized' - available=$(df -Pk "$state_root" 2>/dev/null | awk 'NR==2 {print $4}') - [[ "$available" =~ ^[0-9]+$ ]] || available=$(df -Pk "$(dirname "$state_root")" | awk 'NR==2 {print $4}') + disk_path=$state_root + while [[ ! -e "$disk_path" ]]; do disk_path=$(dirname "$disk_path"); done + available=$(df -Pk "$disk_path" | awk 'NR==2 {print $4}') required=$((cfg[MIN_DISK_GIB] * 1024 * 1024)) ((available >= required)) || block 'insufficient deployer disk capacity' if [[ "$testing" != 1 || ${CI_FLEET_DEPLOYER_TEST_NETWORK:-} != ok ]]; then @@ -340,7 +342,7 @@ raise SystemExit(0 if all(value.get(k)==v for k,v in zip(keys,expected)) else 1) units_match() { local unit for unit in "${unit_names[@]}"; do - [[ -f "$systemd_root/$unit" && ! -L "$systemd_root/$unit" ]] || return 1 + [[ -f "$systemd_root/$unit" && ! -L "$systemd_root/$unit" && $(stat -c '%u:%a' "$systemd_root/$unit") == "$expected_uid:644" ]] || return 1 cmp -s "$unit_source/$unit" "$systemd_root/$unit" || return 1 done for unit in "${timer_names[@]}"; do systemctl is-enabled "$unit" >/dev/null 2>&1 && systemctl is-active "$unit" >/dev/null 2>&1 || return 1; done @@ -684,9 +686,13 @@ perform_rollback() { perform_drain() { if active_deployment; then block 'active deployment prevents drain'; fi secure_directory "$state_root" 700 1 - if [[ -f "$drained" ]]; then report NO_CHANGE no safe-to-maintain "$(rollback_available)"; return; fi - : >"$drained" - chmod 0600 "$drained" + if [[ -e "$drained" || -L "$drained" ]]; then + secure_file "$drained" 'drain marker' + report NO_CHANGE no safe-to-maintain "$(rollback_available)"; return + fi + temporary=$(mktemp "$state_root/.drained.XXXXXX") + chmod 0600 "$temporary" + mv -Tf "$temporary" "$drained" report CHANGED yes safe-to-maintain "$(rollback_available)" } @@ -698,8 +704,13 @@ perform_uninstall() { if active_deployment; then block 'active deployment prevents this operation'; fi acquire_lock secure_directory "$state_root" 700 1 - : >"$drained" - chmod 0600 "$drained" + if [[ -e "$drained" || -L "$drained" ]]; then + secure_file "$drained" 'drain marker' + else + temporary=$(mktemp "$state_root/.drained.XXXXXX") + chmod 0600 "$temporary" + mv -Tf "$temporary" "$drained" + fi if active_deployment; then block 'active deployment started while draining'; fi if [[ -L "$current" ]]; then rm -f "$current"; changed=yes; fi systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 975d280..a0ff1d9 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -94,6 +94,7 @@ exit 2 EOF cat >"$fake_bin/df" <<'EOF' #!/usr/bin/env bash +if [[ -n ${FAKE_DF_REQUIRE_EXISTING:-} && ! -e ${!#} ]]; then exit 1; fi printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\nfixture 104857600 1 %s 1%% /\n' "${FAKE_DISK_AVAILABLE:-104857599}" EOF chmod 0755 "$fake_bin"/* @@ -192,6 +193,8 @@ write_evidence write_config config=$root/etc/ci-fleet-deployer/deployer.conf +FAKE_DF_REQUIRE_EXISTING=1 expect_failure 'installed deployer lock boundary is absent or unsafe' "$installer" --check --config "$config" >/dev/null + write_evidence production example-production write_config production example-production expect_failure 'production authorization evidence must be a regular file' "$installer" --check --config "$config" >/dev/null @@ -247,6 +250,20 @@ CHECKED_AT=2026-08-08T20:00:00Z EOF chmod 0600 "$capability" expect_failure 'installed deployer lock boundary is absent or unsafe' "$installer" --check --config "$config" >/dev/null +python3 - "$capability" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text('\n'.join(x for x in p.read_text().splitlines() if not x.startswith('CAPABILITY_ID='))+'\n') +PY +expect_failure 'GitHub Environment capability evidence is missing identity or UTC time' "$installer" --check --config "$config" >/dev/null +cat >"$capability" </dev/null +chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +[[ $(stat -c %a "$root/etc/systemd/system/ci-fleet-deployer.service") == 644 ]] || fail 'repair did not restore unit mode 0644' interrupted=$root/var/lib/ci-fleet-deployer/.transaction.interrupted mkdir -m 0700 "$interrupted" "$interrupted/units" "$interrupted/state" @@ -432,6 +453,13 @@ rm "$active" drain=$(expect_success "$installer" --drain --config "$config") grep -Fq 'result=CHANGED' <<<"$drain" || fail 'drain marker was not created' [[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'drain state is absent' +rm "$root/var/lib/ci-fleet-deployer/drained" +printf 'unrelated-drain-target\n' >"$tmp/unrelated-drain-target" +ln -s "$tmp/unrelated-drain-target" "$root/var/lib/ci-fleet-deployer/drained" +expect_failure 'drain marker must be a regular file, not a symlink' "$installer" --uninstall --config "$config" >/dev/null +[[ $(<"$tmp/unrelated-drain-target") == unrelated-drain-target ]] || fail 'uninstall followed an unsafe drain marker' +rm "$root/var/lib/ci-fleet-deployer/drained" +install -m 0600 /dev/null "$root/var/lib/ci-fleet-deployer/drained" unrelated=$root/var/lib/ci-fleet-deployer/operator-note printf 'preserve\n' >"$unrelated" @@ -465,6 +493,11 @@ p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20 PY expect_failure 'deployment request does not match protected approval APPROVAL_ID' "$runtime" deploy >/dev/null cp "$approval" "$request"; chmod 0600 "$request" +mv "$checkpoint" "$checkpoint.saved" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'checkpoint evidence must be a regular file' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran without checkpoint evidence' +mv "$checkpoint.saved" "$checkpoint" printf 'unrelated-audit\n' >"$tmp/unrelated-audit" ln -s "$tmp/unrelated-audit" "$root/var/log/ci-fleet-deployer/audit.log" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) @@ -473,7 +506,8 @@ expect_failure 'deployer audit log must be a regular file, not a symlink' "$runt rm "$root/var/log/ci-fleet-deployer/audit.log" expect_success "$runtime" deploy >/dev/null [[ ! -e "$request" && -f "$root/var/lib/ci-fleet-deployer/last-request.conf" ]] || fail 'completed request was not consumed atomically' -cp "$root/var/lib/ci-fleet-deployer/last-request.conf" "$request"; chmod 0600 "$request" +{ printf '# semantic replay with different bytes\n'; tac "$root/var/lib/ci-fleet-deployer/last-request.conf"; } >"$request" +chmod 0600 "$request" expect_failure 'deployment request was already completed' "$runtime" deploy >/dev/null grep -Fq 'sha256:bbbbbbbb' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'audit log omitted the immutable digest' if grep -Fq 'registry.example.invalid' "$root/var/log/ci-fleet-deployer/audit.log"; then fail 'audit log exposed a private-capable endpoint'; fi From 19a0c4e2007bde630802f48494caa07c910106f2 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 8 Aug 2026 22:53:39 -0500 Subject: [PATCH 04/90] fix: retain deployed safety evidence --- docs/DEPLOYER-HOST.md | 2 +- scripts/deployer-runtime.sh | 52 ++++++++++++++++++++++++++++++-- scripts/install-deployer.sh | 21 ++++++++++--- scripts/test-install-deployer.sh | 40 ++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 7 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index a1df26c..ede2464 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -243,7 +243,7 @@ A cross-environment or cross-target in-place upgrade is rejected. Build a separa ## Submit an approved deployment -Place one root-owned mode-`0600` request at `/var/lib/ci-fleet-deployer/request.conf`. It uses the same exact-head approval fields shown above. No secret values belong in the request. Before invoking the adapter, runtime revalidates the protected approval and checkpoint evidence. After success it atomically moves the request to `last-request.conf`; resubmitting the same semantic request fails closed regardless of field order or comments. +Place one root-owned mode-`0600` request at `/var/lib/ci-fleet-deployer/request.conf`. It uses the same exact-head approval fields shown above. No secret values belong in the request. Before invoking the adapter, runtime revalidates host-role isolation and all protected approval, capability, checkpoint, and production evidence, then permanently consumes the semantic request identity; failed attempts require fresh approval. After success it records the deployed policy as the application-compatible rollback point and atomically moves the request to `last-request.conf`. Then run on the deployer host; this changes the application target through its adapter: diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 149b3ed..0cb3b2a 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -19,7 +19,12 @@ evidence_dir=$(root_path /etc/ci-fleet-deployer/evidence) active=$state_root/active-operation drained=$state_root/drained last_request=$state_root/last-request.conf +consumed_root=$state_root/consumed-requests +install_state=$state_root/install-state.json +deployed_policy=$state_root/deployed-policy.conf +deployed_state=$state_root/deployed-state.json audit_log=$log_root/audit.log +systemd_root=$(root_path /etc/systemd/system) die() { printf 'ERROR: %s\n' "$*" >&2; exit 2; } expected_uid=0 @@ -34,6 +39,27 @@ secure_directory() { local path=$1 description=$2 [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:700" ]] || die "$description has unsafe owner, mode, or type" } +reject_mixed_role() { + local unit runner_unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" + for unit in ci-fleet-health.service ci-fleet-reconcile.service ci-fleet-cleanup.service actions.runner.service; do + [[ ! -e "$systemd_root/$unit" ]] || die 'ordinary CI controller or runner state is present' + done + shopt -s nullglob + for runner_unit in "$systemd_root"/actions.runner.*.service "$systemd_root"/multi-user.target.wants/actions.runner.*.service; do + shopt -u nullglob + [[ -n "$runner_unit" ]] && die 'ordinary GitHub Actions runner service is present' + done + shopt -u nullglob + for path in "$(root_path /etc/ci-fleet/ci-fleet.env)" "$(root_path /opt/ci-fleet/current)" "$(root_path /var/lib/ci-fleet/install-state.json)"; do + [[ ! -e "$path" && ! -L "$path" ]] || die 'ordinary CI controller or runner state is present' + done + output=$(docker ps -a --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || die 'Docker workload inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || die 'unrelated Docker workload is present'; done <<<"$output" + output=$(docker network ls --filter type=custom --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || die 'Docker network inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || die 'incompatible custom Docker network is present'; done <<<"$output" + output=$(docker volume ls --format '{{.Name}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || die 'Docker volume inventory failed' + while IFS= read -r line; do [[ -z "$line" || ${line#*|} == "$expected" ]] || die 'incompatible Docker volume is present'; done <<<"$output" +} inside() { local path=$1 base=$2 [[ $(realpath -m -- "$path") == "$(realpath -m -- "$base")/"* ]] @@ -58,7 +84,7 @@ parse_file() { secure_file "$config" 'deployer configuration' config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' parse_file "$config" cfg configuration "$config_keys" -for key in ENVIRONMENT TARGET_ID ADAPTER_PATH ADAPTER_SHA256 CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done +for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 APPROVAL_PROVIDER CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done [[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 @@ -86,6 +112,7 @@ case "$operation" in ;; deploy) [[ ! -e "$drained" ]] || die 'deployer is drained' + reject_mixed_role secure_file "$request" 'deployment request' request_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' parse_file "$request" req request "$request_keys" @@ -123,6 +150,16 @@ case "$operation" in for key in $approval_keys; do [[ -v "approved[$key]" && ${req[$key]} == "${approved[$key]}" ]] || die "deployment request does not match protected approval $key" done + if [[ ${cfg[APPROVAL_PROVIDER]} == github-environment ]]; then + [[ -v 'cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]' ]] || die 'GitHub Environment approval is missing capability evidence' + inside "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" "$evidence_dir" || die 'capability evidence is outside the protected evidence directory' + secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' + capability_keys='SCHEMA_VERSION ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' + parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' "$capability_keys" + for key in $capability_keys; do [[ -v "capability[$key]" ]] || die "capability evidence is missing $key"; done + [[ ${capability[SCHEMA_VERSION]} == 1 && ${capability[ENVIRONMENT_PROTECTION]} == verified && ${capability[EXACT_HEAD]} == "${req[SOURCE_COMMIT]}" ]] || die 'GitHub Environment capability evidence is not exact-head verified' + [[ ${capability[CAPABILITY_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${capability[CHECKED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'GitHub Environment capability evidence is malformed' + fi if [[ ${cfg[ENVIRONMENT]} == production ]]; then [[ -v 'cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]' ]] || die 'production policy is missing separate authorization evidence' inside "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" "$evidence_dir" || die 'production authorization evidence is outside the protected evidence directory' @@ -136,15 +173,26 @@ case "$operation" in [[ -v "production[$key]" && ${req[$key]} == "${production[$key]}" ]] || die "deployment request does not match production authorization $key" done fi + if [[ -e "$consumed_root" || -L "$consumed_root" ]]; then secure_directory "$consumed_root" 'consumed request directory'; else install -d -m 0700 "$consumed_root"; fi + request_id=$(for key in $request_keys; do printf '%s=%s\0' "$key" "${req[$key]}"; done | sha256sum | cut -d' ' -f1) + consumed_marker=$consumed_root/$request_id + [[ ! -e "$consumed_marker" && ! -L "$consumed_marker" ]] || die 'deployment request was already consumed' if [[ -e "$audit_log" || -L "$audit_log" ]]; then secure_file "$audit_log" 'deployer audit log'; else install -m 0600 /dev/null "$audit_log"; fi : >>"$audit_log" + secure_file "$install_state" 'deployer install state' + for path in "$deployed_policy" "$deployed_state"; do [[ ! -e "$path" && ! -L "$path" ]] || secure_file "$path" 'deployed rollback state'; done + install -m 0600 "$config" "$state_root/.deployed-policy.new" + install -m 0600 "$install_state" "$state_root/.deployed-state.new" + install -m 0600 /dev/null "$consumed_marker" umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" mv -Tf "$temporary" "$active" - trap 'rm -f "$active"' EXIT INT TERM + trap 'rm -f "$active" "$state_root/.deployed-policy.new" "$state_root/.deployed-state.new"' EXIT INT TERM systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ --why='approved deployment is active' -- "${cfg[ADAPTER_PATH]}" deploy + mv -Tf "$state_root/.deployed-policy.new" "$deployed_policy" + mv -Tf "$state_root/.deployed-state.new" "$deployed_state" mv -Tf "$request" "$last_request" printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=success\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 713c9df..55ba0b7 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -80,6 +80,8 @@ state_file=$state_root/install-state.json active_policy=$state_root/active-policy.conf previous_state=$state_root/last-known-good.json previous_policy=$state_root/last-known-good-policy.conf +deployed_state=$state_root/deployed-state.json +deployed_policy=$state_root/deployed-policy.conf drained=$state_root/drained active_operation=$state_root/active-operation lock_root=$(root_path /var/lock/ci-fleet-deployer) @@ -310,9 +312,10 @@ active_deployment() { pid=$(awk -F= '$1=="pid" {print $2}' "$active_operation") started=$(awk -F= '$1=="started_at" {print $2}' "$active_operation") [[ "$pid" =~ ^[1-9][0-9]*$ && "$started" =~ ^[0-9]+$ ]] || return 0 - if kill -0 "$pid" 2>/dev/null; then return 0; fi now=$(date +%s) - ((now - started <= 3600)) + ((now - started <= 3600)) || return 1 + kill -0 "$pid" 2>/dev/null || return 0 + return 0 } release_complete() { @@ -488,6 +491,8 @@ recover_interrupted_transaction() { finalize_committed_rollback() { local marker=$transaction_dir/application-rollback-committed [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' + install -m 0600 "$active_policy" "$deployed_policy" + install -m 0600 "$state_file" "$deployed_state" rm -f "$previous_state" "$previous_policy" transaction_committed=1 rm -rf -- "$transaction_dir" @@ -640,8 +645,12 @@ PY secure_directory "$log_root" 700 1 begin_transaction if ((had_state && candidate_changed)); then - install -m 0600 "$state_file" "$previous_state" - install -m 0600 "$active_policy" "$previous_policy" + if [[ -e "$deployed_state" || -L "$deployed_state" || -e "$deployed_policy" || -L "$deployed_policy" ]]; then + secure_file "$deployed_state" 'deployed rollback state' + secure_file "$deployed_policy" 'deployed rollback policy' + install -m 0600 "$deployed_state" "$previous_state" + install -m 0600 "$deployed_policy" "$previous_policy" + fi fi ln -sfn "releases/$core_ref" "$install_root/.current.new" install_units @@ -652,6 +661,10 @@ PY mv -Tf "$install_root/.current.new" "$current" rm -f "$drained" policy_adapter_operation "$active_policy" health 'candidate policy' || die 'candidate health check failed after activation' + if [[ ! -e "$deployed_state" && ! -L "$deployed_state" && ! -e "$deployed_policy" && ! -L "$deployed_policy" ]]; then + install -m 0600 "$state_file" "$deployed_state" + install -m 0600 "$active_policy" "$deployed_policy" + fi commit_transaction health=healthy report CHANGED yes run-check "$(rollback_available)" diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index a0ff1d9..720be68 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -349,6 +349,10 @@ chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" expect_success "$installer" --repair --config "$config" >/dev/null [[ ! -e "$stale_stage" && ! -e "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'bounded stale transaction recovery did not converge' +printf 'pid=%s\nstarted_at=1\n' "$$" >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'old operation marker survived PID reuse' exec 8<"$root/var/lock/ci-fleet-deployer" flock -n 8 || fail 'fixture could not acquire installer lock' @@ -422,6 +426,13 @@ rm "$FAKE_ADAPTER_FAIL"; unset FAKE_ADAPTER_FAIL upgrade=$(expect_success "$installer" --upgrade --config "$config") grep -Fq 'result=CHANGED' <<<"$upgrade" || fail 'upgrade did not activate new immutable artifact' +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=approval-20260808-2')) +PY +expect_success "$installer" --upgrade --config "$config" >/dev/null +grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/last-known-good.json" || fail 'second undeployed upgrade replaced the deployed rollback point' printf 'rollback\n' >"$tmp/fail-rollback" export FAKE_ADAPTER_FAIL=$tmp/fail-rollback expect_failure 'application adapter rollback failed' "$installer" --rollback --config "$config" >/dev/null @@ -479,6 +490,13 @@ repeat_uninstall=$(expect_success "$installer" --uninstall --config "$config") grep -Fq 'result=NO_CHANGE' <<<"$repeat_uninstall" || fail 'repeated uninstall was not idempotent' # Runtime contract: exact-head request/evidence, drain and scoped adapter calls. +write_evidence staging example-staging +write_config staging example-staging +python3 - "$config" "$capability" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=manual-exact-head', 'APPROVAL_PROVIDER=github-environment') + 'APPROVAL_CAPABILITY_EVIDENCE_PATH='+sys.argv[2]+'\n') +PY expect_success "$installer" --install --config "$config" >/dev/null request=$root/var/lib/ci-fleet-deployer/request.conf cp "$approval" "$request" @@ -486,6 +504,11 @@ chmod 0600 "$request" export CI_FLEET_DEPLOYER_CONFIG=$config CI_FLEET_DEPLOYER_REQUEST=$request expect_success "$runtime" health >/dev/null expect_success "$runtime" cleanup >/dev/null +printf 'runner\n' >"$root/etc/systemd/system/actions.runner.late-added.service" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran after role isolation drift' +rm "$root/etc/systemd/system/actions.runner.late-added.service" python3 - "$request" <<'PY' from pathlib import Path import sys @@ -498,6 +521,11 @@ deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'checkpoint evidence must be a regular file' "$runtime" deploy >/dev/null [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran without checkpoint evidence' mv "$checkpoint.saved" "$checkpoint" +mv "$capability" "$capability.saved" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'GitHub capability evidence must be a regular file' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran without GitHub capability evidence' +mv "$capability.saved" "$capability" printf 'unrelated-audit\n' >"$tmp/unrelated-audit" ln -s "$tmp/unrelated-audit" "$root/var/log/ci-fleet-deployer/audit.log" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) @@ -509,6 +537,18 @@ expect_success "$runtime" deploy >/dev/null { printf '# semantic replay with different bytes\n'; tac "$root/var/lib/ci-fleet-deployer/last-request.conf"; } >"$request" chmod 0600 "$request" expect_failure 'deployment request was already completed' "$runtime" deploy >/dev/null +cp "$root/var/lib/ci-fleet-deployer/last-request.conf" "$tmp/request-a" +cp "$approval" "$tmp/approval-a" +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=approval-20260808-2').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:01:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +expect_success "$runtime" deploy >/dev/null +cp "$tmp/approval-a" "$approval"; chmod 0600 "$approval" +cp "$tmp/request-a" "$request"; chmod 0600 "$request" +expect_failure 'deployment request was already consumed' "$runtime" deploy >/dev/null grep -Fq 'sha256:bbbbbbbb' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'audit log omitted the immutable digest' if grep -Fq 'registry.example.invalid' "$root/var/log/ci-fleet-deployer/audit.log"; then fail 'audit log exposed a private-capable endpoint'; fi for operation in health cleanup deploy; do grep -Fxq "$operation" "$FAKE_ADAPTER_LOG" || fail "runtime did not invoke adapter $operation"; done From 08e1d5de70c15815b4a4ee846fcfd85673f3de73 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 8 Aug 2026 23:32:52 -0500 Subject: [PATCH 05/90] fix: close deployer runtime recovery gaps --- docs/DEPLOYER-HOST.md | 4 +- scripts/deployer-runtime.sh | 69 ++++++++++++++++++++------- scripts/install-deployer.sh | 82 +++++++++++++++++++++----------- scripts/test-install-deployer.sh | 80 +++++++++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 46 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index ede2464..08fbff4 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -215,7 +215,7 @@ sudo ./scripts/install-deployer.sh \ --upgrade --config /etc/ci-fleet-deployer/deployer.conf ``` -The adapter candidate passes `validate` before activation. Core stages a complete commit-pinned release, atomically switches it, writes active policy/state with mode `0600`, and keeps the prior state and policy as last-known-good. A failed candidate does not replace the current release. +The adapter candidate passes `validate` before activation. Core stages a complete commit-pinned release, atomically switches it, writes active policy/state with mode `0600`, and derives last-known-good from the atomically published deployed policy/state snapshot rather than an undeployed candidate. A failed candidate does not replace the current release or remove drain state. Drain before reboot or maintenance; changes drain state and refuses while a deployment is active: @@ -225,7 +225,7 @@ sudo ./scripts/install-deployer.sh \ sudo systemctl start ci-fleet-deployer-drain.service ``` -Rollback changes the active core/application state but deliberately does not overwrite the operator-owned desired configuration or evidence. It refuses while a deployment is active: +Rollback changes the active core/application state but deliberately does not overwrite or depend on the operator-owned candidate evidence or registry preflight; it uses the retained last-known-good policy and local adapter. It refuses while a deployment is active: ```bash sudo ./scripts/install-deployer.sh \ diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 0cb3b2a..2de1292 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -21,8 +21,8 @@ drained=$state_root/drained last_request=$state_root/last-request.conf consumed_root=$state_root/consumed-requests install_state=$state_root/install-state.json -deployed_policy=$state_root/deployed-policy.conf -deployed_state=$state_root/deployed-state.json +deployed_root=$state_root/deployed +deployed_current=$deployed_root/current audit_log=$log_root/audit.log systemd_root=$(root_path /etc/systemd/system) @@ -64,6 +64,16 @@ inside() { local path=$1 base=$2 [[ $(realpath -m -- "$path") == "$(realpath -m -- "$base")/"* ]] } +valid_utc() { + local value=$1 + [[ "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] && [[ $(date -u -d "$value" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null) == "$value" ]] +} +not_drained() { + if [[ -e "$drained" || -L "$drained" ]]; then + secure_file "$drained" 'drain marker' + die 'deployer is drained' + fi +} parse_file() { local path=$1 prefix=$2 kind=$3 allowed=$4 line key value declare -gA "$prefix=()" @@ -84,7 +94,7 @@ parse_file() { secure_file "$config" 'deployer configuration' config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' parse_file "$config" cfg configuration "$config_keys" -for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 APPROVAL_PROVIDER CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done +for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done [[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 @@ -107,11 +117,13 @@ case "$operation" in "${cfg[ADAPTER_PATH]}" "$operation" ;; cleanup) - [[ ! -e "$drained" ]] || die 'deployer is drained' + not_drained + reject_mixed_role "${cfg[ADAPTER_PATH]}" cleanup ;; deploy) - [[ ! -e "$drained" ]] || die 'deployer is drained' + not_drained + [[ ! -e "$active" && ! -L "$active" ]] || die 'active operation marker requires recovery' reject_mixed_role secure_file "$request" 'deployment request' request_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' @@ -125,7 +137,7 @@ case "$operation" in done [[ ${req[SOURCE_COMMIT]} =~ ^[0-9a-f]{40}$ && ${req[ARTIFACT_IMAGE]} =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?/[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || die 'deployment request is not immutable and qualified' for key in APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID; do [[ ${req[$key]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ ]] || die "deployment request has an unsafe $key"; done - [[ ${req[APPROVED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'deployment request has an invalid approval time' + valid_utc "${req[APPROVED_AT]}" || die 'deployment request has an invalid approval time' if [[ -e "$last_request" || -L "$last_request" ]]; then secure_file "$last_request" 'last completed deployment request' parse_file "$last_request" completed 'last completed deployment request' "$request_keys" @@ -141,7 +153,7 @@ case "$operation" in parse_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" checkpoint 'checkpoint evidence' "$checkpoint_keys" for key in $checkpoint_keys; do [[ -v "checkpoint[$key]" ]] || die "checkpoint evidence is missing $key"; done [[ ${checkpoint[SCHEMA_VERSION]} == 1 && ${checkpoint[ENVIRONMENT]} == "${req[ENVIRONMENT]}" && ${checkpoint[TARGET_ID]} == "${req[TARGET_ID]}" ]] || die 'checkpoint evidence does not match the deployment target' - [[ ${checkpoint[CHECKPOINT_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${checkpoint[RECORDED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'checkpoint evidence is malformed' + if [[ ! ${checkpoint[CHECKPOINT_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${checkpoint[RECORDED_AT]}"; then die 'checkpoint evidence is malformed'; fi [[ -v 'cfg[APPROVAL_EVIDENCE_PATH]' ]] || die 'installed policy is missing approval evidence' inside "${cfg[APPROVAL_EVIDENCE_PATH]}" "$evidence_dir" || die 'approval evidence is outside the protected evidence directory' secure_file "${cfg[APPROVAL_EVIDENCE_PATH]}" 'approval evidence' @@ -158,7 +170,7 @@ case "$operation" in parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' "$capability_keys" for key in $capability_keys; do [[ -v "capability[$key]" ]] || die "capability evidence is missing $key"; done [[ ${capability[SCHEMA_VERSION]} == 1 && ${capability[ENVIRONMENT_PROTECTION]} == verified && ${capability[EXACT_HEAD]} == "${req[SOURCE_COMMIT]}" ]] || die 'GitHub Environment capability evidence is not exact-head verified' - [[ ${capability[CAPABILITY_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${capability[CHECKED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'GitHub Environment capability evidence is malformed' + if [[ ! ${capability[CAPABILITY_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${capability[CHECKED_AT]}"; then die 'GitHub Environment capability evidence is malformed'; fi fi if [[ ${cfg[ENVIRONMENT]} == production ]]; then [[ -v 'cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]' ]] || die 'production policy is missing separate authorization evidence' @@ -168,35 +180,58 @@ case "$operation" in parse_file "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" production 'production authorization evidence' "$production_keys" for key in $production_keys; do [[ -v "production[$key]" ]] || die "production authorization evidence is missing $key"; done [[ ${production[SCHEMA_VERSION]} == 1 && ${production[ENVIRONMENT]} == production ]] || die 'production authorization evidence has the wrong scope' - [[ ${production[AUTHORIZED_BY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${production[GATE_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${production[AUTHORIZED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || die 'production authorization evidence is malformed' + if [[ ! ${production[AUTHORIZED_BY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ || ! ${production[GATE_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${production[AUTHORIZED_AT]}"; then die 'production authorization evidence is malformed'; fi for key in ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "production[$key]" && ${req[$key]} == "${production[$key]}" ]] || die "deployment request does not match production authorization $key" done fi + [[ ${cfg[CREDENTIAL_SCOPE]} == "${cfg[ENVIRONMENT]}" ]] || die 'credential scope does not match the deployment environment' + case ${cfg[CREDENTIAL_PROVIDER]} in + file) + inside "${cfg[CREDENTIAL_REF]}" "$(root_path /etc/ci-fleet-deployer/credentials)" || die 'credential reference is outside the protected credential directory' + secure_file "${cfg[CREDENTIAL_REF]}" 'credential file' + ;; + external) [[ ${cfg[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die 'external credential reference is malformed' ;; + *) die 'unsupported credential provider' ;; + esac if [[ -e "$consumed_root" || -L "$consumed_root" ]]; then secure_directory "$consumed_root" 'consumed request directory'; else install -d -m 0700 "$consumed_root"; fi request_id=$(for key in $request_keys; do printf '%s=%s\0' "$key" "${req[$key]}"; done | sha256sum | cut -d' ' -f1) consumed_marker=$consumed_root/$request_id [[ ! -e "$consumed_marker" && ! -L "$consumed_marker" ]] || die 'deployment request was already consumed' if [[ -e "$audit_log" || -L "$audit_log" ]]; then secure_file "$audit_log" 'deployer audit log'; else install -m 0600 /dev/null "$audit_log"; fi - : >>"$audit_log" + exec 8>>"$audit_log" secure_file "$install_state" 'deployer install state' - for path in "$deployed_policy" "$deployed_state"; do [[ ! -e "$path" && ! -L "$path" ]] || secure_file "$path" 'deployed rollback state'; done - install -m 0600 "$config" "$state_root/.deployed-policy.new" - install -m 0600 "$install_state" "$state_root/.deployed-state.new" + if [[ -e "$deployed_root" || -L "$deployed_root" ]]; then secure_directory "$deployed_root" 'deployed snapshot directory'; else install -d -m 0700 "$deployed_root"; fi + if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then + [[ -L "$deployed_current" ]] || die 'deployed snapshot pointer is absent or unsafe' + deployed_snapshot=$(readlink -f "$deployed_current") + inside "$deployed_snapshot" "$deployed_root" || die 'deployed snapshot pointer escapes managed state' + secure_directory "$deployed_snapshot" 'deployed snapshot' + secure_file "$deployed_snapshot/policy.conf" 'deployed rollback policy' + secure_file "$deployed_snapshot/state.json" 'deployed rollback state' + fi + snapshot=$(mktemp -d "$deployed_root/.snapshot.XXXXXX") + chmod 0700 "$snapshot" + install -m 0600 "$config" "$snapshot/policy.conf" + install -m 0600 "$install_state" "$snapshot/state.json" install -m 0600 /dev/null "$consumed_marker" umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" mv -Tf "$temporary" "$active" - trap 'rm -f "$active" "$state_root/.deployed-policy.new" "$state_root/.deployed-state.new"' EXIT INT TERM + trap 'rm -f "$active"; [[ -z ${snapshot:-} || ! -d $snapshot ]] || rm -rf -- "$snapshot"' EXIT INT TERM systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ --why='approved deployment is active' -- "${cfg[ADAPTER_PATH]}" deploy - mv -Tf "$state_root/.deployed-policy.new" "$deployed_policy" - mv -Tf "$state_root/.deployed-state.new" "$deployed_state" + pointer=$deployed_root/.current.$$ + ln -s "${snapshot##*/}" "$pointer" + mv -Tf "$pointer" "$deployed_current" + snapshot= mv -Tf "$request" "$last_request" + secure_file "$audit_log" 'deployer audit log' + [[ $(stat -Lc '%d:%i' /proc/self/fd/8) == $(stat -c '%d:%i' "$audit_log") ]] || die 'deployer audit log changed during deployment' printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=success\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" \ - >>"$audit_log" + >&8 ;; esac diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 55ba0b7..7bc8837 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -80,8 +80,8 @@ state_file=$state_root/install-state.json active_policy=$state_root/active-policy.conf previous_state=$state_root/last-known-good.json previous_policy=$state_root/last-known-good-policy.conf -deployed_state=$state_root/deployed-state.json -deployed_policy=$state_root/deployed-policy.conf +deployed_root=$state_root/deployed +deployed_current=$deployed_root/current drained=$state_root/drained active_operation=$state_root/active-operation lock_root=$(root_path /var/lock/ci-fleet-deployer) @@ -105,6 +105,10 @@ inside() { normalized_base=$(realpath -m -- "$base") [[ "$normalized" == "$normalized_base/"* ]] } +valid_utc() { + local value=$1 + [[ "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] && [[ $(date -u -d "$value" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null) == "$value" ]] +} secure_file() { local path=$1 description=$2 mode=${3:-600} [[ ! -L "$path" && -f "$path" ]] || block "$description must be a regular file, not a symlink" @@ -144,15 +148,16 @@ validate_config() { secure_directory "$etc_root" 700 0 || block 'configuration directory is missing' secure_file "$config" 'configuration file' parse_file "$config" cfg configuration "$config_keys" - local key + local key candidate_environment candidate_target candidate_core candidate_artifact for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID; do [[ -v "cfg[$key]" ]] || block "configuration is missing required key: $key" done [[ ${cfg[SCHEMA_VERSION]} == 1 ]] || block 'unsupported configuration schema' - environment=${cfg[ENVIRONMENT]}; target=${cfg[TARGET_ID]}; core_ref=${cfg[CORE_REF]:-unknown}; artifact=${cfg[ARTIFACT_IMAGE]:-unknown} - [[ "$environment" =~ ^[a-z][a-z0-9-]{0,31}$ ]] || block 'invalid explicit environment' - [[ "$target" =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'invalid explicit target identity' - if [[ "$mode" == drain || "$mode" == uninstall ]]; then return; fi + candidate_environment=${cfg[ENVIRONMENT]}; candidate_target=${cfg[TARGET_ID]} + [[ "$candidate_environment" =~ ^[a-z][a-z0-9-]{0,31}$ ]] || block 'invalid explicit environment' + [[ "$candidate_target" =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'invalid explicit target identity' + environment=$candidate_environment; target=$candidate_target + if [[ "$mode" == drain || "$mode" == uninstall || "$mode" == rollback ]]; then return; fi secure_directory "$etc_root/adapters" 700 0 || block 'adapter directory is missing' secure_directory "$etc_root/credentials" 700 0 || block 'credential directory is missing' secure_directory "$etc_root/evidence" 700 0 || block 'evidence directory is missing' @@ -161,8 +166,10 @@ validate_config() { done [[ ${cfg[DEPLOYER_IDENTITY]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'invalid deployer identity' [[ ${cfg[CREDENTIAL_SCOPE]} == "$environment" ]] || block 'credential scope must exactly match the explicit environment' - [[ "$core_ref" =~ ^[0-9a-f]{40}$ && ${cfg[SOURCE_COMMIT]} =~ ^[0-9a-f]{40}$ ]] || block 'core and source revisions must be full lowercase commit SHAs' - [[ ${cfg[ARTIFACT_IMAGE]} =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?/[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || block 'artifact image must be an immutable qualified digest reference' + candidate_core=${cfg[CORE_REF]}; candidate_artifact=${cfg[ARTIFACT_IMAGE]} + [[ "$candidate_core" =~ ^[0-9a-f]{40}$ && ${cfg[SOURCE_COMMIT]} =~ ^[0-9a-f]{40}$ ]] || block 'core and source revisions must be full lowercase commit SHAs' + [[ "$candidate_artifact" =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?/[a-z0-9][a-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]] || block 'artifact image must be an immutable qualified digest reference' + core_ref=$candidate_core; artifact=$candidate_artifact [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || block 'adapter digest must be lowercase SHA-256' [[ ${cfg[NETWORK_HOST]} =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$ ]] || block 'invalid network prerequisite host' [[ ${cfg[MIN_DISK_GIB]} =~ ^[1-9][0-9]{0,3}$ ]] || block 'MIN_DISK_GIB must be a positive integer' @@ -201,7 +208,7 @@ validate_production_gate() { [[ ${production_gate[SCHEMA_VERSION]} == 1 && ${production_gate[ENVIRONMENT]} == production ]] || block 'production authorization evidence has the wrong scope' for key in TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do [[ ${production_gate[$key]} == "${cfg[$key]}" ]] || block "production authorization evidence does not match exact $key"; done [[ ${production_gate[AUTHORIZED_BY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${production_gate[GATE_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || block 'production authorization identity is malformed' - [[ ${production_gate[AUTHORIZED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || block 'production authorization timestamp must be UTC RFC3339' + valid_utc "${production_gate[AUTHORIZED_AT]}" || block 'production authorization timestamp must be UTC RFC3339' } validate_evidence() { @@ -218,7 +225,7 @@ validate_evidence() { [[ ${approval[$key]} == "${cfg[$key]}" ]] || block "approval evidence does not match exact $key" done [[ ${approval[APPROVAL_IDENTITY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${approval[POLICY_IDENTITY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${approval[APPROVAL_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || block 'approval identity is malformed' - [[ ${approval[APPROVED_AT]} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || block 'approval timestamp must be UTC RFC3339' + valid_utc "${approval[APPROVED_AT]}" || block 'approval timestamp must be UTC RFC3339' case ${cfg[APPROVAL_PROVIDER]} in manual-exact-head|external-exact-head) ;; github-environment) @@ -227,7 +234,7 @@ validate_evidence() { secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' 'SCHEMA_VERSION ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' [[ ${capability[SCHEMA_VERSION]:-} == 1 && ${capability[ENVIRONMENT_PROTECTION]:-} == verified && ${capability[EXACT_HEAD]:-} == "${cfg[SOURCE_COMMIT]}" ]] || block 'GitHub Environment capability evidence is not exact-head verified' - [[ ${capability[CAPABILITY_ID]:-} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${capability[CHECKED_AT]:-} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || block 'GitHub Environment capability evidence is missing identity or UTC time' + if [[ ! ${capability[CAPABILITY_ID]:-} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${capability[CHECKED_AT]:-}"; then block 'GitHub Environment capability evidence is missing identity or UTC time'; fi ;; *) block 'unsupported approval provider' ;; esac @@ -235,7 +242,7 @@ validate_evidence() { secure_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" 'checkpoint evidence' parse_file "${cfg[CHECKPOINT_EVIDENCE_PATH]}" checkpoint 'checkpoint evidence' 'SCHEMA_VERSION ENVIRONMENT TARGET_ID CHECKPOINT_ID RECORDED_AT' [[ ${checkpoint[SCHEMA_VERSION]:-} == 1 && ${checkpoint[ENVIRONMENT]:-} == "$environment" && ${checkpoint[TARGET_ID]:-} == "$target" ]] || block 'checkpoint evidence does not match the explicit environment and target' - [[ ${checkpoint[CHECKPOINT_ID]:-} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${checkpoint[RECORDED_AT]:-} =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || block 'checkpoint evidence is malformed' + if [[ ! ${checkpoint[CHECKPOINT_ID]:-} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${checkpoint[RECORDED_AT]:-}"; then block 'checkpoint evidence is malformed'; fi } require_host() { @@ -269,7 +276,7 @@ require_host() { require_maintenance_host() { local command - for command in bash awk stat readlink realpath install cp rm mkdir chmod mv flock kill systemctl date; do + for command in bash awk cut stat sha256sum readlink realpath install cp rm mkdir mktemp chmod ln mv flock kill timeout env python3 systemctl systemd-analyze date; do command -v "$command" >/dev/null || block "$command is required for maintenance" done [[ -d "$systemd_root" && ! -L "$systemd_root" ]] || block 'systemd unit directory is unavailable' @@ -491,8 +498,7 @@ recover_interrupted_transaction() { finalize_committed_rollback() { local marker=$transaction_dir/application-rollback-committed [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' - install -m 0600 "$active_policy" "$deployed_policy" - install -m 0600 "$state_file" "$deployed_state" + publish_deployed_snapshot "$active_policy" "$state_file" rm -f "$previous_state" "$previous_policy" transaction_committed=1 rm -rf -- "$transaction_dir" @@ -557,6 +563,32 @@ with open(sys.argv[1],"w",encoding="utf-8") as f: json.dump(dict(zip(keys,values mv -Tf "$temporary" "$destination" } +load_deployed_snapshot() { + local snapshot + secure_directory "$deployed_root" 700 0 || block 'deployed snapshot directory is missing' + [[ -L "$deployed_current" ]] || block 'deployed snapshot pointer is absent or unsafe' + snapshot=$(readlink -f "$deployed_current") + inside "$snapshot" "$deployed_root" || block 'deployed snapshot pointer escapes managed state' + secure_directory "$snapshot" 700 0 || block 'deployed snapshot is unsafe' + secure_file "$snapshot/policy.conf" 'deployed rollback policy' + secure_file "$snapshot/state.json" 'deployed rollback state' + deployed_snapshot_policy=$snapshot/policy.conf + deployed_snapshot_state=$snapshot/state.json +} + +publish_deployed_snapshot() { + local policy=$1 state=$2 snapshot pointer + secure_directory "$deployed_root" 700 1 + if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then load_deployed_snapshot; fi + snapshot=$(mktemp -d "$deployed_root/.snapshot.XXXXXX") + chmod 0700 "$snapshot" + install -m 0600 "$policy" "$snapshot/policy.conf" + install -m 0600 "$state" "$snapshot/state.json" + pointer=$deployed_root/.current.$$ + ln -s "${snapshot##*/}" "$pointer" + mv -Tf "$pointer" "$deployed_current" +} + install_units() { local unit systemd_mode [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' @@ -645,11 +677,10 @@ PY secure_directory "$log_root" 700 1 begin_transaction if ((had_state && candidate_changed)); then - if [[ -e "$deployed_state" || -L "$deployed_state" || -e "$deployed_policy" || -L "$deployed_policy" ]]; then - secure_file "$deployed_state" 'deployed rollback state' - secure_file "$deployed_policy" 'deployed rollback policy' - install -m 0600 "$deployed_state" "$previous_state" - install -m 0600 "$deployed_policy" "$previous_policy" + if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then + load_deployed_snapshot + install -m 0600 "$deployed_snapshot_state" "$previous_state" + install -m 0600 "$deployed_snapshot_policy" "$previous_policy" fi fi ln -sfn "releases/$core_ref" "$install_root/.current.new" @@ -659,13 +690,10 @@ PY mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_root/.install-state.new" "$state_file" mv -Tf "$install_root/.current.new" "$current" - rm -f "$drained" policy_adapter_operation "$active_policy" health 'candidate policy' || die 'candidate health check failed after activation' - if [[ ! -e "$deployed_state" && ! -L "$deployed_state" && ! -e "$deployed_policy" && ! -L "$deployed_policy" ]]; then - install -m 0600 "$state_file" "$deployed_state" - install -m 0600 "$active_policy" "$deployed_policy" - fi + if [[ ! -e "$deployed_current" && ! -L "$deployed_current" ]]; then publish_deployed_snapshot "$active_policy" "$state_file"; fi commit_transaction + rm -f "$drained" health=healthy report CHANGED yes run-check "$(rollback_available)" } @@ -734,7 +762,7 @@ perform_uninstall() { } validate_config -if [[ "$mode" == drain || "$mode" == uninstall ]]; then +if [[ "$mode" == drain || "$mode" == uninstall || "$mode" == rollback ]]; then require_maintenance_host else validate_checkout diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 720be68..0c31ae2 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -109,6 +109,10 @@ cat >"$adapter" <<'EOF' set -Eeuo pipefail printf '%s\n' "$1" >>"${FAKE_ADAPTER_LOG:?}" if [[ ${FAKE_ADAPTER_SLEEP_OPERATION:-} == "$1" ]]; then sleep 2; fi +if [[ "$1" == health && -n ${FAKE_ADAPTER_FAIL_HEALTH_AFTER:-} ]]; then + health_calls=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) + ((health_calls <= FAKE_ADAPTER_FAIL_HEALTH_AFTER)) || exit 42 +fi if [[ -e "${FAKE_ADAPTER_FAIL:-/nonexistent}" ]]; then fail_operation=$(<"$FAKE_ADAPTER_FAIL") [[ "$fail_operation" != all && "$fail_operation" != "$1" ]] || exit 42 @@ -117,6 +121,10 @@ case "$1" in validate|health|cleanup|deploy|rollback) ;; *) exit 2 ;; esac if [[ "$1" == rollback && -n ${CI_FLEET_DEPLOYER_ROLLBACK_COMMIT:-} ]]; then install -m 0600 /dev/null "$CI_FLEET_DEPLOYER_ROLLBACK_COMMIT" fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_AUDIT_PATH:-} ]]; then + rm -f "$FAKE_ADAPTER_AUDIT_PATH" + ln -s "$FAKE_ADAPTER_AUDIT_TARGET" "$FAKE_ADAPTER_AUDIT_PATH" +fi EOF chmod 0700 "$adapter" export FAKE_ADAPTER_LOG=$tmp/adapter.log @@ -204,6 +212,23 @@ rm "$production_gate" write_evidence write_config +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-99-99T99:99:99Z')) +PY +expect_failure 'approval timestamp must be UTC RFC3339' "$installer" --check --config "$config" >/dev/null +write_evidence +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ENVIRONMENT=staging', 'ENVIRONMENT=bad result=CHANGED')) +PY +unsafe_report=$(expect_failure 'invalid explicit environment' "$installer" --check --config "$config") +grep -Fq 'environment=unknown target=unknown' <<<"$unsafe_report" || fail 'report exposed unvalidated configuration fields' +[[ "$unsafe_report" != *'environment=bad result=CHANGED'* ]] || fail 'report allowed field injection' +write_config + # Read-only preflight and strict policy variants fail closed without reading secrets. cp "$root/etc/os-release" "$tmp/os-release" printf 'ID=alpine\nVERSION_ID=3.20\n' >"$root/etc/os-release" @@ -424,6 +449,13 @@ expect_failure 'candidate adapter validation failed' "$installer" --upgrade --co [[ "$old_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'failed candidate replaced healthy state' rm "$FAKE_ADAPTER_FAIL"; unset FAKE_ADAPTER_FAIL +install -m 0600 /dev/null "$root/var/lib/ci-fleet-deployer/drained" +health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) +export FAKE_ADAPTER_FAIL_HEALTH_AFTER=$((health_calls_before + 1)) +expect_failure 'candidate health check failed after activation' "$installer" --upgrade --config "$config" >/dev/null +unset FAKE_ADAPTER_FAIL_HEALTH_AFTER +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'failed convergence removed the drain marker' +rm "$root/var/lib/ci-fleet-deployer/drained" upgrade=$(expect_success "$installer" --upgrade --config "$config") grep -Fq 'result=CHANGED' <<<"$upgrade" || fail 'upgrade did not activate new immutable artifact' python3 - "$approval" <<'PY' @@ -442,7 +474,13 @@ rollback_calls_before=$(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) FAKE_SYSTEMD_VERIFY_EXIT=1 expect_failure 'systemd unit verification failed' "$installer" --rollback --config "$config" >/dev/null [[ $(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) == "$rollback_calls_before" ]] || fail 'application rollback ran before core rollback staging was proven' grep -Fq 'sha256:bbbbbbbb' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'failed rollback did not preserve current core state' +mv "$approval" "$approval.rollback-saved" +mv "$checkpoint" "$checkpoint.rollback-saved" +export FAKE_CURL_EXIT=1 rollback=$(expect_success "$installer" --rollback --config "$config") +unset FAKE_CURL_EXIT +mv "$approval.rollback-saved" "$approval" +mv "$checkpoint.rollback-saved" "$checkpoint" grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'rollback did not restore last-known-good state' grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'rollback state lacks prior artifact' grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$rollback" || fail 'rollback report lacks the exact operator reconciliation action' @@ -504,11 +542,29 @@ chmod 0600 "$request" export CI_FLEET_DEPLOYER_CONFIG=$config CI_FLEET_DEPLOYER_REQUEST=$request expect_success "$runtime" health >/dev/null expect_success "$runtime" cleanup >/dev/null +printf 'pid=999999\nstarted_at=1\n' >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'active operation marker requires recovery' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment replaced an unresolved active marker' +rm "$root/var/lib/ci-fleet-deployer/active-operation" printf 'runner\n' >"$root/etc/systemd/system/actions.runner.late-added.service" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +cleanup_calls_before=$(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" deploy >/dev/null +expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" cleanup >/dev/null [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran after role isolation drift' +[[ $(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) == "$cleanup_calls_before" ]] || fail 'cleanup ran after role isolation drift' rm "$root/etc/systemd/system/actions.runner.late-added.service" +python3 - "$approval" "$request" <<'PY' +from pathlib import Path +import sys +for name in sys.argv[1:]: + p=Path(name); p.write_text(p.read_text().replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-99-99T99:99:99Z')) +PY +expect_failure 'deployment request has an invalid approval time' "$runtime" deploy >/dev/null +write_evidence staging example-staging +cp "$approval" "$request"; chmod 0600 "$request" python3 - "$request" <<'PY' from pathlib import Path import sys @@ -526,14 +582,34 @@ deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'GitHub capability evidence must be a regular file' "$runtime" deploy >/dev/null [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran without GitHub capability evidence' mv "$capability.saved" "$capability" +chmod 0644 "$credential" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'credential file has unsafe owner or mode' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran with unsafe file credentials' +chmod 0600 "$credential" printf 'unrelated-audit\n' >"$tmp/unrelated-audit" ln -s "$tmp/unrelated-audit" "$root/var/log/ci-fleet-deployer/audit.log" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" && $(<"$tmp/unrelated-audit") == unrelated-audit ]] || fail 'unsafe audit storage was touched after adapter execution' rm "$root/var/log/ci-fleet-deployer/audit.log" +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=audit-replacement-attempt')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_AUDIT_TARGET=$tmp/unrelated-audit +expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_AUDIT_PATH FAKE_ADAPTER_AUDIT_TARGET +[[ $(<"$tmp/unrelated-audit") == unrelated-audit ]] || fail 'adapter audit replacement redirected the trusted append' +rm "$root/var/log/ci-fleet-deployer/audit.log" +write_evidence staging example-staging +cp "$approval" "$request"; chmod 0600 "$request" expect_success "$runtime" deploy >/dev/null [[ ! -e "$request" && -f "$root/var/lib/ci-fleet-deployer/last-request.conf" ]] || fail 'completed request was not consumed atomically' +deployed_current=$root/var/lib/ci-fleet-deployer/deployed/current +[[ -L "$deployed_current" && -f "$deployed_current/policy.conf" && -f "$deployed_current/state.json" ]] || fail 'deployed policy/state pair was not published through one pointer' { printf '# semantic replay with different bytes\n'; tac "$root/var/lib/ci-fleet-deployer/last-request.conf"; } >"$request" chmod 0600 "$request" expect_failure 'deployment request was already completed' "$runtime" deploy >/dev/null @@ -555,6 +631,10 @@ for operation in health cleanup deploy; do grep -Fxq "$operation" "$FAKE_ADAPTER : >"$root/var/lib/ci-fleet-deployer/drained" expect_failure 'deployer is drained' "$runtime" deploy >/dev/null expect_failure 'deployer is drained' "$runtime" cleanup >/dev/null +rm "$root/var/lib/ci-fleet-deployer/drained" +ln -s "$tmp/missing-drain-target" "$root/var/lib/ci-fleet-deployer/drained" +expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" deploy >/dev/null +expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" cleanup >/dev/null grep -Fq 'DEPLOYER-HOST.md' "$repo_root/docs/README.md" || fail 'operator index does not link the deployer runbook' [[ -x "$repo_root/scripts/test-deployer-units.sh" ]] || fail 'real systemd unit verification is not wired' From 37d29628fc2bff66a5c6547d298742859a3b3397 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 8 Aug 2026 23:40:57 -0500 Subject: [PATCH 06/90] test: normalize deployer drain marker mode --- scripts/test-install-deployer.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 0c31ae2..44d0471 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -629,6 +629,7 @@ grep -Fq 'sha256:bbbbbbbb' "$root/var/log/ci-fleet-deployer/audit.log" || fail ' if grep -Fq 'registry.example.invalid' "$root/var/log/ci-fleet-deployer/audit.log"; then fail 'audit log exposed a private-capable endpoint'; fi for operation in health cleanup deploy; do grep -Fxq "$operation" "$FAKE_ADAPTER_LOG" || fail "runtime did not invoke adapter $operation"; done : >"$root/var/lib/ci-fleet-deployer/drained" +chmod 0600 "$root/var/lib/ci-fleet-deployer/drained" expect_failure 'deployer is drained' "$runtime" deploy >/dev/null expect_failure 'deployer is drained' "$runtime" cleanup >/dev/null rm "$root/var/lib/ci-fleet-deployer/drained" From 773c3ae74c53714b520285cccf95d3bd1b708a9f Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 00:03:35 -0500 Subject: [PATCH 07/90] fix: serialize runtime policy and recovery state --- docs/DEPLOYER-HOST.md | 7 +++++++ scripts/deployer-runtime.sh | 13 +++++++++---- scripts/install-deployer.sh | 18 ++++++++++++++---- scripts/test-install-deployer.sh | 16 +++++++++++++++- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index 08fbff4..6bbb0a6 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -225,6 +225,13 @@ sudo ./scripts/install-deployer.sh \ sudo systemctl start ci-fleet-deployer-drain.service ``` +After maintenance, explicitly resume through the same serialized installer boundary. This validates and removes only the managed drain marker and is idempotent: + +```bash +sudo ./scripts/install-deployer.sh \ + --resume --config /etc/ci-fleet-deployer/deployer.conf +``` + Rollback changes the active core/application state but deliberately does not overwrite or depend on the operator-owned candidate evidence or registry preflight; it uses the retained last-known-good policy and local adapter. It refuses while a deployment is active: ```bash diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 2de1292..16e824e 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -91,6 +91,15 @@ parse_file() { done <"$path" } +secure_directory "$state_root" 'deployer state directory' +secure_directory "$lock_dir" 'deployer lock directory' +exec 9<"$lock_dir" +flock -n 9 || die 'another deployer operation is running' +shopt -s nullglob +transactions=("$state_root"/.transaction.*) +shopt -u nullglob +((${#transactions[@]} == 0)) || die 'interrupted installer transaction requires recovery' + secure_file "$config" 'deployer configuration' config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' parse_file "$config" cfg configuration "$config_keys" @@ -100,11 +109,7 @@ for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 C secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 [[ $(sha256sum "${cfg[ADAPTER_PATH]}" | cut -d' ' -f1) == "${cfg[ADAPTER_SHA256]}" ]] || die 'application adapter digest mismatch' -secure_directory "$state_root" 'deployer state directory' secure_directory "$log_root" 'deployer log directory' -secure_directory "$lock_dir" 'deployer lock directory' -exec 9<"$lock_dir" -flock -n 9 || die 'another deployer operation is running' case "$operation" in drain) diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 7bc8837..e76e48b 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -33,7 +33,7 @@ trap on_exit EXIT usage() { cat >&2 <<'EOF' -usage: install-deployer.sh --check|--install|--upgrade|--repair|--rollback|--drain|--uninstall --config /etc/ci-fleet-deployer/deployer.conf +usage: install-deployer.sh --check|--install|--upgrade|--repair|--rollback|--drain|--resume|--uninstall --config /etc/ci-fleet-deployer/deployer.conf Modes are explicit and mutually exclusive. Configuration and credential references are host-local; secret values are never accepted as arguments. @@ -50,7 +50,7 @@ block() { error_reported=1; printf 'BLOCKED: %s\n' "$*" >&2; report BLOCKED no r while (($#)); do case "$1" in - --check|--install|--upgrade|--repair|--rollback|--drain|--uninstall) + --check|--install|--upgrade|--repair|--rollback|--drain|--resume|--uninstall) [[ -z "$mode" ]] || die 'select exactly one operating mode' mode=${1#--}; action=$mode; shift ;; --config) (($# >= 2)) || die '--config requires a value'; config=$2; shift 2 ;; @@ -157,7 +157,7 @@ validate_config() { [[ "$candidate_environment" =~ ^[a-z][a-z0-9-]{0,31}$ ]] || block 'invalid explicit environment' [[ "$candidate_target" =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'invalid explicit target identity' environment=$candidate_environment; target=$candidate_target - if [[ "$mode" == drain || "$mode" == uninstall || "$mode" == rollback ]]; then return; fi + if [[ "$mode" == drain || "$mode" == resume || "$mode" == uninstall || "$mode" == rollback ]]; then return; fi secure_directory "$etc_root/adapters" 700 0 || block 'adapter directory is missing' secure_directory "$etc_root/credentials" 700 0 || block 'credential directory is missing' secure_directory "$etc_root/evidence" 700 0 || block 'evidence directory is missing' @@ -737,6 +737,15 @@ perform_drain() { report CHANGED yes safe-to-maintain "$(rollback_available)" } +perform_resume() { + secure_directory "$state_root" 700 0 || block 'deployer state directory is missing' + if active_deployment; then block 'active deployment prevents resume'; fi + if [[ ! -e "$drained" && ! -L "$drained" ]]; then report NO_CHANGE no ready-to-deploy "$(rollback_available)"; return; fi + secure_file "$drained" 'drain marker' + rm -f "$drained" + report CHANGED yes ready-to-deploy "$(rollback_available)" +} + perform_uninstall() { local changed=no unit managed_present=no if [[ -e "$state_root" || -L "$state_root" || -e "$lock_root" || -L "$lock_root" || -e "$current" || -L "$current" ]]; then managed_present=yes; fi @@ -762,7 +771,7 @@ perform_uninstall() { } validate_config -if [[ "$mode" == drain || "$mode" == uninstall || "$mode" == rollback ]]; then +if [[ "$mode" == drain || "$mode" == resume || "$mode" == uninstall || "$mode" == rollback ]]; then require_maintenance_host else validate_checkout @@ -773,5 +782,6 @@ case "$mode" in install|upgrade|repair) acquire_lock; perform_converge ;; rollback) acquire_lock; perform_rollback ;; drain) acquire_lock; perform_drain ;; + resume) acquire_lock; perform_resume ;; uninstall) perform_uninstall ;; esac diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 44d0471..e8f110f 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -502,6 +502,12 @@ rm "$active" drain=$(expect_success "$installer" --drain --config "$config") grep -Fq 'result=CHANGED' <<<"$drain" || fail 'drain marker was not created' [[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'drain state is absent' +resume=$(expect_success "$installer" --resume --config "$config") +grep -Fq 'result=CHANGED' <<<"$resume" || fail 'resume did not clear drain state' +[[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'resume retained drain state' +repeat_resume=$(expect_success "$installer" --resume --config "$config") +grep -Fq 'result=NO_CHANGE' <<<"$repeat_resume" || fail 'repeated resume was not idempotent' +expect_success "$installer" --drain --config "$config" >/dev/null rm "$root/var/lib/ci-fleet-deployer/drained" printf 'unrelated-drain-target\n' >"$tmp/unrelated-drain-target" ln -s "$tmp/unrelated-drain-target" "$root/var/lib/ci-fleet-deployer/drained" @@ -542,6 +548,11 @@ chmod 0600 "$request" export CI_FLEET_DEPLOYER_CONFIG=$config CI_FLEET_DEPLOYER_REQUEST=$request expect_success "$runtime" health >/dev/null expect_success "$runtime" cleanup >/dev/null +mkdir "$root/var/lib/ci-fleet-deployer/.transaction.interrupted" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'interrupted installer transaction requires recovery' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran during an interrupted installer transaction' +rmdir "$root/var/lib/ci-fleet-deployer/.transaction.interrupted" printf 'pid=999999\nstarted_at=1\n' >"$root/var/lib/ci-fleet-deployer/active-operation" chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) @@ -640,7 +651,10 @@ expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" c grep -Fq 'DEPLOYER-HOST.md' "$repo_root/docs/README.md" || fail 'operator index does not link the deployer runbook' [[ -x "$repo_root/scripts/test-deployer-units.sh" ]] || fail 'real systemd unit verification is not wired' grep -Fq 'scripts/test-deployer-units.sh' "$repo_root/scripts/validate.sh" || fail 'repository validation omits systemd unit verification' -for phrase in '--check' '--install' '--upgrade' '--repair' '--drain' '--rollback' '--uninstall' 'manual-exact-head' 'github-environment' 'GitHub Free' 'PRODUCTION_AUTHORIZATION_EVIDENCE_PATH' 'CI_FLEET_DEPLOYER_ROLLBACK_COMMIT' 'application-owned' 'REPORT action='; do +lock_line=$(grep -n 'flock -n 9' "$runtime" | cut -d: -f1) +policy_line=$(grep -n "secure_file \"\$config\" 'deployer configuration'" "$runtime" | cut -d: -f1) +((lock_line < policy_line)) || fail 'runtime loads active policy before acquiring the shared operation lock' +for phrase in '--check' '--install' '--upgrade' '--repair' '--drain' '--resume' '--rollback' '--uninstall' 'manual-exact-head' 'github-environment' 'GitHub Free' 'PRODUCTION_AUTHORIZATION_EVIDENCE_PATH' 'CI_FLEET_DEPLOYER_ROLLBACK_COMMIT' 'application-owned' 'REPORT action='; do grep -Fq -- "$phrase" "$repo_root/docs/DEPLOYER-HOST.md" || fail "deployer runbook omits $phrase" done for unit in "$repo_root"/deploy/deployer/*; do From 61b3d0eace6569b46b8a596d5e3b95f07e390589 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 00:39:37 -0500 Subject: [PATCH 08/90] fix: fail closed across deployer recovery paths --- docs/DEPLOYER-HOST.md | 4 +- scripts/deployer-runtime.sh | 19 +++++---- scripts/install-deployer.sh | 66 +++++++++++++++++++------------- scripts/test-install-deployer.sh | 30 +++++++++++++++ 4 files changed, 83 insertions(+), 36 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index 6bbb0a6..53da107 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -67,7 +67,7 @@ adapter rollback Operations have no interactive input. Zero means success; nonzero means failure. Direct installer validation and health calls are limited to two minutes; rollback is limited to 45 minutes. The adapter must avoid child processes that outlive those bounds, redact logs, and never print credential contents, authorization headers, cookies, private endpoints, or secret-manager responses. Core validates immutable identifiers and approval evidence; it cannot validate application-specific correctness. -Rollback must be atomic from the adapter's perspective: nonzero restores the pre-call application state; zero means rollback health is already verified. For rollback only, core exports `CI_FLEET_DEPLOYER_ROLLBACK_COMMIT`; the adapter atomically creates that root-owned mode-`0600` file as its final successful step. Core stages and selects the last-known-good core before invoking the adapter, restores the newer core on an uncommitted failure, and consumes committed last-known-good state only after that marker exists. +Rollback must be atomic from the adapter's perspective: nonzero restores the pre-call application state; zero means rollback health is already verified. Rollback is exposed only through the transactional installer, never as a direct runtime operation. For rollback only, core exports `CI_FLEET_DEPLOYER_ROLLBACK_COMMIT`; the adapter atomically creates that root-owned mode-`0600` file as its final successful step. Core stages and selects the last-known-good core before invoking the adapter, restores the newer core on an uncommitted failure, and consumes committed last-known-good state only after core/application alignment. A committed rollback interrupted after the adapter returns is finalized from the retained transaction on the next serialized installer operation. ## Prepare host-local files @@ -225,7 +225,7 @@ sudo ./scripts/install-deployer.sh \ sudo systemctl start ci-fleet-deployer-drain.service ``` -After maintenance, explicitly resume through the same serialized installer boundary. This validates and removes only the managed drain marker and is idempotent: +After maintenance, explicitly resume through the same serialized installer boundary. This removes the managed drain marker only after full installed-state convergence, role isolation, and active-policy health pass; it is idempotent: ```bash sudo ./scripts/install-deployer.sh \ diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 16e824e..b8b2008 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -3,7 +3,7 @@ set -Eeuo pipefail set +x operation=${1:-} -case "$operation" in health|cleanup|deploy|rollback|drain) ;; *) printf 'ERROR: usage: deployer-runtime.sh health|cleanup|deploy|rollback|drain\n' >&2; exit 2 ;; esac +case "$operation" in health|cleanup|deploy|drain) ;; *) printf 'ERROR: usage: deployer-runtime.sh health|cleanup|deploy|drain\n' >&2; exit 2 ;; esac root=${CI_FLEET_DEPLOYER_ROOT:-} testing=${CI_FLEET_DEPLOYER_TESTING:-0} @@ -100,6 +100,14 @@ transactions=("$state_root"/.transaction.*) shopt -u nullglob ((${#transactions[@]} == 0)) || die 'interrupted installer transaction requires recovery' +if [[ "$operation" == drain ]]; then + [[ ! -e "$active" && ! -L "$active" ]] || die 'active deployment prevents drain' + temporary=$(mktemp "$state_root/.drained.XXXXXX") + chmod 0600 "$temporary" + mv -Tf "$temporary" "$drained" + exit 0 +fi + secure_file "$config" 'deployer configuration' config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' parse_file "$config" cfg configuration "$config_keys" @@ -112,13 +120,8 @@ secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 secure_directory "$log_root" 'deployer log directory' case "$operation" in - drain) - [[ ! -e "$active" ]] || die 'active deployment prevents drain' - temporary=$(mktemp "$state_root/.drained.XXXXXX") - chmod 0600 "$temporary" - mv -Tf "$temporary" "$drained" - ;; - health|rollback) + health) + reject_mixed_role "${cfg[ADAPTER_PATH]}" "$operation" ;; cleanup) diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index e76e48b..9cad76d 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -21,9 +21,15 @@ transaction_dir= transaction_committed=0 recovered_rollback=0 on_exit() { - local status=$? + local status=$? recovery_status=0 if [[ -n ${transaction_dir:-} && ${transaction_committed:-0} != 1 ]]; then - if [[ -f "$transaction_dir/application-rollback-committed" ]]; then finalize_committed_rollback || true; else restore_transaction || true; fi + set +e + if [[ -f "$transaction_dir/application-rollback-committed" ]]; then finalize_committed_rollback; recovery_status=$?; else restore_transaction; recovery_status=$?; fi + set -e + if ((recovery_status != 0)); then + status=$recovery_status + printf 'ERROR: transaction recovery failed; retained %s for the next installer recovery\n' "$transaction_dir" >&2 + fi fi [[ -z ${staging_path:-} ]] || rm -rf -- "$staging_path" if ((status != 0 && error_reported == 0)); then report FAILED no inspect-and-retry "$(rollback_available)" >&2; fi @@ -157,7 +163,7 @@ validate_config() { [[ "$candidate_environment" =~ ^[a-z][a-z0-9-]{0,31}$ ]] || block 'invalid explicit environment' [[ "$candidate_target" =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'invalid explicit target identity' environment=$candidate_environment; target=$candidate_target - if [[ "$mode" == drain || "$mode" == resume || "$mode" == uninstall || "$mode" == rollback ]]; then return; fi + if [[ "$mode" == drain || "$mode" == uninstall || "$mode" == rollback ]]; then return; fi secure_directory "$etc_root/adapters" 700 0 || block 'adapter directory is missing' secure_directory "$etc_root/credentials" 700 0 || block 'credential directory is missing' secure_directory "$etc_root/evidence" 700 0 || block 'evidence directory is missing' @@ -442,35 +448,35 @@ restore_transaction() { local name target_value [[ -n ${transaction_dir:-} && -d $transaction_dir ]] || return 0 transaction_committed=1 - systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true - for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name"; done + systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || return + for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name" || return; done if [[ -f "$transaction_dir/units-present" ]]; then while IFS= read -r name; do [[ " ${unit_names[*]} " == *" $name "* && -f "$transaction_dir/units/$name" && ! -L "$transaction_dir/units/$name" ]] || block 'transaction unit manifest is unsafe' - install -m 0644 "$transaction_dir/units/$name" "$systemd_root/$name" + install -m 0644 "$transaction_dir/units/$name" "$systemd_root/$name" || return done <"$transaction_dir/units-present" fi - for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do rm -f -- "$state_root/$name"; done + for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do rm -f -- "$state_root/$name" || return; done if [[ -f "$transaction_dir/state-present" ]]; then while IFS= read -r name; do [[ " install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf " == *" $name "* && -f "$transaction_dir/state/$name" && ! -L "$transaction_dir/state/$name" ]] || block 'transaction state manifest is unsafe' - install -m 0600 "$transaction_dir/state/$name" "$state_root/$name" + install -m 0600 "$transaction_dir/state/$name" "$state_root/$name" || return done <"$transaction_dir/state-present" fi - rm -f -- "$current" "$install_root/.current.new" "$state_root/.install-state.new" "$active_policy.new" "$state_file.new" + rm -f -- "$current" "$install_root/.current.new" "$state_root/.install-state.new" "$active_policy.new" "$state_file.new" || return if [[ -f "$transaction_dir/current-target" ]]; then target_value=$(<"$transaction_dir/current-target") [[ "$target_value" =~ ^releases/[0-9a-f]{40}$ ]] || block 'transaction current pointer is unsafe' - ln -s "$target_value" "$current" + ln -s "$target_value" "$current" || return fi - systemctl daemon-reload >/dev/null 2>&1 || true + systemctl daemon-reload >/dev/null 2>&1 || return if [[ -f "$transaction_dir/timers-enabled" ]]; then while IFS= read -r name; do [[ " ${timer_names[*]} " == *" $name "* ]] || block 'transaction timer manifest is unsafe' - systemctl enable --now "$name" >/dev/null 2>&1 || true + systemctl enable --now "$name" >/dev/null 2>&1 || return done <"$transaction_dir/timers-enabled" fi - rm -rf -- "$transaction_dir" + rm -rf -- "$transaction_dir" || return transaction_dir= } @@ -498,10 +504,10 @@ recover_interrupted_transaction() { finalize_committed_rollback() { local marker=$transaction_dir/application-rollback-committed [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' - publish_deployed_snapshot "$active_policy" "$state_file" - rm -f "$previous_state" "$previous_policy" + publish_deployed_snapshot "$active_policy" "$state_file" || return + rm -f "$previous_state" "$previous_policy" || return transaction_committed=1 - rm -rf -- "$transaction_dir" + rm -rf -- "$transaction_dir" || return transaction_dir= } @@ -578,15 +584,15 @@ load_deployed_snapshot() { publish_deployed_snapshot() { local policy=$1 state=$2 snapshot pointer - secure_directory "$deployed_root" 700 1 - if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then load_deployed_snapshot; fi - snapshot=$(mktemp -d "$deployed_root/.snapshot.XXXXXX") - chmod 0700 "$snapshot" - install -m 0600 "$policy" "$snapshot/policy.conf" - install -m 0600 "$state" "$snapshot/state.json" + secure_directory "$deployed_root" 700 1 || return + if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then load_deployed_snapshot || return; fi + snapshot=$(mktemp -d "$deployed_root/.snapshot.XXXXXX") || return + chmod 0700 "$snapshot" || return + install -m 0600 "$policy" "$snapshot/policy.conf" || return + install -m 0600 "$state" "$snapshot/state.json" || return pointer=$deployed_root/.current.$$ - ln -s "${snapshot##*/}" "$pointer" - mv -Tf "$pointer" "$deployed_current" + ln -s "${snapshot##*/}" "$pointer" || return + mv -Tf "$pointer" "$deployed_current" || return } install_units() { @@ -638,6 +644,11 @@ policy_adapter_operation() { } perform_check() { + local transactions=() + shopt -s nullglob + transactions=("$state_root"/.transaction.*) + shopt -u nullglob + ((${#transactions[@]} == 0)) || block 'interrupted installer transaction requires recovery' if active_deployment; then block 'active deployment prevents a consistent check'; fi converged || block 'installed deployer state is absent or drifted' policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer health check failed' @@ -655,6 +666,8 @@ v=json.load(open(sys.argv[1])); print(v.get('environment',''),v.get('target','') PY ) [[ "$old_environment" == "$environment" && "$old_target" == "$target" ]] || block 'installed environment and target identity cannot change in place' + [[ -e "$deployed_current" || -L "$deployed_current" ]] || block 'deployed rollback snapshot is missing; restore it before convergence' + load_deployed_snapshot if [[ "$mode" == install && -L "$current" ]] && ! state_matches; then block 'install cannot select a new candidate; use --upgrade or --repair'; fi elif [[ "$mode" == upgrade ]]; then block '--upgrade requires an existing installation' @@ -693,7 +706,6 @@ PY policy_adapter_operation "$active_policy" health 'candidate policy' || die 'candidate health check failed after activation' if [[ ! -e "$deployed_current" && ! -L "$deployed_current" ]]; then publish_deployed_snapshot "$active_policy" "$state_file"; fi commit_transaction - rm -f "$drained" health=healthy report CHANGED yes run-check "$(rollback_available)" } @@ -742,6 +754,8 @@ perform_resume() { if active_deployment; then block 'active deployment prevents resume'; fi if [[ ! -e "$drained" && ! -L "$drained" ]]; then report NO_CHANGE no ready-to-deploy "$(rollback_available)"; return; fi secure_file "$drained" 'drain marker' + converged || block 'installed deployer state is absent or drifted; repair before resume' + policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer health check failed; repair before resume' rm -f "$drained" report CHANGED yes ready-to-deploy "$(rollback_available)" } @@ -771,7 +785,7 @@ perform_uninstall() { } validate_config -if [[ "$mode" == drain || "$mode" == resume || "$mode" == uninstall || "$mode" == rollback ]]; then +if [[ "$mode" == drain || "$mode" == uninstall || "$mode" == rollback ]]; then require_maintenance_host else validate_checkout diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index e8f110f..b4908fd 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -51,6 +51,7 @@ set -u root=${CI_FLEET_DEPLOYER_ROOT:-} log=${FAKE_SYSTEMCTL_LOG:-/dev/null} printf '%s\n' "$*" >>"$log" +[[ -z ${FAKE_SYSTEMCTL_FAIL_COMMAND:-} || ${1:-} != "$FAKE_SYSTEMCTL_FAIL_COMMAND" ]] || exit 1 case "${1:-}" in is-system-running) [[ -z "${FAKE_SYSTEMD_FAIL:-}" ]] || exit 1 @@ -349,6 +350,7 @@ printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$interrupted printf '%s\n' ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer >"$interrupted/timers-enabled" printf 'interrupted\n' >>"$root/var/lib/ci-fleet-deployer/install-state.json" printf 'interrupted\n' >>"$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'interrupted installer transaction requires recovery' "$installer" --check --config "$config" >/dev/null expect_success "$installer" --repair --config "$config" >/dev/null [[ ! -e "$interrupted" ]] || fail 'interrupted transaction was not recovered' expect_success "$installer" --check --config "$config" >/dev/null @@ -434,11 +436,25 @@ approval_error=$(expect_failure 'malformed approval evidence line' "$installer" [[ "$approval_error" != *CANARY_SECRET_VALUE_DO_NOT_PRINT* ]] || fail 'secret content leaked beside evidence failure' write_evidence +printf '# force-transaction\n' >>"$root/etc/systemd/system/ci-fleet-deployer.service" +health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) +export FAKE_ADAPTER_FAIL_HEALTH_AFTER=$((health_calls_before + 1)) FAKE_SYSTEMCTL_FAIL_COMMAND=disable +expect_failure 'candidate health check failed after activation' "$installer" --repair --config "$config" >/dev/null +unset FAKE_ADAPTER_FAIL_HEALTH_AFTER FAKE_SYSTEMCTL_FAIL_COMMAND +compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null || fail 'failed restoration deleted its recovery transaction' +expect_success "$installer" --repair --config "$config" >/dev/null +if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'retry did not recover the retained transaction'; fi + old_state=$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json") new_image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' image=$new_image write_evidence write_config +deployed_current=$root/var/lib/ci-fleet-deployer/deployed/current +deployed_target=$(readlink "$deployed_current") +rm "$deployed_current" +expect_failure 'deployed rollback snapshot is missing; restore it before convergence' "$installer" --upgrade --config "$config" >/dev/null +ln -s "$deployed_target" "$deployed_current" export FAKE_ADAPTER_SLEEP_OPERATION=validate CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS=1 expect_failure 'candidate adapter validation failed' "$installer" --upgrade --config "$config" >/dev/null unset FAKE_ADAPTER_SLEEP_OPERATION CI_FLEET_DEPLOYER_TEST_TIMEOUT_SECONDS @@ -502,6 +518,11 @@ rm "$active" drain=$(expect_success "$installer" --drain --config "$config") grep -Fq 'result=CHANGED' <<<"$drain" || fail 'drain marker was not created' [[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'drain state is absent' +chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'installed deployer state is absent or drifted; repair before resume' "$installer" --resume --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'failed resume removed drain state' +expect_success "$installer" --repair --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'repair implicitly resumed the deployer' resume=$(expect_success "$installer" --resume --config "$config") grep -Fq 'result=CHANGED' <<<"$resume" || fail 'resume did not clear drain state' [[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'resume retained drain state' @@ -546,6 +567,12 @@ request=$root/var/lib/ci-fleet-deployer/request.conf cp "$approval" "$request" chmod 0600 "$request" export CI_FLEET_DEPLOYER_CONFIG=$config CI_FLEET_DEPLOYER_REQUEST=$request +expect_failure 'usage: deployer-runtime.sh health|cleanup|deploy|drain' "$runtime" rollback >/dev/null +mv "$adapter" "$adapter.saved" +expect_success "$runtime" drain >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'runtime drain required a healthy adapter' +rm "$root/var/lib/ci-fleet-deployer/drained" +mv "$adapter.saved" "$adapter" expect_success "$runtime" health >/dev/null expect_success "$runtime" cleanup >/dev/null mkdir "$root/var/lib/ci-fleet-deployer/.transaction.interrupted" @@ -562,10 +589,13 @@ rm "$root/var/lib/ci-fleet-deployer/active-operation" printf 'runner\n' >"$root/etc/systemd/system/actions.runner.late-added.service" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) cleanup_calls_before=$(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) +health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" deploy >/dev/null expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" cleanup >/dev/null +expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" health >/dev/null [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran after role isolation drift' [[ $(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) == "$cleanup_calls_before" ]] || fail 'cleanup ran after role isolation drift' +[[ $(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) == "$health_calls_before" ]] || fail 'health adapter ran after role isolation drift' rm "$root/etc/systemd/system/actions.runner.late-added.service" python3 - "$approval" "$request" <<'PY' from pathlib import Path From 5ddea4d9535685317d0a08b0840eb14cbf266e44 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 01:26:27 -0500 Subject: [PATCH 09/90] fix: preserve validated deployer boundaries --- scripts/deployer-runtime.sh | 56 +++++++++++++++++++--------- scripts/install-deployer.sh | 23 +++++++++--- scripts/test-install-deployer.sh | 63 ++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 23 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index b8b2008..d5e5ba3 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -91,6 +91,18 @@ parse_file() { done <"$path" } +validate_credential() { + [[ ${cfg[CREDENTIAL_SCOPE]} == "${cfg[ENVIRONMENT]}" ]] || die 'credential scope does not match the deployment environment' + case ${cfg[CREDENTIAL_PROVIDER]} in + file) + inside "${cfg[CREDENTIAL_REF]}" "$(root_path /etc/ci-fleet-deployer/credentials)" || die 'credential reference is outside the protected credential directory' + secure_file "${cfg[CREDENTIAL_REF]}" 'credential file' + ;; + external) [[ ${cfg[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die 'external credential reference is malformed' ;; + *) die 'unsupported credential provider' ;; + esac +} + secure_directory "$state_root" 'deployer state directory' secure_directory "$lock_dir" 'deployer lock directory' exec 9<"$lock_dir" @@ -116,6 +128,7 @@ for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 C [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 [[ $(sha256sum "${cfg[ADAPTER_PATH]}" | cut -d' ' -f1) == "${cfg[ADAPTER_SHA256]}" ]] || die 'application adapter digest mismatch' +validate_credential secure_directory "$log_root" 'deployer log directory' @@ -193,15 +206,6 @@ case "$operation" in [[ -v "production[$key]" && ${req[$key]} == "${production[$key]}" ]] || die "deployment request does not match production authorization $key" done fi - [[ ${cfg[CREDENTIAL_SCOPE]} == "${cfg[ENVIRONMENT]}" ]] || die 'credential scope does not match the deployment environment' - case ${cfg[CREDENTIAL_PROVIDER]} in - file) - inside "${cfg[CREDENTIAL_REF]}" "$(root_path /etc/ci-fleet-deployer/credentials)" || die 'credential reference is outside the protected credential directory' - secure_file "${cfg[CREDENTIAL_REF]}" 'credential file' - ;; - external) [[ ${cfg[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die 'external credential reference is malformed' ;; - *) die 'unsupported credential provider' ;; - esac if [[ -e "$consumed_root" || -L "$consumed_root" ]]; then secure_directory "$consumed_root" 'consumed request directory'; else install -d -m 0700 "$consumed_root"; fi request_id=$(for key in $request_keys; do printf '%s=%s\0' "$key" "${req[$key]}"; done | sha256sum | cut -d' ' -f1) consumed_marker=$consumed_root/$request_id @@ -210,26 +214,42 @@ case "$operation" in exec 8>>"$audit_log" secure_file "$install_state" 'deployer install state' if [[ -e "$deployed_root" || -L "$deployed_root" ]]; then secure_directory "$deployed_root" 'deployed snapshot directory'; else install -d -m 0700 "$deployed_root"; fi - if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then - [[ -L "$deployed_current" ]] || die 'deployed snapshot pointer is absent or unsafe' - deployed_snapshot=$(readlink -f "$deployed_current") - inside "$deployed_snapshot" "$deployed_root" || die 'deployed snapshot pointer escapes managed state' - secure_directory "$deployed_snapshot" 'deployed snapshot' - secure_file "$deployed_snapshot/policy.conf" 'deployed rollback policy' - secure_file "$deployed_snapshot/state.json" 'deployed rollback state' - fi + [[ -e "$deployed_current" || -L "$deployed_current" ]] || die 'deployed rollback snapshot is missing' + [[ -L "$deployed_current" ]] || die 'deployed snapshot pointer is absent or unsafe' + deployed_snapshot=$(readlink -f "$deployed_current") + inside "$deployed_snapshot" "$deployed_root" || die 'deployed snapshot pointer escapes managed state' + secure_directory "$deployed_snapshot" 'deployed snapshot' + secure_file "$deployed_snapshot/policy.conf" 'deployed rollback policy' + secure_file "$deployed_snapshot/state.json" 'deployed rollback state' snapshot=$(mktemp -d "$deployed_root/.snapshot.XXXXXX") chmod 0700 "$snapshot" install -m 0600 "$config" "$snapshot/policy.conf" install -m 0600 "$install_state" "$snapshot/state.json" + snapshot_policy_sha=$(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) + snapshot_state_sha=$(sha256sum "$snapshot/state.json" | cut -d' ' -f1) install -m 0600 /dev/null "$consumed_marker" umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" mv -Tf "$temporary" "$active" - trap 'rm -f "$active"; [[ -z ${snapshot:-} || ! -d $snapshot ]] || rm -rf -- "$snapshot"' EXIT INT TERM + trap 'rm -f "$active"' EXIT INT TERM + set +e systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ --why='approved deployment is active' -- "${cfg[ADAPTER_PATH]}" deploy + adapter_status=$? + set -e + if ((adapter_status != 0)); then + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=failed status=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" "$adapter_status" >&8 + die 'deployment adapter failed after approval consumption' + fi + secure_directory "$deployed_root" 'deployed snapshot directory' + inside "$snapshot" "$deployed_root" || die 'prepared deployed snapshot escaped managed state' + secure_directory "$snapshot" 'prepared deployed snapshot' + secure_file "$snapshot/policy.conf" 'prepared deployed policy' + secure_file "$snapshot/state.json" 'prepared deployed state' + [[ $(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) == "$snapshot_policy_sha" && $(sha256sum "$snapshot/state.json" | cut -d' ' -f1) == "$snapshot_state_sha" ]] || die 'prepared deployed snapshot changed during deployment' pointer=$deployed_root/.current.$$ ln -s "${snapshot##*/}" "$pointer" mv -Tf "$pointer" "$deployed_current" diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 9cad76d..f8570d8 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -6,6 +6,7 @@ export PYTHONDONTWRITEBYTECODE=1 repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) mode= config= +validated_config= error_reported=0 root=${CI_FLEET_DEPLOYER_ROOT:-} testing=${CI_FLEET_DEPLOYER_TESTING:-0} @@ -32,6 +33,7 @@ on_exit() { fi fi [[ -z ${staging_path:-} ]] || rm -rf -- "$staging_path" + [[ -z ${validated_config:-} ]] || rm -f -- "$validated_config" if ((status != 0 && error_reported == 0)); then report FAILED no inspect-and-retry "$(rollback_available)" >&2; fi return "$status" } @@ -153,6 +155,9 @@ validate_config() { inside "$config" "$etc_root" || block "configuration path must be inside $etc_root" secure_directory "$etc_root" 700 0 || block 'configuration directory is missing' secure_file "$config" 'configuration file' + validated_config=$(mktemp) + install -m 0600 "$config" "$validated_config" + config=$validated_config parse_file "$config" cfg configuration "$config_keys" local key candidate_environment candidate_target candidate_core candidate_artifact for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID; do @@ -282,7 +287,7 @@ require_host() { require_maintenance_host() { local command - for command in bash awk cut stat sha256sum readlink realpath install cp rm mkdir mktemp chmod ln mv flock kill timeout env python3 systemctl systemd-analyze date; do + for command in bash awk cut stat sha256sum readlink realpath install cp rm mkdir mktemp chmod ln mv flock kill timeout env python3 docker systemctl systemd-analyze date; do command -v "$command" >/dev/null || block "$command is required for maintenance" done [[ -d "$systemd_root" && ! -L "$systemd_root" ]] || block 'systemd unit directory is unavailable' @@ -345,11 +350,11 @@ release_complete() { state_matches() { [[ -f "$state_file" && ! -L "$state_file" && $(stat -c '%u:%a' "$state_file") == "$expected_uid:600" ]] || return 1 [[ -f "$active_policy" && ! -L "$active_policy" && $(stat -c '%u:%a' "$active_policy") == "$expected_uid:600" && $(cmp -s "$config" "$active_policy"; echo $?) == 0 ]] || return 1 - printf '%s\n' "$core_ref" "$environment" "$target" "${cfg[DEPLOYER_IDENTITY]}" "${cfg[SOURCE_COMMIT]}" "$artifact" "${approval[APPROVAL_ID]}" "${approval[POLICY_IDENTITY]}" "${cfg[APPROVAL_PROVIDER]}" "${checkpoint[CHECKPOINT_ID]}" | python3 -c ' + printf '%s\n' "$core_ref" "$environment" "$target" "${cfg[DEPLOYER_IDENTITY]}" "${cfg[SOURCE_COMMIT]}" "$artifact" "${approval[APPROVAL_ID]}" "${approval[APPROVAL_IDENTITY]}" "${approval[POLICY_IDENTITY]}" "${cfg[APPROVAL_PROVIDER]}" "${checkpoint[CHECKPOINT_ID]}" | python3 -c ' import json, sys try: value=json.load(open(sys.argv[1], encoding="utf-8")) except (OSError, ValueError): raise SystemExit(1) -keys=("core_ref","environment","target","deployer_identity","source_commit","artifact","approval_id","policy_identity","approval_provider","checkpoint_id") +keys=("core_ref","environment","target","deployer_identity","source_commit","artifact","approval_id","approval_identity","policy_identity","approval_provider","checkpoint_id") expected=[line.rstrip("\n") for line in sys.stdin] raise SystemExit(0 if all(value.get(k)==v for k,v in zip(keys,expected)) else 1) ' "$state_file" @@ -385,6 +390,7 @@ acquire_lock() { secure_directory "$lock_root" 700 1 exec 9<"$lock_root" flock -n 9 || block 'another deployer installer operation is running' + [[ ! -L "$active_operation" ]] || block 'active operation marker is an unsafe symlink' if [[ -e "$active_operation" ]] && ! active_deployment; then [[ ! -L "$active_operation" && -f "$active_operation" && $(stat -c '%u:%a' "$active_operation") == "$expected_uid:600" ]] || block 'stale operation state is unsafe' rm -f "$active_operation" @@ -711,10 +717,16 @@ PY } perform_rollback() { + local -A rollback_policy=() if ((recovered_rollback)); then health=healthy; report CHANGED yes restore-host-policy-evidence-then-check no; return; fi active_deployment && block 'active deployment prevents rollback' [[ -f "$previous_state" && -f "$previous_policy" ]] || block 'no last-known-good release is available' secure_directory "$state_root" 700 0 + secure_file "$previous_policy" 'last-known-good policy' + parse_file "$previous_policy" rollback_policy 'last-known-good policy' "$config_keys" + [[ -v 'rollback_policy[DEPLOYER_IDENTITY]' ]] || block 'last-known-good policy is missing deployer identity' + cfg[DEPLOYER_IDENTITY]=${rollback_policy[DEPLOYER_IDENTITY]} + reject_mixed_role begin_transaction install -m 0600 "$previous_state" "$state_file.new" install -m 0600 "$previous_policy" "$active_policy.new" @@ -752,10 +764,11 @@ perform_drain() { perform_resume() { secure_directory "$state_root" 700 0 || block 'deployer state directory is missing' if active_deployment; then block 'active deployment prevents resume'; fi - if [[ ! -e "$drained" && ! -L "$drained" ]]; then report NO_CHANGE no ready-to-deploy "$(rollback_available)"; return; fi - secure_file "$drained" 'drain marker' + local was_drained=0 + if [[ -e "$drained" || -L "$drained" ]]; then secure_file "$drained" 'drain marker'; was_drained=1; fi converged || block 'installed deployer state is absent or drifted; repair before resume' policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer health check failed; repair before resume' + ((was_drained == 1)) || { report NO_CHANGE no ready-to-deploy "$(rollback_available)"; return; } rm -f "$drained" report CHANGED yes ready-to-deploy "$(rollback_available)" } diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index b4908fd..966c3ba 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -109,6 +109,7 @@ cat >"$adapter" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail printf '%s\n' "$1" >>"${FAKE_ADAPTER_LOG:?}" +if [[ -n ${FAKE_ADAPTER_FORBID_CONFIG_PATH:-} && ${CI_FLEET_DEPLOYER_CONFIG:-} == "$FAKE_ADAPTER_FORBID_CONFIG_PATH" ]]; then exit 43; fi if [[ ${FAKE_ADAPTER_SLEEP_OPERATION:-} == "$1" ]]; then sleep 2; fi if [[ "$1" == health && -n ${FAKE_ADAPTER_FAIL_HEALTH_AFTER:-} ]]; then health_calls=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) @@ -126,6 +127,10 @@ if [[ "$1" == deploy && -n ${FAKE_ADAPTER_AUDIT_PATH:-} ]]; then rm -f "$FAKE_ADAPTER_AUDIT_PATH" ln -s "$FAKE_ADAPTER_AUDIT_TARGET" "$FAKE_ADAPTER_AUDIT_PATH" fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT:-} ]]; then + snapshot=$(find "$FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT" -maxdepth 1 -type d -name '.snapshot.*' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-) + [[ -n "$snapshot" ]] && printf 'adapter-mutation\n' >>"$snapshot/policy.conf" +fi EOF chmod 0700 "$adapter" export FAKE_ADAPTER_LOG=$tmp/adapter.log @@ -320,7 +325,9 @@ fresh_uninstall=$(expect_success "$installer" --uninstall --config "$config") [[ "$after" == "$(find "$root" -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'fresh uninstall mutated an unmanaged host' grep -Fq 'result=NO_CHANGE' <<<"$fresh_uninstall" || fail 'fresh uninstall did not report NO_CHANGE' +export FAKE_ADAPTER_FORBID_CONFIG_PATH=$config first=$(expect_success "$installer" --install --config "$config") +unset FAKE_ADAPTER_FORBID_CONFIG_PATH grep -Fq 'REPORT action=install result=CHANGED environment=staging' <<<"$first" || fail 'fresh install report is incomplete' [[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'fresh install lacks atomic current release' [[ $(stat -c %a "$root/var/lib/ci-fleet-deployer/install-state.json") == 600 ]] || fail 'install state mode is not 0600' @@ -436,6 +443,19 @@ approval_error=$(expect_failure 'malformed approval evidence line' "$installer" [[ "$approval_error" != *CANARY_SECRET_VALUE_DO_NOT_PRINT* ]] || fail 'secret content leaked beside evidence failure' write_evidence +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_IDENTITY=example-reviewer', 'APPROVAL_IDENTITY=alternate-reviewer')) +PY +expect_success "$installer" --repair --config "$config" >/dev/null +python3 - "$root/var/lib/ci-fleet-deployer/install-state.json" <<'PY' || fail 'approval identity drift was not recorded' +import json, sys +assert json.load(open(sys.argv[1]))['approval_identity'] == 'alternate-reviewer' +PY +write_evidence +expect_success "$installer" --repair --config "$config" >/dev/null + printf '# force-transaction\n' >>"$root/etc/systemd/system/ci-fleet-deployer.service" health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) export FAKE_ADAPTER_FAIL_HEALTH_AFTER=$((health_calls_before + 1)) FAKE_SYSTEMCTL_FAIL_COMMAND=disable @@ -490,6 +510,10 @@ rollback_calls_before=$(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) FAKE_SYSTEMD_VERIFY_EXIT=1 expect_failure 'systemd unit verification failed' "$installer" --rollback --config "$config" >/dev/null [[ $(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) == "$rollback_calls_before" ]] || fail 'application rollback ran before core rollback staging was proven' grep -Fq 'sha256:bbbbbbbb' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'failed rollback did not preserve current core state' +printf 'runner\n' >"$root/etc/systemd/system/actions.runner.rollback-drift.service" +expect_failure 'ordinary GitHub Actions runner service is present' "$installer" --rollback --config "$config" >/dev/null +[[ $(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) == "$rollback_calls_before" ]] || fail 'rollback adapter ran after role isolation drift' +rm "$root/etc/systemd/system/actions.runner.rollback-drift.service" mv "$approval" "$approval.rollback-saved" mv "$checkpoint" "$checkpoint.rollback-saved" export FAKE_CURL_EXIT=1 @@ -510,6 +534,10 @@ write_evidence write_config staging example-staging active=$root/var/lib/ci-fleet-deployer/active-operation +printf 'unrelated-active\n' >"$tmp/unrelated-active" +ln -s "$tmp/unrelated-active" "$active" +expect_failure 'active operation marker is an unsafe symlink' "$installer" --repair --config "$config" >/dev/null +rm "$active" printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$active" chmod 0600 "$active" expect_failure 'active deployment prevents this operation' "$installer" --upgrade --config "$config" >/dev/null @@ -526,6 +554,9 @@ expect_success "$installer" --repair --config "$config" >/dev/null resume=$(expect_success "$installer" --resume --config "$config") grep -Fq 'result=CHANGED' <<<"$resume" || fail 'resume did not clear drain state' [[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'resume retained drain state' +chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'installed deployer state is absent or drifted; repair before resume' "$installer" --resume --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null repeat_resume=$(expect_success "$installer" --resume --config "$config") grep -Fq 'result=NO_CHANGE' <<<"$repeat_resume" || fail 'repeated resume was not idempotent' expect_success "$installer" --drain --config "$config" >/dev/null @@ -575,6 +606,11 @@ rm "$root/var/lib/ci-fleet-deployer/drained" mv "$adapter.saved" "$adapter" expect_success "$runtime" health >/dev/null expect_success "$runtime" cleanup >/dev/null +chmod 0644 "$credential" +cleanup_calls_before=$(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) +expect_failure 'credential file has unsafe owner or mode' "$runtime" cleanup >/dev/null +[[ $(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) == "$cleanup_calls_before" ]] || fail 'cleanup ran with unsafe file credentials' +chmod 0600 "$credential" mkdir "$root/var/lib/ci-fleet-deployer/.transaction.interrupted" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'interrupted installer transaction requires recovery' "$runtime" deploy >/dev/null @@ -623,12 +659,17 @@ deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'GitHub capability evidence must be a regular file' "$runtime" deploy >/dev/null [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran without GitHub capability evidence' mv "$capability.saved" "$capability" +deployed_target=$(readlink "$deployed_current") +rm "$deployed_current" +expect_failure 'deployed rollback snapshot is missing' "$runtime" deploy >/dev/null +ln -s "$deployed_target" "$deployed_current" chmod 0644 "$credential" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'credential file has unsafe owner or mode' "$runtime" deploy >/dev/null [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran with unsafe file credentials' chmod 0600 "$credential" printf 'unrelated-audit\n' >"$tmp/unrelated-audit" +rm -f "$root/var/log/ci-fleet-deployer/audit.log" ln -s "$tmp/unrelated-audit" "$root/var/log/ci-fleet-deployer/audit.log" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null @@ -637,6 +678,28 @@ rm "$root/var/log/ci-fleet-deployer/audit.log" python3 - "$approval" <<'PY' from pathlib import Path import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=failed-adapter-attempt')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_FAIL=$tmp/fail-adapter +printf 'deploy\n' >"$FAKE_ADAPTER_FAIL" +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-adapter" +grep -Fq 'approval=failed-adapter-attempt policy=example-staging-policy-v1 result=failed status=42' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'consumed failed deployment was not audited' +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=snapshot-mutation-attempt')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT=$root/var/lib/ci-fleet-deployer/deployed +expect_failure 'prepared deployed snapshot changed during deployment' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=audit-replacement-attempt')) PY cp "$approval" "$request"; chmod 0600 "$request" From deb781caefb1fdc4e16de1b447c3c29342e564dc Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 02:10:36 -0500 Subject: [PATCH 10/90] fix: pin deployer transaction inputs atomically --- scripts/deployer-runtime.sh | 29 ++++++++++++++++------ scripts/install-deployer.sh | 42 ++++++++++++++++++++++++++------ scripts/test-install-deployer.sh | 27 ++++++++++++++++++++ 3 files changed, 82 insertions(+), 16 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index d5e5ba3..b52aefb 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -16,6 +16,8 @@ state_root=$(root_path /var/lib/ci-fleet-deployer) log_root=$(root_path /var/log/ci-fleet-deployer) lock_dir=$(root_path /var/lock/ci-fleet-deployer) evidence_dir=$(root_path /etc/ci-fleet-deployer/evidence) +deployer_etc=$(root_path /etc/ci-fleet-deployer) +credential_dir=$deployer_etc/credentials active=$state_root/active-operation drained=$state_root/drained last_request=$state_root/last-request.conf @@ -32,12 +34,14 @@ expected_uid=0 secure_file() { local path=$1 description=$2 mode=${3:-600} [[ ! -L "$path" && -f "$path" ]] || die "$description must be a regular file, not a symlink" + [[ "$path" == "$(realpath -m -- "$path")" ]] || die "$description path contains a symlink or non-canonical component" [[ $(realpath -e -- "$path") == $(realpath -m -- "$path") ]] || die "$description path contains a symlink" [[ $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || die "$description has unsafe owner or mode" } secure_directory() { local path=$1 description=$2 [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:700" ]] || die "$description has unsafe owner, mode, or type" + [[ "$path" == "$(realpath -m -- "$path")" ]] || die "$description path contains a symlink or non-canonical component" } reject_mixed_role() { local unit runner_unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" @@ -92,10 +96,12 @@ parse_file() { } validate_credential() { + secure_directory "$deployer_etc" 'deployer configuration directory' + secure_directory "$credential_dir" 'credential directory' [[ ${cfg[CREDENTIAL_SCOPE]} == "${cfg[ENVIRONMENT]}" ]] || die 'credential scope does not match the deployment environment' case ${cfg[CREDENTIAL_PROVIDER]} in file) - inside "${cfg[CREDENTIAL_REF]}" "$(root_path /etc/ci-fleet-deployer/credentials)" || die 'credential reference is outside the protected credential directory' + inside "${cfg[CREDENTIAL_REF]}" "$credential_dir" || die 'credential reference is outside the protected credential directory' secure_file "${cfg[CREDENTIAL_REF]}" 'credential file' ;; external) [[ ${cfg[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die 'external credential reference is malformed' ;; @@ -127,7 +133,9 @@ for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 C [[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 -[[ $(sha256sum "${cfg[ADAPTER_PATH]}" | cut -d' ' -f1) == "${cfg[ADAPTER_SHA256]}" ]] || die 'application adapter digest mismatch' +exec 7<"${cfg[ADAPTER_PATH]}" +[[ $(sha256sum /proc/$$/fd/7 | cut -d' ' -f1) == "${cfg[ADAPTER_SHA256]}" ]] || die 'application adapter digest mismatch' +adapter_path=/proc/$$/fd/7 validate_credential secure_directory "$log_root" 'deployer log directory' @@ -135,20 +143,23 @@ secure_directory "$log_root" 'deployer log directory' case "$operation" in health) reject_mixed_role - "${cfg[ADAPTER_PATH]}" "$operation" + "$adapter_path" "$operation" ;; cleanup) not_drained reject_mixed_role - "${cfg[ADAPTER_PATH]}" cleanup + "$adapter_path" cleanup ;; deploy) not_drained [[ ! -e "$active" && ! -L "$active" ]] || die 'active operation marker requires recovery' reject_mixed_role secure_file "$request" 'deployment request' + request_snapshot=$(mktemp "$state_root/.request.XXXXXX") + install -m 0600 "$request" "$request_snapshot" + trap 'rm -f "${request_snapshot:-}"' EXIT INT TERM request_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' - parse_file "$request" req request "$request_keys" + parse_file "$request_snapshot" req request "$request_keys" for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT; do [[ -v "req[$key]" ]] || die "deployment request is missing $key" done @@ -232,10 +243,10 @@ case "$operation" in temporary=$(mktemp "$state_root/.active.XXXXXX") printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" mv -Tf "$temporary" "$active" - trap 'rm -f "$active"' EXIT INT TERM + trap 'rm -f "$active" "${request_snapshot:-}"' EXIT INT TERM set +e systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ - --why='approved deployment is active' -- "${cfg[ADAPTER_PATH]}" deploy + --why='approved deployment is active' -- env CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? set -e if ((adapter_status != 0)); then @@ -254,7 +265,9 @@ case "$operation" in ln -s "${snapshot##*/}" "$pointer" mv -Tf "$pointer" "$deployed_current" snapshot= - mv -Tf "$request" "$last_request" + if [[ -f "$request" && ! -L "$request" ]] && cmp -s "$request_snapshot" "$request"; then rm -f "$request"; fi + mv -Tf "$request_snapshot" "$last_request" + request_snapshot= secure_file "$audit_log" 'deployer audit log' [[ $(stat -Lc '%d:%i' /proc/self/fd/8) == $(stat -c '%d:%i' "$audit_log") ]] || die 'deployer audit log changed during deployment' printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=success\n' \ diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index f8570d8..6e956b8 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -19,11 +19,15 @@ artifact=unknown health=unknown staging_path= transaction_dir= +transaction_preparing=0 transaction_committed=0 recovered_rollback=0 on_exit() { local status=$? recovery_status=0 - if [[ -n ${transaction_dir:-} && ${transaction_committed:-0} != 1 ]]; then + if [[ -n ${transaction_dir:-} && ${transaction_preparing:-0} == 1 ]]; then + rm -rf -- "$transaction_dir" + transaction_dir= + elif [[ -n ${transaction_dir:-} && ${transaction_committed:-0} != 1 ]]; then set +e if [[ -f "$transaction_dir/application-rollback-committed" ]]; then finalize_committed_rollback; recovery_status=$?; else restore_transaction; recovery_status=$?; fi set -e @@ -120,12 +124,14 @@ valid_utc() { secure_file() { local path=$1 description=$2 mode=${3:-600} [[ ! -L "$path" && -f "$path" ]] || block "$description must be a regular file, not a symlink" + [[ "$path" == "$(realpath -m -- "$path")" ]] || block "$description path contains a symlink or non-canonical component" [[ $(realpath -e -- "$path") == $(realpath -m -- "$path") ]] || block "$description path contains a symlink" [[ $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || block "$description must be owned by root with mode 0$mode" } secure_directory() { local path=$1 mode=$2 create=${3:-0} [[ ! -L "$path" ]] || die "unsafe symlinked managed directory: $path" + [[ "$path" == "$(realpath -m -- "$path")" ]] || die "managed directory path contains a symlink or non-canonical component: $path" if [[ ! -e "$path" ]]; then [[ "$create" == 1 ]] || return 1 install -d -m "$mode" "$path" @@ -287,7 +293,7 @@ require_host() { require_maintenance_host() { local command - for command in bash awk cut stat sha256sum readlink realpath install cp rm mkdir mktemp chmod ln mv flock kill timeout env python3 docker systemctl systemd-analyze date; do + for command in bash awk cut stat sha256sum readlink realpath install cp rm mkdir mktemp chmod ln mv flock kill timeout env python3 systemctl systemd-analyze date; do command -v "$command" >/dev/null || block "$command is required for maintenance" done [[ -d "$systemd_root" && ! -L "$systemd_root" ]] || block 'systemd unit directory is unavailable' @@ -403,6 +409,12 @@ acquire_lock() { done shopt -u nullglob fi + shopt -s nullglob + for path in "$state_root"/.transaction-preparing.*; do + [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:700" ]] || block 'incomplete transaction preparation is unsafe' + rm -rf -- "$path" + done + shopt -u nullglob recover_interrupted_transaction } @@ -413,7 +425,7 @@ acquire_check_lock() { } begin_transaction() { - local name path current_target + local name path current_target transaction_name transaction_ready for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do path=$state_root/$name [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:600" ]] || block 'managed transaction state has an unsafe type, owner, or mode' @@ -427,7 +439,8 @@ begin_transaction() { path=$systemd_root/$name [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c %u "$path") == "$expected_uid" ]] || block 'managed systemd unit has an unsafe owner or type' done - transaction_dir=$(mktemp -d "$state_root/.transaction.XXXXXX") + transaction_dir=$(mktemp -d "$state_root/.transaction-preparing.XXXXXX") + transaction_preparing=1 chmod 0700 "$transaction_dir" install -d -m 0700 "$transaction_dir/units" "$transaction_dir/state" for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do @@ -448,6 +461,11 @@ begin_transaction() { for name in "${timer_names[@]}"; do if systemctl is-enabled "$name" >/dev/null 2>&1; then printf '%s\n' "$name" >>"$transaction_dir/timers-enabled"; fi done + transaction_name=${transaction_dir##*/} + transaction_ready=$state_root/.transaction.${transaction_name#.transaction-preparing.} + mv "$transaction_dir" "$transaction_ready" + transaction_dir=$transaction_ready + transaction_preparing=0 } restore_transaction() { @@ -637,6 +655,14 @@ run_adapter() { fi } +run_verified_adapter() { + local policy=$1 path=$2 digest=$3 operation_name=$4 marker=${5:-} + secure_file "$path" "$operation_name adapter" 700 + exec 7<"$path" + [[ $(sha256sum /proc/$$/fd/7 | cut -d' ' -f1) == "$digest" ]] || die "$operation_name adapter digest mismatch" + run_adapter "$policy" "/proc/$$/fd/7" "$operation_name" "$marker" +} + policy_adapter_operation() { local policy=$1 operation_name=$2 description=$3 marker=${4:-} key local -A policy_cfg=() @@ -644,9 +670,7 @@ policy_adapter_operation() { parse_file "$policy" policy_cfg "$description" "$config_keys" for key in ADAPTER_PATH ADAPTER_SHA256; do [[ -v "policy_cfg[$key]" ]] || die "$description is missing $key"; done [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" - secure_file "${policy_cfg[ADAPTER_PATH]}" "$description adapter" 700 - [[ $(sha256sum "${policy_cfg[ADAPTER_PATH]}" | cut -d' ' -f1) == "${policy_cfg[ADAPTER_SHA256]}" ]] || die "$description adapter digest mismatch" - run_adapter "$policy" "${policy_cfg[ADAPTER_PATH]}" "$operation_name" "$marker" >/dev/null 2>&1 + run_verified_adapter "$policy" "${policy_cfg[ADAPTER_PATH]}" "${policy_cfg[ADAPTER_SHA256]}" "$operation_name" "$marker" >/dev/null 2>&1 } perform_check() { @@ -690,7 +714,7 @@ PY release_complete "$old_release" || block 'active deployer release is incomplete' policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement' fi - run_adapter "$config" "${cfg[ADAPTER_PATH]}" validate >/dev/null 2>&1 || die 'candidate adapter validation failed' + run_verified_adapter "$config" "${cfg[ADAPTER_PATH]}" "${cfg[ADAPTER_SHA256]}" validate >/dev/null 2>&1 || die 'candidate adapter validation failed' install_release secure_directory "$state_root" 700 1 secure_directory "$log_root" 700 1 @@ -726,6 +750,7 @@ perform_rollback() { parse_file "$previous_policy" rollback_policy 'last-known-good policy' "$config_keys" [[ -v 'rollback_policy[DEPLOYER_IDENTITY]' ]] || block 'last-known-good policy is missing deployer identity' cfg[DEPLOYER_IDENTITY]=${rollback_policy[DEPLOYER_IDENTITY]} + command -v docker >/dev/null || block 'docker is required for rollback isolation validation' reject_mixed_role begin_transaction install -m 0600 "$previous_state" "$state_file.new" @@ -768,6 +793,7 @@ perform_resume() { if [[ -e "$drained" || -L "$drained" ]]; then secure_file "$drained" 'drain marker'; was_drained=1; fi converged || block 'installed deployer state is absent or drifted; repair before resume' policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer health check failed; repair before resume' + health=healthy ((was_drained == 1)) || { report NO_CHANGE no ready-to-deploy "$(rollback_available)"; return; } rm -f "$drained" report CHANGED yes ready-to-deploy "$(rollback_available)" diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 966c3ba..bdae740 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -110,6 +110,12 @@ cat >"$adapter" <<'EOF' set -Eeuo pipefail printf '%s\n' "$1" >>"${FAKE_ADAPTER_LOG:?}" if [[ -n ${FAKE_ADAPTER_FORBID_CONFIG_PATH:-} && ${CI_FLEET_DEPLOYER_CONFIG:-} == "$FAKE_ADAPTER_FORBID_CONFIG_PATH" ]]; then exit 43; fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_FORBID_REQUEST_PATH:-} && ${CI_FLEET_DEPLOYER_REQUEST:-} == "$FAKE_ADAPTER_FORBID_REQUEST_PATH" ]]; then exit 44; fi +if [[ -n ${FAKE_ADAPTER_REPLACE_PATH:-} && -e $FAKE_ADAPTER_REPLACE_PATH ]]; then + mv "$FAKE_ADAPTER_REPLACE_PATH" "$FAKE_ADAPTER_REPLACE_PATH.saved" + printf '#!/usr/bin/env bash\nexit 45\n' >"$FAKE_ADAPTER_REPLACE_PATH" + chmod 0700 "$FAKE_ADAPTER_REPLACE_PATH" +fi if [[ ${FAKE_ADAPTER_SLEEP_OPERATION:-} == "$1" ]]; then sleep 2; fi if [[ "$1" == health && -n ${FAKE_ADAPTER_FAIL_HEALTH_AFTER:-} ]]; then health_calls=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) @@ -344,6 +350,12 @@ expect_failure 'installed deployer state is absent or drifted' "$installer" --ch expect_success "$installer" --repair --config "$config" >/dev/null [[ $(stat -c %a "$root/etc/systemd/system/ci-fleet-deployer.service") == 644 ]] || fail 'repair did not restore unit mode 0644' +preparing=$root/var/lib/ci-fleet-deployer/.transaction-preparing.interrupted +mkdir -m 0700 "$preparing" +state_before_preparing=$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json") +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$preparing" && "$state_before_preparing" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'incomplete transaction preparation was treated as recovery state' + interrupted=$root/var/lib/ci-fleet-deployer/.transaction.interrupted mkdir -m 0700 "$interrupted" "$interrupted/units" "$interrupted/state" for name in install-state.json active-policy.conf; do @@ -546,6 +558,9 @@ rm "$active" drain=$(expect_success "$installer" --drain --config "$config") grep -Fq 'result=CHANGED' <<<"$drain" || fail 'drain marker was not created' [[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'drain state is absent' +mv "$fake_bin/docker" "$fake_bin/docker.unavailable" +expect_success "$installer" --drain --config "$config" >/dev/null +mv "$fake_bin/docker.unavailable" "$fake_bin/docker" chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_failure 'installed deployer state is absent or drifted; repair before resume' "$installer" --resume --config "$config" >/dev/null [[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'failed resume removed drain state' @@ -553,6 +568,7 @@ expect_success "$installer" --repair --config "$config" >/dev/null [[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'repair implicitly resumed the deployer' resume=$(expect_success "$installer" --resume --config "$config") grep -Fq 'result=CHANGED' <<<"$resume" || fail 'resume did not clear drain state' +grep -Fq 'health=healthy' <<<"$resume" || fail 'resume report omitted verified health' [[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'resume retained drain state' chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_failure 'installed deployer state is absent or drifted; repair before resume' "$installer" --resume --config "$config" >/dev/null @@ -604,6 +620,10 @@ expect_success "$runtime" drain >/dev/null [[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'runtime drain required a healthy adapter' rm "$root/var/lib/ci-fleet-deployer/drained" mv "$adapter.saved" "$adapter" +export FAKE_ADAPTER_REPLACE_PATH=$adapter +expect_success "$runtime" health >/dev/null +unset FAKE_ADAPTER_REPLACE_PATH +rm "$adapter"; mv "$adapter.saved" "$adapter" expect_success "$runtime" health >/dev/null expect_success "$runtime" cleanup >/dev/null chmod 0644 "$credential" @@ -611,6 +631,11 @@ cleanup_calls_before=$(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) expect_failure 'credential file has unsafe owner or mode' "$runtime" cleanup >/dev/null [[ $(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) == "$cleanup_calls_before" ]] || fail 'cleanup ran with unsafe file credentials' chmod 0600 "$credential" +mv "$root/etc/ci-fleet-deployer/credentials" "$root/etc/ci-fleet-deployer/credentials.real" +ln -s "$root/etc/ci-fleet-deployer/credentials.real" "$root/etc/ci-fleet-deployer/credentials" +expect_failure 'credential directory has unsafe owner, mode, or type' "$runtime" cleanup >/dev/null +rm "$root/etc/ci-fleet-deployer/credentials" +mv "$root/etc/ci-fleet-deployer/credentials.real" "$root/etc/ci-fleet-deployer/credentials" mkdir "$root/var/lib/ci-fleet-deployer/.transaction.interrupted" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'interrupted installer transaction requires recovery' "$runtime" deploy >/dev/null @@ -710,7 +735,9 @@ unset FAKE_ADAPTER_AUDIT_PATH FAKE_ADAPTER_AUDIT_TARGET rm "$root/var/log/ci-fleet-deployer/audit.log" write_evidence staging example-staging cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_FORBID_REQUEST_PATH=$request expect_success "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_FORBID_REQUEST_PATH [[ ! -e "$request" && -f "$root/var/lib/ci-fleet-deployer/last-request.conf" ]] || fail 'completed request was not consumed atomically' deployed_current=$root/var/lib/ci-fleet-deployer/deployed/current [[ -L "$deployed_current" && -f "$deployed_current/policy.conf" && -f "$deployed_current/state.json" ]] || fail 'deployed policy/state pair was not published through one pointer' From 817c0bff696d4bb497eeeff20633e04f4357e465 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 12:01:55 -0500 Subject: [PATCH 11/90] fix: close deployer recovery and audit review gaps - Permit the conventional /var/lock -> /run/lock symlink layout instead of rejecting the canonical parent path - Write a structured failed audit record through fd 8 for every failure after approval consumption, including post-adapter snapshot/publication failures, with the failing phase recorded - Reject unknown approval provider values instead of silently skipping GitHub capability validation - Validate the retained last-known-good state schema and cross-match it with the retained policy before rollback starts - Allow --repair to restore a missing or drifted owned active policy by skipping only the untrusted old-policy health probe while retaining candidate validation and post-activation health gates - Fix the repair transaction-failure fixture to fail the candidate health call under the new single-health-call sequence and assert the exact call count --- scripts/deployer-runtime.sh | 49 ++++++++++++++++++++++---------- scripts/install-deployer.sh | 20 ++++++++----- scripts/test-install-deployer.sh | 37 ++++++++++++++++++++++-- 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index b52aefb..ac1bf1b 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -29,6 +29,18 @@ audit_log=$log_root/audit.log systemd_root=$(root_path /etc/systemd/system) die() { printf 'ERROR: %s\n' "$*" >&2; exit 2; } +deploy_exit() { + local status=$? + local recorded_status=${adapter_status:-$status} + if [[ ${audit_pending:-0} == 1 ]]; then + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=failed phase=%s status=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" \ + "${audit_phase:-post-consumption}" "$recorded_status" >&8 || true + fi + rm -f "$active" "${request_snapshot:-}" || true + return "$status" +} expected_uid=0 [[ "$testing" != 1 ]] || expected_uid=$(id -u) secure_file() { @@ -41,7 +53,6 @@ secure_file() { secure_directory() { local path=$1 description=$2 [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:700" ]] || die "$description has unsafe owner, mode, or type" - [[ "$path" == "$(realpath -m -- "$path")" ]] || die "$description path contains a symlink or non-canonical component" } reject_mixed_role() { local unit runner_unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" @@ -194,16 +205,20 @@ case "$operation" in for key in $approval_keys; do [[ -v "approved[$key]" && ${req[$key]} == "${approved[$key]}" ]] || die "deployment request does not match protected approval $key" done - if [[ ${cfg[APPROVAL_PROVIDER]} == github-environment ]]; then - [[ -v 'cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]' ]] || die 'GitHub Environment approval is missing capability evidence' - inside "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" "$evidence_dir" || die 'capability evidence is outside the protected evidence directory' - secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' - capability_keys='SCHEMA_VERSION ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' - parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' "$capability_keys" - for key in $capability_keys; do [[ -v "capability[$key]" ]] || die "capability evidence is missing $key"; done - [[ ${capability[SCHEMA_VERSION]} == 1 && ${capability[ENVIRONMENT_PROTECTION]} == verified && ${capability[EXACT_HEAD]} == "${req[SOURCE_COMMIT]}" ]] || die 'GitHub Environment capability evidence is not exact-head verified' - if [[ ! ${capability[CAPABILITY_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${capability[CHECKED_AT]}"; then die 'GitHub Environment capability evidence is malformed'; fi - fi + case ${cfg[APPROVAL_PROVIDER]} in + manual-exact-head) ;; + github-environment) + [[ -v 'cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]' ]] || die 'GitHub Environment approval is missing capability evidence' + inside "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" "$evidence_dir" || die 'capability evidence is outside the protected evidence directory' + secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' + capability_keys='SCHEMA_VERSION ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' + parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' "$capability_keys" + for key in $capability_keys; do [[ -v "capability[$key]" ]] || die "capability evidence is missing $key"; done + [[ ${capability[SCHEMA_VERSION]} == 1 && ${capability[ENVIRONMENT_PROTECTION]} == verified && ${capability[EXACT_HEAD]} == "${req[SOURCE_COMMIT]}" ]] || die 'GitHub Environment capability evidence is not exact-head verified' + if [[ ! ${capability[CAPABILITY_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${capability[CHECKED_AT]}"; then die 'GitHub Environment capability evidence is malformed'; fi + ;; + *) die 'unsupported approval provider' ;; + esac if [[ ${cfg[ENVIRONMENT]} == production ]]; then [[ -v 'cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]' ]] || die 'production policy is missing separate authorization evidence' inside "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" "$evidence_dir" || die 'production authorization evidence is outside the protected evidence directory' @@ -239,22 +254,25 @@ case "$operation" in snapshot_policy_sha=$(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) snapshot_state_sha=$(sha256sum "$snapshot/state.json" | cut -d' ' -f1) install -m 0600 /dev/null "$consumed_marker" + audit_pending=1 + audit_phase=pre-adapter + adapter_status= + trap deploy_exit EXIT + trap 'exit 2' INT TERM umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" mv -Tf "$temporary" "$active" - trap 'rm -f "$active" "${request_snapshot:-}"' EXIT INT TERM set +e systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ --why='approved deployment is active' -- env CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? set -e if ((adapter_status != 0)); then - printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=failed status=%s\n' \ - "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ - "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" "$adapter_status" >&8 + audit_phase=adapter die 'deployment adapter failed after approval consumption' fi + audit_phase=post-adapter secure_directory "$deployed_root" 'deployed snapshot directory' inside "$snapshot" "$deployed_root" || die 'prepared deployed snapshot escaped managed state' secure_directory "$snapshot" 'prepared deployed snapshot' @@ -274,5 +292,6 @@ case "$operation" in "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" \ >&8 + audit_pending=0 ;; esac diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 6e956b8..d907d95 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -131,7 +131,6 @@ secure_file() { secure_directory() { local path=$1 mode=$2 create=${3:-0} [[ ! -L "$path" ]] || die "unsafe symlinked managed directory: $path" - [[ "$path" == "$(realpath -m -- "$path")" ]] || die "managed directory path contains a symlink or non-canonical component: $path" if [[ ! -e "$path" ]]; then [[ "$create" == 1 ]] || return 1 install -d -m "$mode" "$path" @@ -712,7 +711,7 @@ PY if [[ -L "$current" ]]; then old_release=$(readlink -f "$current") release_complete "$old_release" || block 'active deployer release is incomplete' - policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement' + if [[ "$mode" != repair ]]; then policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement'; fi fi run_verified_adapter "$config" "${cfg[ADAPTER_PATH]}" "${cfg[ADAPTER_SHA256]}" validate >/dev/null 2>&1 || die 'candidate adapter validation failed' install_release @@ -741,6 +740,7 @@ PY } perform_rollback() { + local source_commit deployer_identity key local -A rollback_policy=() if ((recovered_rollback)); then health=healthy; report CHANGED yes restore-host-policy-evidence-then-check no; return; fi active_deployment && block 'active deployment prevents rollback' @@ -748,17 +748,23 @@ perform_rollback() { secure_directory "$state_root" 700 0 secure_file "$previous_policy" 'last-known-good policy' parse_file "$previous_policy" rollback_policy 'last-known-good policy' "$config_keys" - [[ -v 'rollback_policy[DEPLOYER_IDENTITY]' ]] || block 'last-known-good policy is missing deployer identity' + for key in CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "rollback_policy[$key]" ]] || block "last-known-good policy is missing $key"; done + read -r core_ref environment target source_commit artifact deployer_identity < <(python3 - "$previous_state" <<'PY' +import json, sys +try: value=json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): raise SystemExit(1) +print(*(value.get(k, '') for k in ('core_ref','environment','target','source_commit','artifact','deployer_identity'))) +PY + ) || block 'last-known-good state is malformed' + [[ "$core_ref" =~ ^[0-9a-f]{40}$ && "$source_commit" =~ ^[0-9a-f]{40}$ && "$artifact" =~ @sha256:[0-9a-f]{64}$ ]] || block 'last-known-good state has unsafe immutable identifiers' + [[ "$core_ref" == "${rollback_policy[CORE_REF]}" && "$environment" == "${rollback_policy[ENVIRONMENT]}" && "$target" == "${rollback_policy[TARGET_ID]}" && "$source_commit" == "${rollback_policy[SOURCE_COMMIT]}" && "$artifact" == "${rollback_policy[ARTIFACT_IMAGE]}" && "$deployer_identity" == "${rollback_policy[DEPLOYER_IDENTITY]}" ]] || block 'last-known-good state and policy do not match' cfg[DEPLOYER_IDENTITY]=${rollback_policy[DEPLOYER_IDENTITY]} command -v docker >/dev/null || block 'docker is required for rollback isolation validation' reject_mixed_role begin_transaction install -m 0600 "$previous_state" "$state_file.new" install -m 0600 "$previous_policy" "$active_policy.new" - core_ref=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["core_ref"])' "$previous_state") - environment=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["environment"])' "$previous_state") - target=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["target"])' "$previous_state") - artifact=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["artifact"])' "$previous_state") + release_complete "$releases/$core_ref" || die 'last-known-good release is incomplete' unit_source=$releases/$core_ref/deploy/deployer ln -sfn "releases/$core_ref" "$install_root/.current.new" diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index bdae740..ecbdb01 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -412,6 +412,12 @@ expect_failure 'unsafe symlinked managed directory' "$installer" --repair --conf rm "$root/var/lock/ci-fleet-deployer" mv "$root/var/lock/ci-fleet-deployer.real" "$root/var/lock/ci-fleet-deployer" +mkdir -p "$root/run" +mv "$root/var/lock" "$root/run/lock" +ln -s ../run/lock "$root/var/lock" +expect_success "$installer" --check --config "$config" >/dev/null +rm "$root/var/lock"; mv "$root/run/lock" "$root/var/lock" + unit_path=$root/etc/systemd/system/ci-fleet-deployer.service mv "$unit_path" "$unit_path.real" printf 'unrelated-unit\n' >"$tmp/unrelated-unit" @@ -468,11 +474,19 @@ PY write_evidence expect_success "$installer" --repair --config "$config" >/dev/null +mv "$root/var/lib/ci-fleet-deployer/active-policy.conf" "$root/var/lib/ci-fleet-deployer/active-policy.missing" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/active-policy.conf" ]] || fail 'repair did not restore a missing active policy' +rm "$root/var/lib/ci-fleet-deployer/active-policy.missing" + printf '# force-transaction\n' >>"$root/etc/systemd/system/ci-fleet-deployer.service" health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) -export FAKE_ADAPTER_FAIL_HEALTH_AFTER=$((health_calls_before + 1)) FAKE_SYSTEMCTL_FAIL_COMMAND=disable +# Repair bypasses only the untrusted old-policy health probe, so the candidate +# health check after activation is the next (and only) adapter health call. +export FAKE_ADAPTER_FAIL_HEALTH_AFTER=$((health_calls_before)) FAKE_SYSTEMCTL_FAIL_COMMAND=disable expect_failure 'candidate health check failed after activation' "$installer" --repair --config "$config" >/dev/null unset FAKE_ADAPTER_FAIL_HEALTH_AFTER FAKE_SYSTEMCTL_FAIL_COMMAND +[[ $(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) == $((health_calls_before + 1)) ]] || fail 'repair did not run exactly the candidate health check' compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null || fail 'failed restoration deleted its recovery transaction' expect_success "$installer" --repair --config "$config" >/dev/null if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'retry did not recover the retained transaction'; fi @@ -522,6 +536,13 @@ rollback_calls_before=$(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) FAKE_SYSTEMD_VERIFY_EXIT=1 expect_failure 'systemd unit verification failed' "$installer" --rollback --config "$config" >/dev/null [[ $(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) == "$rollback_calls_before" ]] || fail 'application rollback ran before core rollback staging was proven' grep -Fq 'sha256:bbbbbbbb' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'failed rollback did not preserve current core state' +cp "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$tmp/last-known-good.saved" +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good.json" <<'PY' +import json, sys +p=sys.argv[1]; value=json.load(open(p)); value['core_ref']='2222222222222222222222222222222222222222'; json.dump(value,open(p,'w'),indent=2,sort_keys=True) +PY +expect_failure 'last-known-good state and policy do not match' "$installer" --rollback --config "$config" >/dev/null +cp "$tmp/last-known-good.saved" "$root/var/lib/ci-fleet-deployer/last-known-good.json"; chmod 0600 "$root/var/lib/ci-fleet-deployer/last-known-good.json" printf 'runner\n' >"$root/etc/systemd/system/actions.runner.rollback-drift.service" expect_failure 'ordinary GitHub Actions runner service is present' "$installer" --rollback --config "$config" >/dev/null [[ $(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) == "$rollback_calls_before" ]] || fail 'rollback adapter ran after role isolation drift' @@ -658,6 +679,17 @@ expect_failure 'ordinary GitHub Actions runner service is present' "$runtime" he [[ $(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) == "$cleanup_calls_before" ]] || fail 'cleanup ran after role isolation drift' [[ $(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) == "$health_calls_before" ]] || fail 'health adapter ran after role isolation drift' rm "$root/etc/systemd/system/actions.runner.late-added.service" +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=github-environment', 'APPROVAL_PROVIDER=github-environmnt')) +PY +expect_failure 'unsupported approval provider' "$runtime" deploy >/dev/null +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=github-environmnt', 'APPROVAL_PROVIDER=github-environment')) +PY python3 - "$approval" "$request" <<'PY' from pathlib import Path import sys @@ -710,7 +742,7 @@ export FAKE_ADAPTER_FAIL=$tmp/fail-adapter printf 'deploy\n' >"$FAKE_ADAPTER_FAIL" expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-adapter" -grep -Fq 'approval=failed-adapter-attempt policy=example-staging-policy-v1 result=failed status=42' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'consumed failed deployment was not audited' +grep -Fq 'approval=failed-adapter-attempt policy=example-staging-policy-v1 result=failed phase=adapter status=42' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'consumed failed deployment was not audited' write_evidence staging example-staging python3 - "$approval" <<'PY' from pathlib import Path @@ -721,6 +753,7 @@ cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT=$root/var/lib/ci-fleet-deployer/deployed expect_failure 'prepared deployed snapshot changed during deployment' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT +grep -Fq 'approval=snapshot-mutation-attempt policy=example-staging-policy-v1 result=failed phase=post-adapter' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'post-adapter deployment failure was not audited' write_evidence staging example-staging python3 - "$approval" <<'PY' from pathlib import Path From 9d78330ec1307b6118ea294b3a7a886059816f1d Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 13:49:47 -0500 Subject: [PATCH 12/90] fix: harden deployer runtime and rollback recovery review gaps - Accept the documented external-exact-head approval provider at runtime so hosts using it are not blocked by the exhaustive provider check - Require SCHEMA_VERSION=1 on the active policy before any adapter runs so missing or future schema versions fail closed - Stop after finalizing an interrupted committed rollback in acquire_lock instead of continuing into convergence, so operators perform the reported evidence reconciliation before any reactivation - Add focused regressions for each behavior --- scripts/deployer-runtime.sh | 3 +- scripts/install-deployer.sh | 3 +- scripts/test-install-deployer.sh | 68 +++++++++++++++++++++++++++++--- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index ac1bf1b..4b20f6e 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -141,6 +141,7 @@ secure_file "$config" 'deployer configuration' config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' parse_file "$config" cfg configuration "$config_keys" for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done +[[ ${cfg[SCHEMA_VERSION]:-} == 1 ]] || die 'configuration has an unsupported or missing schema version' [[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 @@ -206,7 +207,7 @@ case "$operation" in [[ -v "approved[$key]" && ${req[$key]} == "${approved[$key]}" ]] || die "deployment request does not match protected approval $key" done case ${cfg[APPROVAL_PROVIDER]} in - manual-exact-head) ;; + manual-exact-head|external-exact-head) ;; github-environment) [[ -v 'cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]' ]] || die 'GitHub Environment approval is missing capability evidence' inside "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" "$evidence_dir" || die 'capability evidence is outside the protected evidence directory' diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index d907d95..9b6e155 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -838,7 +838,8 @@ else fi case "$mode" in check) acquire_check_lock; perform_check ;; - install|upgrade|repair) acquire_lock; perform_converge ;; + install|upgrade|repair) acquire_lock + if ((recovered_rollback)); then health=healthy; report CHANGED yes restore-host-policy-evidence-then-check no; else perform_converge; fi ;; rollback) acquire_lock; perform_rollback ;; drain) acquire_lock; perform_drain ;; resume) acquire_lock; perform_resume ;; diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index ecbdb01..e0cfc1b 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -559,6 +559,29 @@ grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/install-state.json" grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$rollback" || fail 'rollback report lacks the exact operator reconciliation action' grep -Fq 'sha256:bbbbbbbb' "$config" || fail 'rollback unexpectedly rewrote operator-owned desired policy' +# An interrupted committed rollback must report recovery without converging. +recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.rollback-interrupted +rm -rf "$root/var/lib/ci-fleet-deployer/deployed" +mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$recovery_transaction/state/$name"; printf '%s\n' "$name" >>"$recovery_transaction/state-present"; } +done +for path in "$root"/etc/systemd/system/ci-fleet-deployer*; do + name=${path##*/}; cp "$path" "$recovery_transaction/units/$name"; printf '%s\n' "$name" >>"$recovery_transaction/units-present" +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$recovery_transaction/current-target" +printf '%s\n' ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer >"$recovery_transaction/timers-enabled" +install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) +recovery=$(expect_success "$installer" --upgrade --config "$config") +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$recovery" || fail 'interrupted committed rollback recovery lacks the operator reconciliation action' +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'convergence ran after committed rollback recovery' +[[ $(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) == "$health_calls_before" ]] || fail 'convergence health ran after committed rollback recovery' +[[ ! -e "$recovery_transaction" ]] || fail 'committed rollback recovery retained its transaction' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + write_production_gate write_evidence production example-production write_config production example-production @@ -688,21 +711,51 @@ expect_failure 'unsupported approval provider' "$runtime" deploy >/dev/null python3 - "$config" <<'PY' from pathlib import Path import sys -p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=github-environmnt', 'APPROVAL_PROVIDER=github-environment')) +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('SCHEMA_VERSION=1', 'SCHEMA_VERSION=2', 1)) +PY +expect_failure 'configuration has an unsupported or missing schema version' "$runtime" deploy >/dev/null +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('SCHEMA_VERSION=2', 'SCHEMA_VERSION=1', 1)) +PY +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=github-environmnt', 'APPROVAL_PROVIDER=external-exact-head').replace('APPROVAL_CAPABILITY_EVIDENCE_PATH='+str(p.parent/'evidence/github-capability.conf'), '').rstrip()+'\n') +PY +cp "$approval" "$request"; chmod 0600 "$request" +expect_success "$runtime" deploy >/dev/null +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=approval-20260808-2').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:01:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +python3 - "$config" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=external-exact-head', 'APPROVAL_PROVIDER=github-environment') + 'APPROVAL_CAPABILITY_EVIDENCE_PATH='+str(p.parent/'evidence/github-capability.conf')+'\n') PY python3 - "$approval" "$request" <<'PY' from pathlib import Path import sys for name in sys.argv[1:]: - p=Path(name); p.write_text(p.read_text().replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-99-99T99:99:99Z')) + p=Path(name); p.write_text(p.read_text().replace('APPROVED_AT=2026-08-08T20:01:00Z', 'APPROVED_AT=2026-99-99T99:99:99Z')) PY expect_failure 'deployment request has an invalid approval time' "$runtime" deploy >/dev/null write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=approval-20260808-3').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:02:00Z')) +PY cp "$approval" "$request"; chmod 0600 "$request" python3 - "$request" <<'PY' from pathlib import Path import sys -p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=forged-approval')) +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-3', 'APPROVAL_ID=forged-approval')) PY expect_failure 'deployment request does not match protected approval APPROVAL_ID' "$runtime" deploy >/dev/null cp "$approval" "$request"; chmod 0600 "$request" @@ -732,10 +785,11 @@ deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" && $(<"$tmp/unrelated-audit") == unrelated-audit ]] || fail 'unsafe audit storage was touched after adapter execution' rm "$root/var/log/ci-fleet-deployer/audit.log" +write_evidence staging example-staging python3 - "$approval" <<'PY' from pathlib import Path import sys -p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=failed-adapter-attempt')) +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=failed-adapter-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:03:00Z')) PY cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_FAIL=$tmp/fail-adapter @@ -743,11 +797,13 @@ printf 'deploy\n' >"$FAKE_ADAPTER_FAIL" expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-adapter" grep -Fq 'approval=failed-adapter-attempt policy=example-staging-policy-v1 result=failed phase=adapter status=42' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'consumed failed deployment was not audited' +rm -f "$root/var/lib/ci-fleet-deployer/last-request.conf" +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" write_evidence staging example-staging python3 - "$approval" <<'PY' from pathlib import Path import sys -p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=snapshot-mutation-attempt')) +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=snapshot-mutation-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:04:00Z')) PY cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT=$root/var/lib/ci-fleet-deployer/deployed @@ -758,7 +814,7 @@ write_evidence staging example-staging python3 - "$approval" <<'PY' from pathlib import Path import sys -p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=audit-replacement-attempt')) +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=audit-replacement-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:00Z')) PY cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_AUDIT_TARGET=$tmp/unrelated-audit From efa522473c1e417e946f5d780a5bcc84ac7ed437 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 14:54:04 -0500 Subject: [PATCH 13/90] fix: close deployer isolation and rollback recovery review gaps - Reject non-canonical credential references in the installer to match the runtime's strict path validation - Refuse convergence when install state is lost but a release or managed units survive, instead of initializing rollback state as a fresh install - Recheck host role isolation immediately before candidate adapter execution and activation - Retire committed rollback transactions atomically by renaming them out of the recoverable namespace before deletion - Snapshot the validated active policy and pass that immutable path to every runtime adapter invocation - Allow --rollback to run from the cross-validated retained pair when the operator-owned candidate configuration is missing or malformed - Add focused regressions for each behavior --- scripts/deployer-runtime.sh | 12 ++++++++---- scripts/install-deployer.sh | 27 +++++++++++++++++++++++++-- scripts/test-install-deployer.sh | 30 ++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 4b20f6e..d270c2a 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -38,7 +38,7 @@ deploy_exit() { "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" \ "${audit_phase:-post-consumption}" "$recorded_status" >&8 || true fi - rm -f "$active" "${request_snapshot:-}" || true + rm -f "$active" "${request_snapshot:-}" "${policy_snapshot:-}" || true return "$status" } expected_uid=0 @@ -138,6 +138,10 @@ if [[ "$operation" == drain ]]; then fi secure_file "$config" 'deployer configuration' +policy_snapshot=$(mktemp "$state_root/.active-policy.XXXXXX") +install -m 0600 "$config" "$policy_snapshot" +config=$policy_snapshot +trap 'rm -f "${policy_snapshot:-}"' EXIT config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER APPROVAL_EVIDENCE_PATH APPROVAL_CAPABILITY_EVIDENCE_PATH PRODUCTION_AUTHORIZATION_EVIDENCE_PATH CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE NETWORK_HOST MIN_DISK_GIB REQUIRE_COMPOSE' parse_file "$config" cfg configuration "$config_keys" for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done @@ -155,12 +159,12 @@ secure_directory "$log_root" 'deployer log directory' case "$operation" in health) reject_mixed_role - "$adapter_path" "$operation" + env CI_FLEET_DEPLOYER_CONFIG="$config" "$adapter_path" "$operation" ;; cleanup) not_drained reject_mixed_role - "$adapter_path" cleanup + env CI_FLEET_DEPLOYER_CONFIG="$config" "$adapter_path" cleanup ;; deploy) not_drained @@ -266,7 +270,7 @@ case "$operation" in mv -Tf "$temporary" "$active" set +e systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ - --why='approved deployment is active' -- env CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy + --why='approved deployment is active' -- env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? set -e if ((adapter_status != 0)); then diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 9b6e155..28c2967 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -159,6 +159,21 @@ parse_file() { validate_config() { inside "$config" "$etc_root" || block "configuration path must be inside $etc_root" secure_directory "$etc_root" 700 0 || block 'configuration directory is missing' + if [[ "$mode" == rollback ]]; then + # Rollback must work from the retained pair even when the operator-owned + # candidate configuration is missing or malformed. + declare -gA cfg=() + if [[ -f "$config" && ! -L "$config" && $(stat -c '%u:%a' "$config" 2>/dev/null) == "$expected_uid:600" ]]; then + validated_config=$(mktemp) + install -m 0600 "$config" "$validated_config" + config=$validated_config + parse_file "$config" cfg configuration "$config_keys" + if [[ ${cfg[SCHEMA_VERSION]:-} == 1 && ${cfg[ENVIRONMENT]:-} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]:-} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]]; then + environment=${cfg[ENVIRONMENT]}; target=${cfg[TARGET_ID]} + fi + fi + return + fi secure_file "$config" 'configuration file' validated_config=$(mktemp) install -m 0600 "$config" "$validated_config" @@ -197,6 +212,7 @@ validate_config() { file) inside "${cfg[CREDENTIAL_REF]}" "$etc_root/credentials" || block 'credential reference is outside the approved credential directory' [[ ! -L ${cfg[CREDENTIAL_REF]} && -f ${cfg[CREDENTIAL_REF]} ]] || block 'credential reference must be a regular file, not a symlink' + [[ ${cfg[CREDENTIAL_REF]} == "$(realpath -m -- "${cfg[CREDENTIAL_REF]}")" && $(realpath -e -- "${cfg[CREDENTIAL_REF]}") == "${cfg[CREDENTIAL_REF]}" ]] || block 'credential reference contains a symlink or non-canonical component' [[ $(stat -c '%u:%a' "${cfg[CREDENTIAL_REF]}") == "$expected_uid:600" ]] || block 'credential file must be owner-only mode 0600' ;; external) @@ -525,13 +541,15 @@ recover_interrupted_transaction() { } finalize_committed_rollback() { - local marker=$transaction_dir/application-rollback-committed + local marker=$transaction_dir/application-rollback-committed retired [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' publish_deployed_snapshot "$active_policy" "$state_file" || return rm -f "$previous_state" "$previous_policy" || return transaction_committed=1 - rm -rf -- "$transaction_dir" || return + retired=$state_root/.retired.$$.transaction + mv -Tf "$transaction_dir" "$retired" || return transaction_dir= + rm -rf -- "$retired" } commit_transaction() { @@ -688,6 +706,9 @@ perform_check() { perform_converge() { local had_state=0 old_environment old_target candidate_changed=1 old_release [[ -f "$state_file" ]] && had_state=1 + if ((!had_state)) && { [[ -L "$current" ]] || compgen -G "$systemd_root/ci-fleet-deployer*" >/dev/null; }; then + block 'installed deployer state is absent or drifted; restore install state before convergence' + fi if ((had_state)); then read -r old_environment old_target < <(python3 - "$state_file" <<'PY' import json,sys @@ -713,6 +734,7 @@ PY release_complete "$old_release" || block 'active deployer release is incomplete' if [[ "$mode" != repair ]]; then policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement'; fi fi + reject_mixed_role run_verified_adapter "$config" "${cfg[ADAPTER_PATH]}" "${cfg[ADAPTER_SHA256]}" validate >/dev/null 2>&1 || die 'candidate adapter validation failed' install_release secure_directory "$state_root" 700 1 @@ -758,6 +780,7 @@ PY ) || block 'last-known-good state is malformed' [[ "$core_ref" =~ ^[0-9a-f]{40}$ && "$source_commit" =~ ^[0-9a-f]{40}$ && "$artifact" =~ @sha256:[0-9a-f]{64}$ ]] || block 'last-known-good state has unsafe immutable identifiers' [[ "$core_ref" == "${rollback_policy[CORE_REF]}" && "$environment" == "${rollback_policy[ENVIRONMENT]}" && "$target" == "${rollback_policy[TARGET_ID]}" && "$source_commit" == "${rollback_policy[SOURCE_COMMIT]}" && "$artifact" == "${rollback_policy[ARTIFACT_IMAGE]}" && "$deployer_identity" == "${rollback_policy[DEPLOYER_IDENTITY]}" ]] || block 'last-known-good state and policy do not match' + [[ "$environment" =~ ^[a-z][a-z0-9-]{0,31}$ && "$target" =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'last-known-good identity is malformed' cfg[DEPLOYER_IDENTITY]=${rollback_policy[DEPLOYER_IDENTITY]} command -v docker >/dev/null || block 'docker is required for rollback isolation validation' reject_mixed_role diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index e0cfc1b..47ad6b5 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -109,6 +109,7 @@ cat >"$adapter" <<'EOF' #!/usr/bin/env bash set -Eeuo pipefail printf '%s\n' "$1" >>"${FAKE_ADAPTER_LOG:?}" +[[ -z ${FAKE_ADAPTER_RECORD_CONFIG:-} ]] || { printf '%s\n' "${CI_FLEET_DEPLOYER_CONFIG:-unset}" >"$FAKE_ADAPTER_RECORD_CONFIG"; : >"$FAKE_ADAPTER_RECORD_CONFIG.seen"; cp "${CI_FLEET_DEPLOYER_CONFIG:-/dev/null}" "$FAKE_ADAPTER_RECORD_CONFIG.content" 2>/dev/null || true; } if [[ -n ${FAKE_ADAPTER_FORBID_CONFIG_PATH:-} && ${CI_FLEET_DEPLOYER_CONFIG:-} == "$FAKE_ADAPTER_FORBID_CONFIG_PATH" ]]; then exit 43; fi if [[ "$1" == deploy && -n ${FAKE_ADAPTER_FORBID_REQUEST_PATH:-} && ${CI_FLEET_DEPLOYER_REQUEST:-} == "$FAKE_ADAPTER_FORBID_REQUEST_PATH" ]]; then exit 44; fi if [[ -n ${FAKE_ADAPTER_REPLACE_PATH:-} && -e $FAKE_ADAPTER_REPLACE_PATH ]]; then @@ -455,6 +456,13 @@ PY expect_failure 'credential reference must be a regular file, not a symlink' "$installer" --check --config "$config" >/dev/null rm "$root/etc/ci-fleet-deployer/credentials/symlinked" write_config +python3 - "$config" "$root/etc/ci-fleet-deployer/credentials/./application.credential" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_REF='+str(p.parent/'credentials/application.credential'), 'CREDENTIAL_REF='+sys.argv[2])) +PY +expect_failure 'credential reference contains a symlink or non-canonical component' "$installer" --check --config "$config" >/dev/null +write_config printf 'bad\n' >>"$approval" approval_error=$(expect_failure 'malformed approval evidence line' "$installer" --check --config "$config") @@ -582,6 +590,21 @@ grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$recovery" || fail ' expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null +# A lost install state with surviving release/units must not be treated as a fresh install. +mv "$root/var/lib/ci-fleet-deployer/install-state.json" "$tmp/install-state.saved" +expect_failure 'restore install state before convergence' "$installer" --repair --config "$config" >/dev/null +mv "$tmp/install-state.saved" "$root/var/lib/ci-fleet-deployer/install-state.json" +chmod 0600 "$root/var/lib/ci-fleet-deployer/install-state.json" +expect_success "$installer" --repair --config "$config" >/dev/null + +# Rollback must work from the retained pair without a usable candidate config. +mv "$config" "$tmp/config.saved" +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'config-independent rollback did not report change' +mv "$tmp/config.saved" "$config"; chmod 0600 "$config" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + write_production_gate write_evidence production example-production write_config production example-production @@ -670,6 +693,13 @@ unset FAKE_ADAPTER_REPLACE_PATH rm "$adapter"; mv "$adapter.saved" "$adapter" expect_success "$runtime" health >/dev/null expect_success "$runtime" cleanup >/dev/null +adapter_config=$tmp/adapter-config +FAKE_ADAPTER_RECORD_CONFIG=$adapter_config expect_success "$runtime" health >/dev/null +unset FAKE_ADAPTER_RECORD_CONFIG +[[ -e "$adapter_config.seen" ]] || fail 'adapter config snapshot was not recorded' +[[ $(<"$adapter_config") == "$root"/var/lib/ci-fleet-deployer/.active-policy.* ]] || fail 'adapter did not receive an immutable policy snapshot path' +cmp -s "$adapter_config.content" "$root/var/lib/ci-fleet-deployer/active-policy.conf" || fail 'adapter policy snapshot content differs from the validated policy' +compgen -G "$root/var/lib/ci-fleet-deployer/.active-policy.*" >/dev/null && fail 'policy snapshot was not cleaned up' chmod 0644 "$credential" cleanup_calls_before=$(grep -Fxc cleanup "$FAKE_ADAPTER_LOG" || true) expect_failure 'credential file has unsafe owner or mode' "$runtime" cleanup >/dev/null From 9e52f59d3db3e5f31d87865eb26758f4b9e25983 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 15:54:11 -0500 Subject: [PATCH 14/90] fix: harden deployer transactions, credentials, and audit identity - Retire successful transactions atomically before deletion so an interrupted cleanup cannot be misread as an uncommitted transaction - Treat a malformed candidate configuration as absent during rollback instead of failing before the retained pair is used - Require a valid CORE_REF in the runtime active policy before any adapter runs so a successful deployment cannot destroy the rollback path - Revalidate retained credential provider, scope, containment, type, ownership, and mode before every direct installer adapter call - Recheck runtime host isolation immediately before approval consumption and adapter execution - Record approval identity, checkpoint, and production authorization identities in deployment success and failure audit records - Report rollback_available only when the retained pair passes schema, ownership, mode, and cross-field validation - Add focused regressions for each behavior --- scripts/deployer-runtime.sh | 13 ++++++--- scripts/install-deployer.sh | 50 +++++++++++++++++++++++++++++--- scripts/test-install-deployer.sh | 33 +++++++++++++++++++-- 3 files changed, 86 insertions(+), 10 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index d270c2a..14f32fb 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -33,9 +33,10 @@ deploy_exit() { local status=$? local recorded_status=${adapter_status:-$status} if [[ ${audit_pending:-0} == 1 ]]; then - printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=failed phase=%s status=%s\n' \ + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=failed phase=%s status=%s\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ - "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ + "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ "${audit_phase:-post-consumption}" "$recorded_status" >&8 || true fi rm -f "$active" "${request_snapshot:-}" "${policy_snapshot:-}" || true @@ -43,6 +44,7 @@ deploy_exit() { } expected_uid=0 [[ "$testing" != 1 ]] || expected_uid=$(id -u) +declare -A production=() secure_file() { local path=$1 description=$2 mode=${3:-600} [[ ! -L "$path" && -f "$path" ]] || die "$description must be a regular file, not a symlink" @@ -146,6 +148,7 @@ config_keys='SCHEMA_VERSION CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADA parse_file "$config" cfg configuration "$config_keys" for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done [[ ${cfg[SCHEMA_VERSION]:-} == 1 ]] || die 'configuration has an unsupported or missing schema version' +[[ ${cfg[CORE_REF]:-} =~ ^[0-9a-f]{40}$ ]] || die 'configuration is missing a valid core revision' [[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 @@ -258,6 +261,7 @@ case "$operation" in install -m 0600 "$install_state" "$snapshot/state.json" snapshot_policy_sha=$(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) snapshot_state_sha=$(sha256sum "$snapshot/state.json" | cut -d' ' -f1) + reject_mixed_role install -m 0600 /dev/null "$consumed_marker" audit_pending=1 audit_phase=pre-adapter @@ -293,9 +297,10 @@ case "$operation" in request_snapshot= secure_file "$audit_log" 'deployer audit log' [[ $(stat -Lc '%d:%i' /proc/self/fd/8) == $(stat -c '%d:%i' "$audit_log") ]] || die 'deployer audit log changed during deployment' - printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s policy=%s result=success\n' \ + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=success\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ - "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[POLICY_IDENTITY]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ + "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ >&8 audit_pending=0 ;; diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 28c2967..77bc584 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -56,7 +56,32 @@ report() { printf 'REPORT action=%s result=%s environment=%s target=%s version=%s digest=%s health=%s changed=%s rollback_available=%s next=%s\n' \ "$action" "$result" "$environment" "$target" "$core_ref" "${artifact#*@}" "$health" "$changed" "$rollback" "$next" } -rollback_available() { [[ -n ${previous_state:-} && -f ${previous_state:-/nonexistent} && -n ${previous_policy:-} && -f ${previous_policy:-/nonexistent} ]] && printf yes || printf no; } +rollback_available() { + [[ -n ${previous_state:-} && -f ${previous_state:-/nonexistent} && ! -L ${previous_state:-/nonexistent} && -n ${previous_policy:-} && -f ${previous_policy:-/nonexistent} && ! -L ${previous_policy:-/nonexistent} ]] || { printf no; return; } + [[ $(stat -c '%u:%a' "$previous_state" 2>/dev/null) == "$expected_uid:600" && $(stat -c '%u:%a' "$previous_policy" 2>/dev/null) == "$expected_uid:600" ]] || { printf no; return; } + python3 - "$previous_state" "$previous_policy" <<'PY' >/dev/null 2>&1 || { printf no; return; } +import json, sys +try: + state = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): + raise SystemExit(1) +policy = {} +try: + for line in open(sys.argv[2], encoding='utf-8'): + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + key, sep, value = line.partition('=') + if not sep: + raise SystemExit(1) + policy[key] = value +except OSError: + raise SystemExit(1) +pairs = (('core_ref','CORE_REF'), ('environment','ENVIRONMENT'), ('target','TARGET_ID'), ('source_commit','SOURCE_COMMIT'), ('artifact','ARTIFACT_IMAGE'), ('deployer_identity','DEPLOYER_IDENTITY')) +raise SystemExit(0 if all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) else 1) +PY + printf yes +} die() { error_reported=1; printf 'ERROR: %s\n' "$*" >&2; report FAILED no inspect-and-retry "$(rollback_available)" >&2; exit 2; } block() { error_reported=1; printf 'BLOCKED: %s\n' "$*" >&2; report BLOCKED no resolve-precondition "$(rollback_available)" >&2; exit 3; } @@ -167,7 +192,8 @@ validate_config() { validated_config=$(mktemp) install -m 0600 "$config" "$validated_config" config=$validated_config - parse_file "$config" cfg configuration "$config_keys" + cfg_dump=$(parse_file "$config" cfg configuration "$config_keys" && declare -p cfg) 2>/dev/null || cfg_dump= + if [[ -n "$cfg_dump" ]]; then eval "$cfg_dump"; else cfg=(); fi if [[ ${cfg[SCHEMA_VERSION]:-} == 1 && ${cfg[ENVIRONMENT]:-} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]:-} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]]; then environment=${cfg[ENVIRONMENT]}; target=${cfg[TARGET_ID]} fi @@ -553,9 +579,12 @@ finalize_committed_rollback() { } commit_transaction() { + local retired transaction_committed=1 - rm -rf -- "$transaction_dir" + retired=$state_root/.retired.$$.transaction + mv -Tf "$transaction_dir" "$retired" || return transaction_dir= + rm -rf -- "$retired" } atomic_replace_directory() { @@ -685,8 +714,21 @@ policy_adapter_operation() { local -A policy_cfg=() secure_file "$policy" "$description" parse_file "$policy" policy_cfg "$description" "$config_keys" - for key in ADAPTER_PATH ADAPTER_SHA256; do [[ -v "policy_cfg[$key]" ]] || die "$description is missing $key"; done + for key in ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE ENVIRONMENT; do [[ -v "policy_cfg[$key]" ]] || die "$description is missing $key"; done [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" + [[ ${policy_cfg[CREDENTIAL_SCOPE]} == "${policy_cfg[ENVIRONMENT]}" ]] || die "$description credential scope does not match its environment" + case ${policy_cfg[CREDENTIAL_PROVIDER]} in + file) + inside "${policy_cfg[CREDENTIAL_REF]}" "$etc_root/credentials" || die "$description credential reference is outside the approved credential directory" + [[ ! -L ${policy_cfg[CREDENTIAL_REF]} && -f ${policy_cfg[CREDENTIAL_REF]} ]] || die "$description credential reference must be a regular file, not a symlink" + [[ ${policy_cfg[CREDENTIAL_REF]} == "$(realpath -m -- "${policy_cfg[CREDENTIAL_REF]}")" && $(realpath -e -- "${policy_cfg[CREDENTIAL_REF]}") == "${policy_cfg[CREDENTIAL_REF]}" ]] || die "$description credential reference contains a symlink or non-canonical component" + [[ $(stat -c '%u:%a' "${policy_cfg[CREDENTIAL_REF]}") == "$expected_uid:600" ]] || die "$description credential file must be owner-only mode 0600" + ;; + external) + [[ ${policy_cfg[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die "$description has an invalid external secret-manager adapter reference" + ;; + *) die "$description CREDENTIAL_PROVIDER must be file or external" ;; + esac run_verified_adapter "$policy" "${policy_cfg[ADAPTER_PATH]}" "${policy_cfg[ADAPTER_SHA256]}" "$operation_name" "$marker" >/dev/null 2>&1 } diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 47ad6b5..814a14e 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -519,6 +519,10 @@ expect_failure 'candidate adapter validation failed' "$installer" --upgrade --co [[ "$old_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'failed candidate replaced healthy state' rm "$FAKE_ADAPTER_FAIL"; unset FAKE_ADAPTER_FAIL +chmod 0644 "$credential" +expect_failure 'credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null +chmod 0600 "$credential" + install -m 0600 /dev/null "$root/var/lib/ci-fleet-deployer/drained" health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) export FAKE_ADAPTER_FAIL_HEALTH_AFTER=$((health_calls_before + 1)) @@ -603,8 +607,22 @@ rollback=$(expect_success "$installer" --rollback --config "$config") grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'config-independent rollback did not report change' mv "$tmp/config.saved" "$config"; chmod 0600 "$config" expect_success "$installer" --repair --config "$config" >/dev/null +printf 'malformed line\n' >"$config"; chmod 0600 "$config" +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'malformed-config rollback did not report change' +write_config +expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null +# rollback_available must reflect a validated retained pair. +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'expected a retained rollback point' +chmod 0644 "$root/var/lib/ci-fleet-deployer/last-known-good.json" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'drifted retained pair still reported rollback_available=yes' +chmod 0600 "$root/var/lib/ci-fleet-deployer/last-known-good.json" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=yes' <<<"$available_check" || fail 'valid retained pair was not reported rollback_available=yes' + write_production_gate write_evidence production example-production write_config production example-production @@ -752,6 +770,17 @@ PY python3 - "$config" <<'PY' from pathlib import Path import sys +p=Path(sys.argv[1]); p.write_text('\n'.join(x for x in p.read_text().splitlines() if not x.startswith('CORE_REF='))+'\n') +PY +expect_failure 'configuration is missing a valid core revision' "$runtime" deploy >/dev/null +python3 - "$config" "$core_ref" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text() + 'CORE_REF='+sys.argv[2]+'\n') +PY +python3 - "$config" <<'PY' +from pathlib import Path +import sys p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_PROVIDER=github-environmnt', 'APPROVAL_PROVIDER=external-exact-head').replace('APPROVAL_CAPABILITY_EVIDENCE_PATH='+str(p.parent/'evidence/github-capability.conf'), '').rstrip()+'\n') PY cp "$approval" "$request"; chmod 0600 "$request" @@ -826,7 +855,7 @@ export FAKE_ADAPTER_FAIL=$tmp/fail-adapter printf 'deploy\n' >"$FAKE_ADAPTER_FAIL" expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-adapter" -grep -Fq 'approval=failed-adapter-attempt policy=example-staging-policy-v1 result=failed phase=adapter status=42' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'consumed failed deployment was not audited' +grep -Fq 'approval=failed-adapter-attempt approver=example-reviewer policy=example-staging-policy-v1 checkpoint=checkpoint-20260808-1 authorized_by=none gate=none result=failed phase=adapter status=42' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'consumed failed deployment was not audited' rm -f "$root/var/lib/ci-fleet-deployer/last-request.conf" rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" write_evidence staging example-staging @@ -839,7 +868,7 @@ cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT=$root/var/lib/ci-fleet-deployer/deployed expect_failure 'prepared deployed snapshot changed during deployment' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT -grep -Fq 'approval=snapshot-mutation-attempt policy=example-staging-policy-v1 result=failed phase=post-adapter' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'post-adapter deployment failure was not audited' +grep -Fq 'approval=snapshot-mutation-attempt approver=example-reviewer policy=example-staging-policy-v1 checkpoint=checkpoint-20260808-1 authorized_by=none gate=none result=failed phase=post-adapter' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'post-adapter deployment failure was not audited' write_evidence staging example-staging python3 - "$approval" <<'PY' from pathlib import Path From 0242374e66fadd793dd7c56a36d9dc707e71cc6d Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 16:48:35 -0500 Subject: [PATCH 15/90] fix: track deployed pointer and validate state across recovery paths - Record the deployed snapshot pointer in installation transactions and restore or remove it during recovery so an interrupted first install cannot leave an obsolete rollback pointer - Validate install-state core revision against the active policy before publishing a deployed rollback snapshot - Apply the strict retained-pair validation (allowed keys, duplicates, schema, immutable identifiers) when reporting rollback_available - Allow uninstall to run from fixed managed state when the candidate configuration is missing or malformed - Compose the runtime EXIT traps so rejected requests never leave stale active-policy snapshots - Add focused regressions for each behavior --- scripts/deployer-runtime.sh | 10 ++++++++- scripts/install-deployer.sh | 36 ++++++++++++++++++++++++++------ scripts/test-install-deployer.sh | 12 ++++++++++- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 14f32fb..e87c165 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -176,7 +176,7 @@ case "$operation" in secure_file "$request" 'deployment request' request_snapshot=$(mktemp "$state_root/.request.XXXXXX") install -m 0600 "$request" "$request_snapshot" - trap 'rm -f "${request_snapshot:-}"' EXIT INT TERM + trap 'rm -f "${request_snapshot:-}" "${policy_snapshot:-}"' EXIT INT TERM request_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' parse_file "$request_snapshot" req request "$request_keys" for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT; do @@ -247,6 +247,14 @@ case "$operation" in if [[ -e "$audit_log" || -L "$audit_log" ]]; then secure_file "$audit_log" 'deployer audit log'; else install -m 0600 /dev/null "$audit_log"; fi exec 8>>"$audit_log" secure_file "$install_state" 'deployer install state' + state_core_ref=$(python3 - "$install_state" <<'PY' +import json, sys +try: value = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): raise SystemExit(1) +print(value.get('core_ref', '')) +PY + ) || die 'deployer install state is malformed' + [[ "$state_core_ref" =~ ^[0-9a-f]{40}$ && "$state_core_ref" == "${cfg[CORE_REF]}" ]] || die 'deployer install state does not match the active policy core revision' if [[ -e "$deployed_root" || -L "$deployed_root" ]]; then secure_directory "$deployed_root" 'deployed snapshot directory'; else install -d -m 0700 "$deployed_root"; fi [[ -e "$deployed_current" || -L "$deployed_current" ]] || die 'deployed rollback snapshot is missing' [[ -L "$deployed_current" ]] || die 'deployed snapshot pointer is absent or unsafe' diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 77bc584..74ac841 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -61,6 +61,7 @@ rollback_available() { [[ $(stat -c '%u:%a' "$previous_state" 2>/dev/null) == "$expected_uid:600" && $(stat -c '%u:%a' "$previous_policy" 2>/dev/null) == "$expected_uid:600" ]] || { printf no; return; } python3 - "$previous_state" "$previous_policy" <<'PY' >/dev/null 2>&1 || { printf no; return; } import json, sys +allowed = {"SCHEMA_VERSION","CORE_REF","ENVIRONMENT","TARGET_ID","DEPLOYER_IDENTITY","ADAPTER_PATH","ADAPTER_SHA256","CREDENTIAL_PROVIDER","CREDENTIAL_REF","CREDENTIAL_SCOPE","APPROVAL_PROVIDER","APPROVAL_EVIDENCE_PATH","APPROVAL_CAPABILITY_EVIDENCE_PATH","PRODUCTION_AUTHORIZATION_EVIDENCE_PATH","CHECKPOINT_EVIDENCE_PATH","SOURCE_COMMIT","ARTIFACT_IMAGE","NETWORK_HOST","MIN_DISK_GIB","REQUIRE_COMPOSE"} try: state = json.load(open(sys.argv[1], encoding='utf-8')) except (OSError, ValueError): @@ -72,13 +73,19 @@ try: if not line or line.startswith('#'): continue key, sep, value = line.partition('=') - if not sep: + if not sep or key not in allowed or key in policy or not value: raise SystemExit(1) policy[key] = value except OSError: raise SystemExit(1) +import re +sha = re.compile(r'^[0-9a-f]{40}$') pairs = (('core_ref','CORE_REF'), ('environment','ENVIRONMENT'), ('target','TARGET_ID'), ('source_commit','SOURCE_COMMIT'), ('artifact','ARTIFACT_IMAGE'), ('deployer_identity','DEPLOYER_IDENTITY')) -raise SystemExit(0 if all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) else 1) +ok = (all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) + and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) + and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact'])) + and policy.get('SCHEMA_VERSION') == '1') +raise SystemExit(0 if ok else 1) PY printf yes } @@ -184,9 +191,9 @@ parse_file() { validate_config() { inside "$config" "$etc_root" || block "configuration path must be inside $etc_root" secure_directory "$etc_root" 700 0 || block 'configuration directory is missing' - if [[ "$mode" == rollback ]]; then - # Rollback must work from the retained pair even when the operator-owned - # candidate configuration is missing or malformed. + if [[ "$mode" == rollback || "$mode" == uninstall ]]; then + # Rollback and uninstall must work even when the operator-owned candidate + # configuration is missing or malformed. declare -gA cfg=() if [[ -f "$config" && ! -L "$config" && $(stat -c '%u:%a' "$config" 2>/dev/null) == "$expected_uid:600" ]]; then validated_config=$(mktemp) @@ -466,7 +473,7 @@ acquire_check_lock() { } begin_transaction() { - local name path current_target transaction_name transaction_ready + local name path current_target transaction_name transaction_ready deployed_target for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do path=$state_root/$name [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:600" ]] || block 'managed transaction state has an unsafe type, owner, or mode' @@ -502,6 +509,13 @@ begin_transaction() { for name in "${timer_names[@]}"; do if systemctl is-enabled "$name" >/dev/null 2>&1; then printf '%s\n' "$name" >>"$transaction_dir/timers-enabled"; fi done + if [[ -L "$deployed_current" ]]; then + deployed_target=$(readlink "$deployed_current") + [[ "$deployed_target" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'deployed snapshot pointer is unsafe' + printf '%s\n' "$deployed_target" >"$transaction_dir/deployed-target" + elif [[ -e "$deployed_current" ]]; then block 'deployed snapshot pointer has an unsafe type' + else printf 'absent\n' >"$transaction_dir/deployed-target" + fi transaction_name=${transaction_dir##*/} transaction_ready=$state_root/.transaction.${transaction_name#.transaction-preparing.} mv "$transaction_dir" "$transaction_ready" @@ -534,6 +548,16 @@ restore_transaction() { [[ "$target_value" =~ ^releases/[0-9a-f]{40}$ ]] || block 'transaction current pointer is unsafe' ln -s "$target_value" "$current" || return fi + if [[ -f "$transaction_dir/deployed-target" ]]; then + target_value=$(<"$transaction_dir/deployed-target") + if [[ "$target_value" == absent ]]; then + rm -f -- "$deployed_current" || return + else + [[ "$target_value" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'transaction deployed pointer is unsafe' + rm -f -- "$deployed_current" || return + ln -s "$target_value" "$deployed_current" || return + fi + fi systemctl daemon-reload >/dev/null 2>&1 || return if [[ -f "$transaction_dir/timers-enabled" ]]; then while IFS= read -r name; do diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 814a14e..4a41bb7 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -610,8 +610,17 @@ expect_success "$installer" --repair --config "$config" >/dev/null printf 'malformed line\n' >"$config"; chmod 0600 "$config" rollback=$(expect_success "$installer" --rollback --config "$config") grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'malformed-config rollback did not report change' +expect_success "$installer" --uninstall --config "$config" >/dev/null +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'malformed-config uninstall retained the activation pointer' write_config -expect_success "$installer" --repair --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --install --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null # rollback_available must reflect a validated retained pair. @@ -804,6 +813,7 @@ for name in sys.argv[1:]: p=Path(name); p.write_text(p.read_text().replace('APPROVED_AT=2026-08-08T20:01:00Z', 'APPROVED_AT=2026-99-99T99:99:99Z')) PY expect_failure 'deployment request has an invalid approval time' "$runtime" deploy >/dev/null +compgen -G "$root/var/lib/ci-fleet-deployer/.active-policy.*" >/dev/null && fail 'rejected request left a stale policy snapshot' write_evidence staging example-staging python3 - "$approval" <<'PY' from pathlib import Path From c6cffb3d33fc93054e0a496286345ef0283050f4 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 17:33:16 -0500 Subject: [PATCH 16/90] fix: validate deployer identity, isolation, and boundaries end to end - Validate every install-state identity field against the active policy before publishing a deployed rollback snapshot - Recheck host isolation immediately before candidate activation, after the potentially slow candidate validation window - Pass an immutable policy snapshot to every direct installer adapter call so the adapter cannot observe replaced policy bytes - Require schema version 1 on the retained rollback policy before the rollback transaction begins - Remove prepared deployed snapshots on every failed deployment exit - Constrain runtime adapter paths to the protected adapter directory - Fail closed on uninstall when the activation pointer has an unsafe type - Validate the managed state boundary before transaction recovery scans or mutates transaction entries - Extend focused regressions for these paths --- scripts/deployer-runtime.sh | 10 ++++++---- scripts/install-deployer.sh | 13 ++++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index e87c165..fb7b039 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -40,6 +40,7 @@ deploy_exit() { "${audit_phase:-post-consumption}" "$recorded_status" >&8 || true fi rm -f "$active" "${request_snapshot:-}" "${policy_snapshot:-}" || true + [[ -z ${snapshot:-} ]] || rm -rf -- "$snapshot" || true return "$status" } expected_uid=0 @@ -151,6 +152,7 @@ for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 C [[ ${cfg[CORE_REF]:-} =~ ^[0-9a-f]{40}$ ]] || die 'configuration is missing a valid core revision' [[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' +inside "${cfg[ADAPTER_PATH]}" "$(root_path /etc/ci-fleet-deployer/adapters)" || die 'application adapter is outside the protected adapter directory' secure_file "${cfg[ADAPTER_PATH]}" 'application adapter' 700 exec 7<"${cfg[ADAPTER_PATH]}" [[ $(sha256sum /proc/$$/fd/7 | cut -d' ' -f1) == "${cfg[ADAPTER_SHA256]}" ]] || die 'application adapter digest mismatch' @@ -247,14 +249,14 @@ case "$operation" in if [[ -e "$audit_log" || -L "$audit_log" ]]; then secure_file "$audit_log" 'deployer audit log'; else install -m 0600 /dev/null "$audit_log"; fi exec 8>>"$audit_log" secure_file "$install_state" 'deployer install state' - state_core_ref=$(python3 - "$install_state" <<'PY' + python3 - "$install_state" "${cfg[CORE_REF]}" "${cfg[ENVIRONMENT]}" "${cfg[TARGET_ID]}" "${cfg[DEPLOYER_IDENTITY]}" "${cfg[SOURCE_COMMIT]}" "${cfg[ARTIFACT_IMAGE]}" <<'PY' >/dev/null 2>&1 || die 'deployer install state is malformed' import json, sys try: value = json.load(open(sys.argv[1], encoding='utf-8')) except (OSError, ValueError): raise SystemExit(1) -print(value.get('core_ref', '')) +keys = ('core_ref', 'environment', 'target', 'deployer_identity', 'source_commit', 'artifact') +raise SystemExit(0 if all(value.get(k) == v for k, v in zip(keys, sys.argv[2:])) else 1) PY - ) || die 'deployer install state is malformed' - [[ "$state_core_ref" =~ ^[0-9a-f]{40}$ && "$state_core_ref" == "${cfg[CORE_REF]}" ]] || die 'deployer install state does not match the active policy core revision' + [[ ${cfg[CORE_REF]} =~ ^[0-9a-f]{40}$ ]] || die 'configuration is missing a valid core revision' if [[ -e "$deployed_root" || -L "$deployed_root" ]]; then secure_directory "$deployed_root" 'deployed snapshot directory'; else install -d -m 0700 "$deployed_root"; fi [[ -e "$deployed_current" || -L "$deployed_current" ]] || die 'deployed rollback snapshot is missing' [[ -L "$deployed_current" ]] || die 'deployed snapshot pointer is absent or unsafe' diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 74ac841..b8b1f54 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -442,6 +442,7 @@ converged() { managed_boundaries_match && current_matches && state_matches && un acquire_lock() { local path secure_directory "$lock_root" 700 1 + if [[ -e "$state_root" || -L "$state_root" ]]; then secure_directory "$state_root" 700 0 || block 'managed state boundary is unsafe'; fi exec 9<"$lock_root" flock -n 9 || block 'another deployer installer operation is running' [[ ! -L "$active_operation" ]] || block 'active operation marker is an unsafe symlink' @@ -734,9 +735,12 @@ run_verified_adapter() { } policy_adapter_operation() { - local policy=$1 operation_name=$2 description=$3 marker=${4:-} key + local policy=$1 operation_name=$2 description=$3 marker=${4:-} key snapshot local -A policy_cfg=() secure_file "$policy" "$description" + snapshot=$(mktemp "$state_root/.policy-check.XXXXXX") + install -m 0600 "$policy" "$snapshot" + policy=$snapshot parse_file "$policy" policy_cfg "$description" "$config_keys" for key in ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE ENVIRONMENT; do [[ -v "policy_cfg[$key]" ]] || die "$description is missing $key"; done [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" @@ -753,7 +757,11 @@ policy_adapter_operation() { ;; *) die "$description CREDENTIAL_PROVIDER must be file or external" ;; esac + local status run_verified_adapter "$policy" "${policy_cfg[ADAPTER_PATH]}" "${policy_cfg[ADAPTER_SHA256]}" "$operation_name" "$marker" >/dev/null 2>&1 + status=$? + rm -f "$snapshot" + return "$status" } perform_check() { @@ -817,6 +825,7 @@ PY install_units install -m 0600 "$config" "$active_policy.new" write_state "$state_root/.install-state.new" + reject_mixed_role mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_root/.install-state.new" "$state_file" mv -Tf "$install_root/.current.new" "$current" @@ -837,6 +846,7 @@ perform_rollback() { secure_file "$previous_policy" 'last-known-good policy' parse_file "$previous_policy" rollback_policy 'last-known-good policy' "$config_keys" for key in CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "rollback_policy[$key]" ]] || block "last-known-good policy is missing $key"; done + [[ ${rollback_policy[SCHEMA_VERSION]:-} == 1 ]] || block 'last-known-good policy has an unsupported or missing schema version' read -r core_ref environment target source_commit artifact deployer_identity < <(python3 - "$previous_state" <<'PY' import json, sys try: value=json.load(open(sys.argv[1], encoding='utf-8')) @@ -911,6 +921,7 @@ perform_uninstall() { fi if active_deployment; then block 'active deployment started while draining'; fi if [[ -L "$current" ]]; then rm -f "$current"; changed=yes; fi + [[ ! -e "$current" ]] || block 'activation pointer has an unsafe type' systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true for unit in "${unit_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then rm -f "$systemd_root/$unit"; changed=yes; fi; done systemctl daemon-reload >/dev/null 2>&1 || true From 628791d9a2e725d90b6c38b99253fd92fbf686b3 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 18:17:09 -0500 Subject: [PATCH 17/90] fix: preserve deployer recovery invariants across lifecycle paths - Cross-validate deployed snapshot state and policy before promotion to the retained last-known-good pair - Constrain direct installer adapter calls to the protected adapter directory - Retire the previous deployed snapshot after publishing its replacement - Record timer enablement and activation separately so failed candidates restore the exact pre-transaction service state - Reject future active-operation timestamps so a corrupted marker cannot block maintenance indefinitely - Keep the deployer ownership identity immutable across in-place upgrades - Add focused regressions for these behaviors --- scripts/deployer-runtime.sh | 3 +++ scripts/install-deployer.sh | 43 ++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index fb7b039..1a975e8 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -299,9 +299,12 @@ PY secure_file "$snapshot/state.json" 'prepared deployed state' [[ $(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) == "$snapshot_policy_sha" && $(sha256sum "$snapshot/state.json" | cut -d' ' -f1) == "$snapshot_state_sha" ]] || die 'prepared deployed snapshot changed during deployment' pointer=$deployed_root/.current.$$ + retired_snapshot=$(readlink "$deployed_current" 2>/dev/null || true) + [[ -z "$retired_snapshot" || "$retired_snapshot" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'current deployed snapshot pointer is unsafe' ln -s "${snapshot##*/}" "$pointer" mv -Tf "$pointer" "$deployed_current" snapshot= + if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi if [[ -f "$request" && ! -L "$request" ]] && cmp -s "$request_snapshot" "$request"; then rm -f "$request"; fi mv -Tf "$request_snapshot" "$last_request" request_snapshot= diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index b8b1f54..233dba4 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -385,6 +385,7 @@ active_deployment() { started=$(awk -F= '$1=="started_at" {print $2}' "$active_operation") [[ "$pid" =~ ^[1-9][0-9]*$ && "$started" =~ ^[0-9]+$ ]] || return 0 now=$(date +%s) + ((started <= now)) || return 0 ((now - started <= 3600)) || return 1 kill -0 "$pid" 2>/dev/null || return 0 return 0 @@ -509,6 +510,7 @@ begin_transaction() { done for name in "${timer_names[@]}"; do if systemctl is-enabled "$name" >/dev/null 2>&1; then printf '%s\n' "$name" >>"$transaction_dir/timers-enabled"; fi + if systemctl is-active "$name" >/dev/null 2>&1; then printf '%s\n' "$name" >>"$transaction_dir/timers-active"; fi done if [[ -L "$deployed_current" ]]; then deployed_target=$(readlink "$deployed_current") @@ -563,9 +565,15 @@ restore_transaction() { if [[ -f "$transaction_dir/timers-enabled" ]]; then while IFS= read -r name; do [[ " ${timer_names[*]} " == *" $name "* ]] || block 'transaction timer manifest is unsafe' - systemctl enable --now "$name" >/dev/null 2>&1 || return + systemctl enable "$name" >/dev/null 2>&1 || return done <"$transaction_dir/timers-enabled" fi + if [[ -f "$transaction_dir/timers-active" ]]; then + while IFS= read -r name; do + [[ " ${timer_names[*]} " == *" $name "* ]] || block 'transaction timer manifest is unsafe' + systemctl start "$name" >/dev/null 2>&1 || return + done <"$transaction_dir/timers-active" + fi rm -rf -- "$transaction_dir" || return transaction_dir= } @@ -675,6 +683,32 @@ load_deployed_snapshot() { secure_file "$snapshot/state.json" 'deployed rollback state' deployed_snapshot_policy=$snapshot/policy.conf deployed_snapshot_state=$snapshot/state.json + python3 - "$deployed_snapshot_state" "$deployed_snapshot_policy" <<'PY' >/dev/null 2>&1 || block 'deployed rollback snapshot state and policy do not cross-validate' +import json, re, sys +try: + state = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): + raise SystemExit(1) +policy = {} +try: + for line in open(sys.argv[2], encoding='utf-8'): + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + key, sep, value = line.partition('=') + if not sep or key in policy: + raise SystemExit(1) + policy[key] = value +except OSError: + raise SystemExit(1) +sha = re.compile(r'^[0-9a-f]{40}$') +pairs = (('core_ref','CORE_REF'), ('environment','ENVIRONMENT'), ('target','TARGET_ID'), ('source_commit','SOURCE_COMMIT'), ('artifact','ARTIFACT_IMAGE'), ('deployer_identity','DEPLOYER_IDENTITY')) +ok = (policy.get('SCHEMA_VERSION') == '1' + and all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) + and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) + and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact']))) +raise SystemExit(0 if ok else 1) +PY } publish_deployed_snapshot() { @@ -744,6 +778,7 @@ policy_adapter_operation() { parse_file "$policy" policy_cfg "$description" "$config_keys" for key in ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE ENVIRONMENT; do [[ -v "policy_cfg[$key]" ]] || die "$description is missing $key"; done [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" + inside "${policy_cfg[ADAPTER_PATH]}" "$etc_root/adapters" || die "$description adapter path is outside the protected adapter directory" [[ ${policy_cfg[CREDENTIAL_SCOPE]} == "${policy_cfg[ENVIRONMENT]}" ]] || die "$description credential scope does not match its environment" case ${policy_cfg[CREDENTIAL_PROVIDER]} in file) @@ -790,6 +825,12 @@ v=json.load(open(sys.argv[1])); print(v.get('environment',''),v.get('target','') PY ) [[ "$old_environment" == "$environment" && "$old_target" == "$target" ]] || block 'installed environment and target identity cannot change in place' + old_deployer_identity=$(python3 - "$state_file" <<'PY' +import json,sys +print(json.load(open(sys.argv[1])).get('deployer_identity','')) +PY +) + [[ -z "$old_deployer_identity" || "$old_deployer_identity" == "${cfg[DEPLOYER_IDENTITY]}" ]] || block 'installed deployer ownership identity cannot change in place' [[ -e "$deployed_current" || -L "$deployed_current" ]] || block 'deployed rollback snapshot is missing; restore it before convergence' load_deployed_snapshot if [[ "$mode" == install && -L "$current" ]] && ! state_matches; then block 'install cannot select a new candidate; use --upgrade or --repair'; fi From b4710070dc51750b9f6af79005e3676927f04483 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 20:04:03 -0500 Subject: [PATCH 18/90] fix: close deployer publication, rollback, and uninstall safety gaps - reject unknown keys in deployed policy parsing before promotion - recheck role isolation immediately before the rollback adapter call - validate committed rollback policy/state before publishing it - retire the superseded deployed snapshot after rollback publication - mark transactions committed only after atomic retirement succeeds - prevalidate managed unit types before any uninstall mutation - never delete a published deployed snapshot from exit handling --- scripts/deployer-runtime.sh | 2 +- scripts/install-deployer.sh | 49 ++++++++++++++++++++++++++++---- scripts/test-install-deployer.sh | 39 +++++++++++++++++++++---- 3 files changed, 79 insertions(+), 11 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 1a975e8..9f20fc7 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -302,7 +302,7 @@ PY retired_snapshot=$(readlink "$deployed_current" 2>/dev/null || true) [[ -z "$retired_snapshot" || "$retired_snapshot" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'current deployed snapshot pointer is unsafe' ln -s "${snapshot##*/}" "$pointer" - mv -Tf "$pointer" "$deployed_current" + mv -Tf "$pointer" "$deployed_current" || { rm -f "$pointer"; snapshot=; die 'deployed snapshot publication failed'; } snapshot= if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi if [[ -f "$request" && ! -L "$request" ]] && cmp -s "$request_snapshot" "$request"; then rm -f "$request"; fi diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 233dba4..e54b967 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -602,21 +602,50 @@ recover_interrupted_transaction() { finalize_committed_rollback() { local marker=$transaction_dir/application-rollback-committed retired [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' + secure_file "$active_policy" 'active policy' || return + secure_file "$state_file" 'deployer install state' || return + python3 - "$state_file" "$active_policy" <<'PY' >/dev/null 2>&1 || return +import json, re, sys +try: + state = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): + raise SystemExit(1) +policy = {} +allowed = {"SCHEMA_VERSION","CORE_REF","ENVIRONMENT","TARGET_ID","DEPLOYER_IDENTITY","ADAPTER_PATH","ADAPTER_SHA256","CREDENTIAL_PROVIDER","CREDENTIAL_REF","CREDENTIAL_SCOPE","APPROVAL_PROVIDER","APPROVAL_EVIDENCE_PATH","APPROVAL_CAPABILITY_EVIDENCE_PATH","PRODUCTION_AUTHORIZATION_EVIDENCE_PATH","CHECKPOINT_EVIDENCE_PATH","SOURCE_COMMIT","ARTIFACT_IMAGE","NETWORK_HOST","MIN_DISK_GIB","REQUIRE_COMPOSE"} +try: + for line in open(sys.argv[2], encoding='utf-8'): + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + key, sep, value = line.partition('=') + if not sep or key in policy or key not in allowed: + raise SystemExit(1) + policy[key] = value +except OSError: + raise SystemExit(1) +sha = re.compile(r'^[0-9a-f]{40}$') +pairs = (('core_ref','CORE_REF'), ('environment','ENVIRONMENT'), ('target','TARGET_ID'), ('source_commit','SOURCE_COMMIT'), ('artifact','ARTIFACT_IMAGE'), ('deployer_identity','DEPLOYER_IDENTITY')) +ok = (policy.get('SCHEMA_VERSION') == '1' + and all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) + and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) + and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact']))) +raise SystemExit(0 if ok else 1) +PY publish_deployed_snapshot "$active_policy" "$state_file" || return rm -f "$previous_state" "$previous_policy" || return - transaction_committed=1 retired=$state_root/.retired.$$.transaction mv -Tf "$transaction_dir" "$retired" || return transaction_dir= + transaction_committed=1 rm -rf -- "$retired" } commit_transaction() { local retired - transaction_committed=1 retired=$state_root/.retired.$$.transaction mv -Tf "$transaction_dir" "$retired" || return transaction_dir= + transaction_committed=1 rm -rf -- "$retired" } @@ -690,13 +719,14 @@ try: except (OSError, ValueError): raise SystemExit(1) policy = {} +allowed = {"SCHEMA_VERSION","CORE_REF","ENVIRONMENT","TARGET_ID","DEPLOYER_IDENTITY","ADAPTER_PATH","ADAPTER_SHA256","CREDENTIAL_PROVIDER","CREDENTIAL_REF","CREDENTIAL_SCOPE","APPROVAL_PROVIDER","APPROVAL_EVIDENCE_PATH","APPROVAL_CAPABILITY_EVIDENCE_PATH","PRODUCTION_AUTHORIZATION_EVIDENCE_PATH","CHECKPOINT_EVIDENCE_PATH","SOURCE_COMMIT","ARTIFACT_IMAGE","NETWORK_HOST","MIN_DISK_GIB","REQUIRE_COMPOSE"} try: for line in open(sys.argv[2], encoding='utf-8'): line = line.rstrip('\n') if not line or line.startswith('#'): continue key, sep, value = line.partition('=') - if not sep or key in policy: + if not sep or key in policy or key not in allowed: raise SystemExit(1) policy[key] = value except OSError: @@ -712,9 +742,15 @@ PY } publish_deployed_snapshot() { - local policy=$1 state=$2 snapshot pointer + local policy=$1 state=$2 snapshot pointer retired secure_directory "$deployed_root" 700 1 || return - if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then load_deployed_snapshot || return; fi + if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then + load_deployed_snapshot || return + if cmp -s "$deployed_snapshot_policy" "$policy" && cmp -s "$deployed_snapshot_state" "$state"; then return; fi + retired=$(readlink "$deployed_current") + [[ "$retired" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'current deployed snapshot pointer is unsafe' + rm -f -- "$deployed_current" || return + fi snapshot=$(mktemp -d "$deployed_root/.snapshot.XXXXXX") || return chmod 0700 "$snapshot" || return install -m 0600 "$policy" "$snapshot/policy.conf" || return @@ -722,6 +758,7 @@ publish_deployed_snapshot() { pointer=$deployed_root/.current.$$ ln -s "${snapshot##*/}" "$pointer" || return mv -Tf "$pointer" "$deployed_current" || return + if [[ -n ${retired:-} && -d "$deployed_root/$retired" && ! -L "$deployed_root/$retired" ]]; then rm -rf -- "${deployed_root:?}/$retired"; fi } install_units() { @@ -912,6 +949,7 @@ PY mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_file.new" "$state_file" mv -Tf "$install_root/.current.new" "$current" + reject_mixed_role policy_adapter_operation "$active_policy" rollback 'last-known-good policy' "$transaction_dir/application-rollback-committed" || die 'application adapter rollback failed' finalize_committed_rollback recovered_rollback=0 @@ -953,6 +991,7 @@ perform_uninstall() { if active_deployment; then block 'active deployment prevents this operation'; fi acquire_lock secure_directory "$state_root" 700 1 + for unit in "${unit_names[@]}"; do [[ ! -e "$systemd_root/$unit" || -L "$systemd_root/$unit" || -f "$systemd_root/$unit" ]] || block "managed unit $unit has an unsafe type"; done if [[ -e "$drained" || -L "$drained" ]]; then secure_file "$drained" 'drain marker' else diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 4a41bb7..4a19f7d 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -130,9 +130,9 @@ case "$1" in validate|health|cleanup|deploy|rollback) ;; *) exit 2 ;; esac if [[ "$1" == rollback && -n ${CI_FLEET_DEPLOYER_ROLLBACK_COMMIT:-} ]]; then install -m 0600 /dev/null "$CI_FLEET_DEPLOYER_ROLLBACK_COMMIT" fi -if [[ "$1" == deploy && -n ${FAKE_ADAPTER_AUDIT_PATH:-} ]]; then - rm -f "$FAKE_ADAPTER_AUDIT_PATH" - ln -s "$FAKE_ADAPTER_AUDIT_TARGET" "$FAKE_ADAPTER_AUDIT_PATH" +if [[ -n ${FAKE_ADAPTER_MUTATE_AUDIT_PATH:-} ]]; then + rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" + ln -s "$FAKE_ADAPTER_MUTATE_AUDIT_TARGET" "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" fi if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT:-} ]]; then snapshot=$(find "$FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT" -maxdepth 1 -type d -name '.snapshot.*' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-) @@ -695,6 +695,20 @@ grep -Fq 'result=CHANGED' <<<"$uninstall" || fail 'uninstall did not report chan repeat_uninstall=$(expect_success "$installer" --uninstall --config "$config") grep -Fq 'result=NO_CHANGE' <<<"$repeat_uninstall" || fail 'repeated uninstall was not idempotent' +# A drifted managed unit directory must fail closed before any uninstall mutation. +expect_success "$installer" --install --config "$config" >/dev/null +rm "$root/etc/systemd/system/ci-fleet-deployer-drain.service" +mkdir "$root/etc/systemd/system/ci-fleet-deployer-drain.service" +uninstall_before=$(ls -l --time-style=+%s "$root/etc/systemd/system" | sha256sum) +[[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'test setup expected no drain marker before drifted uninstall' +expect_failure 'managed unit ci-fleet-deployer-drain.service has an unsafe type' "$installer" --uninstall --config "$config" >/dev/null +[[ "$uninstall_before" == "$(ls -l --time-style=+%s "$root/etc/systemd/system" | sha256sum)" ]] || fail 'unsafe unit type partially mutated the host during uninstall' +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'failed closed uninstall removed the activation pointer' +[[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'failed closed uninstall created the drain marker' +rmdir "$root/etc/systemd/system/ci-fleet-deployer-drain.service" +expect_success "$installer" --uninstall --config "$config" >/dev/null +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after correcting unit drift did not uninstall' + # Runtime contract: exact-head request/evidence, drain and scoped adapter calls. write_evidence staging example-staging write_config staging example-staging @@ -886,11 +900,26 @@ import sys p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=audit-replacement-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:00Z')) PY cp "$approval" "$request"; chmod 0600 "$request" -export FAKE_ADAPTER_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_AUDIT_TARGET=$tmp/unrelated-audit +export FAKE_ADAPTER_MUTATE_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_MUTATE_AUDIT_TARGET=$tmp/unrelated-audit expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null -unset FAKE_ADAPTER_AUDIT_PATH FAKE_ADAPTER_AUDIT_TARGET +unset FAKE_ADAPTER_MUTATE_AUDIT_PATH FAKE_ADAPTER_MUTATE_AUDIT_TARGET [[ $(<"$tmp/unrelated-audit") == unrelated-audit ]] || fail 'adapter audit replacement redirected the trusted append' rm "$root/var/log/ci-fleet-deployer/audit.log" + +# Signal at the deployed-snapshot publication boundary must not delete the published snapshot. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=signal-at-publication-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:06:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_MUTATE_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_MUTATE_AUDIT_TARGET=$tmp/unrelated-audit +expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_MUTATE_AUDIT_PATH FAKE_ADAPTER_MUTATE_AUDIT_TARGET +[[ -L "$deployed_current" && -e "$deployed_current" && -f "$deployed_current/policy.conf" && -f "$deployed_current/state.json" ]] || fail 'post-publication failure left deployed/current dangling' +cmp -s "$deployed_current/policy.conf" "$config" || fail 'published deployed policy does not match the deployed configuration' +rm "$root/var/log/ci-fleet-deployer/audit.log" write_evidence staging example-staging cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_FORBID_REQUEST_PATH=$request From 25335bf1bffd68049820df245c3db92603ce5952 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 20:09:57 -0500 Subject: [PATCH 19/90] test: make drifted-unit uninstall check shellcheck-clean --- scripts/test-install-deployer.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 4a19f7d..877d8db 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -699,10 +699,10 @@ grep -Fq 'result=NO_CHANGE' <<<"$repeat_uninstall" || fail 'repeated uninstall w expect_success "$installer" --install --config "$config" >/dev/null rm "$root/etc/systemd/system/ci-fleet-deployer-drain.service" mkdir "$root/etc/systemd/system/ci-fleet-deployer-drain.service" -uninstall_before=$(ls -l --time-style=+%s "$root/etc/systemd/system" | sha256sum) +uninstall_before=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) [[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'test setup expected no drain marker before drifted uninstall' expect_failure 'managed unit ci-fleet-deployer-drain.service has an unsafe type' "$installer" --uninstall --config "$config" >/dev/null -[[ "$uninstall_before" == "$(ls -l --time-style=+%s "$root/etc/systemd/system" | sha256sum)" ]] || fail 'unsafe unit type partially mutated the host during uninstall' +[[ "$uninstall_before" == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'unsafe unit type partially mutated the host during uninstall' [[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'failed closed uninstall removed the activation pointer' [[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'failed closed uninstall created the drain marker' rmdir "$root/etc/systemd/system/ci-fleet-deployer-drain.service" From 9c8c5d3746c38c9a5a8c23f966e2358da2839b8f Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 20:44:26 -0500 Subject: [PATCH 20/90] fix: revalidate trust boundaries before every deployer adapter call - revalidate candidate credential provider, scope, containment, type, ownership, and mode immediately before candidate adapter validation - rerun role isolation inside the shared policy adapter helper so every installer health/rollback invocation rechecks the Docker boundary - clean policy-check snapshots through the exit handler on validation failures - arm runtime failure auditing and exit traps before consuming the approval marker, and disarm auditing when consumption never completed --- scripts/deployer-runtime.sh | 3 ++- scripts/install-deployer.sh | 34 +++++++++++++++++++++----------- scripts/test-install-deployer.sh | 18 +++++++++++++++++ 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 9f20fc7..dbbf009 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -32,6 +32,7 @@ die() { printf 'ERROR: %s\n' "$*" >&2; exit 2; } deploy_exit() { local status=$? local recorded_status=${adapter_status:-$status} + if [[ ${audit_pending:-0} == 1 && -n ${consumed_marker:-} && ! -e "$consumed_marker" ]]; then audit_pending=0; fi if [[ ${audit_pending:-0} == 1 ]]; then printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=failed phase=%s status=%s\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ @@ -272,12 +273,12 @@ PY snapshot_policy_sha=$(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) snapshot_state_sha=$(sha256sum "$snapshot/state.json" | cut -d' ' -f1) reject_mixed_role - install -m 0600 /dev/null "$consumed_marker" audit_pending=1 audit_phase=pre-adapter adapter_status= trap deploy_exit EXIT trap 'exit 2' INT TERM + install -m 0600 /dev/null "$consumed_marker" || { audit_pending=0; die 'deployment request consumption marker failed'; } umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index e54b967..1a0d126 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -38,6 +38,7 @@ on_exit() { fi [[ -z ${staging_path:-} ]] || rm -rf -- "$staging_path" [[ -z ${validated_config:-} ]] || rm -f -- "$validated_config" + [[ -z ${policy_check_snapshot:-} ]] || rm -f -- "$policy_check_snapshot" if ((status != 0 && error_reported == 0)); then report FAILED no inspect-and-retry "$(rollback_available)" >&2; fi return "$status" } @@ -153,6 +154,22 @@ valid_utc() { local value=$1 [[ "$value" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] && [[ $(date -u -d "$value" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null) == "$value" ]] } +credential_reference_safe() { + local provider=$1 reference=$2 description=$3 + case $provider in + file) + inside "$reference" "$etc_root/credentials" || die "$description credential reference is outside the approved credential directory" + [[ ! -L $reference && -f $reference ]] || die "$description credential reference must be a regular file, not a symlink" + [[ $reference == "$(realpath -m -- "$reference")" && $(realpath -e -- "$reference") == "$reference" ]] || die "$description credential reference contains a symlink or non-canonical component" + [[ $(stat -c '%u:%a' "$reference") == "$expected_uid:600" ]] || die "$description credential file must be owner-only mode 0600" + ;; + external) + [[ $reference =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die "$description has an invalid external secret-manager adapter reference" + ;; + *) die "$description CREDENTIAL_PROVIDER must be file or external" ;; + esac +} + secure_file() { local path=$1 description=$2 mode=${3:-600} [[ ! -L "$path" && -f "$path" ]] || block "$description must be a regular file, not a symlink" @@ -809,7 +826,9 @@ policy_adapter_operation() { local policy=$1 operation_name=$2 description=$3 marker=${4:-} key snapshot local -A policy_cfg=() secure_file "$policy" "$description" + reject_mixed_role snapshot=$(mktemp "$state_root/.policy-check.XXXXXX") + policy_check_snapshot=$snapshot install -m 0600 "$policy" "$snapshot" policy=$snapshot parse_file "$policy" policy_cfg "$description" "$config_keys" @@ -817,22 +836,12 @@ policy_adapter_operation() { [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" inside "${policy_cfg[ADAPTER_PATH]}" "$etc_root/adapters" || die "$description adapter path is outside the protected adapter directory" [[ ${policy_cfg[CREDENTIAL_SCOPE]} == "${policy_cfg[ENVIRONMENT]}" ]] || die "$description credential scope does not match its environment" - case ${policy_cfg[CREDENTIAL_PROVIDER]} in - file) - inside "${policy_cfg[CREDENTIAL_REF]}" "$etc_root/credentials" || die "$description credential reference is outside the approved credential directory" - [[ ! -L ${policy_cfg[CREDENTIAL_REF]} && -f ${policy_cfg[CREDENTIAL_REF]} ]] || die "$description credential reference must be a regular file, not a symlink" - [[ ${policy_cfg[CREDENTIAL_REF]} == "$(realpath -m -- "${policy_cfg[CREDENTIAL_REF]}")" && $(realpath -e -- "${policy_cfg[CREDENTIAL_REF]}") == "${policy_cfg[CREDENTIAL_REF]}" ]] || die "$description credential reference contains a symlink or non-canonical component" - [[ $(stat -c '%u:%a' "${policy_cfg[CREDENTIAL_REF]}") == "$expected_uid:600" ]] || die "$description credential file must be owner-only mode 0600" - ;; - external) - [[ ${policy_cfg[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die "$description has an invalid external secret-manager adapter reference" - ;; - *) die "$description CREDENTIAL_PROVIDER must be file or external" ;; - esac + credential_reference_safe "${policy_cfg[CREDENTIAL_PROVIDER]}" "${policy_cfg[CREDENTIAL_REF]}" "$description" local status run_verified_adapter "$policy" "${policy_cfg[ADAPTER_PATH]}" "${policy_cfg[ADAPTER_SHA256]}" "$operation_name" "$marker" >/dev/null 2>&1 status=$? rm -f "$snapshot" + policy_check_snapshot= return "$status" } @@ -887,6 +896,7 @@ PY if [[ "$mode" != repair ]]; then policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement'; fi fi reject_mixed_role + credential_reference_safe "${cfg[CREDENTIAL_PROVIDER]}" "${cfg[CREDENTIAL_REF]}" 'candidate policy' run_verified_adapter "$config" "${cfg[ADAPTER_PATH]}" "${cfg[ADAPTER_SHA256]}" validate >/dev/null 2>&1 || die 'candidate adapter validation failed' install_release secure_directory "$state_root" 700 1 diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 877d8db..e8641ec 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -134,6 +134,7 @@ if [[ -n ${FAKE_ADAPTER_MUTATE_AUDIT_PATH:-} ]]; then rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" ln -s "$FAKE_ADAPTER_MUTATE_AUDIT_TARGET" "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" fi +if [[ -n ${FAKE_ADAPTER_CHMOD_DURING:-} ]]; then chmod 0644 "$FAKE_ADAPTER_CHMOD_DURING"; fi if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT:-} ]]; then snapshot=$(find "$FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT" -maxdepth 1 -type d -name '.snapshot.*' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-) [[ -n "$snapshot" ]] && printf 'adapter-mutation\n' >>"$snapshot/policy.conf" @@ -523,6 +524,13 @@ chmod 0644 "$credential" expect_failure 'credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null chmod 0600 "$credential" +# Candidate validate must revalidate credentials drifted after configuration validation. +export FAKE_ADAPTER_CHMOD_DURING=$credential +expect_failure 'candidate policy credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null +unset FAKE_ADAPTER_CHMOD_DURING +chmod 0600 "$credential" +[[ "$old_state" == "$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json")" ]] || fail 'credential-drifted candidate replaced healthy state' + install -m 0600 /dev/null "$root/var/lib/ci-fleet-deployer/drained" health_calls_before=$(grep -Fxc health "$FAKE_ADAPTER_LOG" || true) export FAKE_ADAPTER_FAIL_HEALTH_AFTER=$((health_calls_before + 1)) @@ -539,6 +547,16 @@ p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20 PY expect_success "$installer" --upgrade --config "$config" >/dev/null grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/last-known-good.json" || fail 'second undeployed upgrade replaced the deployed rollback point' + +# Failed active-policy health checks must not accumulate policy-check snapshots. +compgen -G "$root/var/lib/ci-fleet-deployer/.policy-check.*" >/dev/null && fail 'policy-check snapshot leaked before drifted-health regression' +export FAKE_ADAPTER_FAIL=$tmp/fail-adapter-health +printf 'health\n' >"$FAKE_ADAPTER_FAIL" +expect_failure 'active deployer health check failed' "$installer" --check --config "$config" >/dev/null +expect_failure 'active deployer health check failed' "$installer" --check --config "$config" >/dev/null +unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-adapter-health" +compgen -G "$root/var/lib/ci-fleet-deployer/.policy-check.*" >/dev/null && fail 'failed health validation leaked policy-check snapshots' +expect_success "$installer" --check --config "$config" >/dev/null printf 'rollback\n' >"$tmp/fail-rollback" export FAKE_ADAPTER_FAIL=$tmp/fail-rollback expect_failure 'application adapter rollback failed' "$installer" --rollback --config "$config" >/dev/null From 24c2b4d4c1c657cd89ac42abce20bf383bc2408d Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 21:17:29 -0500 Subject: [PATCH 21/90] fix: fail closed across deployer recovery, signals, and read-only checks - validate full rollback-policy operational fields before promotion - revalidate runtime credentials immediately before health, cleanup, and deploy adapter calls - prevalidate all transaction manifests before any recovery mutation - keep the documented check path free of managed-state writes - exit nonzero from pre-consumption signal traps after cleanup - keep adapter verification diagnostics visible to operators --- scripts/deployer-runtime.sh | 6 ++- scripts/install-deployer.sh | 69 ++++++++++++++++++++++++-------- scripts/test-install-deployer.sh | 34 ++++++++++++++++ 3 files changed, 92 insertions(+), 17 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index dbbf009..11e65a2 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -165,11 +165,13 @@ secure_directory "$log_root" 'deployer log directory' case "$operation" in health) reject_mixed_role + validate_credential env CI_FLEET_DEPLOYER_CONFIG="$config" "$adapter_path" "$operation" ;; cleanup) not_drained reject_mixed_role + validate_credential env CI_FLEET_DEPLOYER_CONFIG="$config" "$adapter_path" cleanup ;; deploy) @@ -179,7 +181,8 @@ case "$operation" in secure_file "$request" 'deployment request' request_snapshot=$(mktemp "$state_root/.request.XXXXXX") install -m 0600 "$request" "$request_snapshot" - trap 'rm -f "${request_snapshot:-}" "${policy_snapshot:-}"' EXIT INT TERM + trap 'rm -f "${request_snapshot:-}" "${policy_snapshot:-}"' EXIT + trap 'exit 2' INT TERM request_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' parse_file "$request_snapshot" req request "$request_keys" for key in SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT; do @@ -273,6 +276,7 @@ PY snapshot_policy_sha=$(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) snapshot_state_sha=$(sha256sum "$snapshot/state.json" | cut -d' ' -f1) reject_mixed_role + validate_credential audit_pending=1 audit_phase=pre-adapter adapter_status= diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 1a0d126..1c1fc49 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -547,25 +547,46 @@ restore_transaction() { local name target_value [[ -n ${transaction_dir:-} && -d $transaction_dir ]] || return 0 transaction_committed=1 + if [[ -f "$transaction_dir/units-present" ]]; then + while IFS= read -r name; do + [[ " ${unit_names[*]} " == *" $name "* && -f "$transaction_dir/units/$name" && ! -L "$transaction_dir/units/$name" ]] || block 'transaction unit manifest is unsafe' + done <"$transaction_dir/units-present" + fi + if [[ -f "$transaction_dir/state-present" ]]; then + while IFS= read -r name; do + [[ " install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf " == *" $name "* && -f "$transaction_dir/state/$name" && ! -L "$transaction_dir/state/$name" ]] || block 'transaction state manifest is unsafe' + done <"$transaction_dir/state-present" + fi + for name in timers-enabled timers-active; do + if [[ -f "$transaction_dir/$name" ]]; then + while IFS= read -r target_value; do + [[ " ${timer_names[*]} " == *" $target_value "* ]] || block 'transaction timer manifest is unsafe' + done <"$transaction_dir/$name" + fi + done + if [[ -f "$transaction_dir/current-target" ]]; then + [[ $(<"$transaction_dir/current-target") =~ ^releases/[0-9a-f]{40}$ ]] || block 'transaction current pointer is unsafe' + fi + if [[ -f "$transaction_dir/deployed-target" ]]; then + target_value=$(<"$transaction_dir/deployed-target") + [[ "$target_value" == absent || "$target_value" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'transaction deployed pointer is unsafe' + fi systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || return for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name" || return; done if [[ -f "$transaction_dir/units-present" ]]; then while IFS= read -r name; do - [[ " ${unit_names[*]} " == *" $name "* && -f "$transaction_dir/units/$name" && ! -L "$transaction_dir/units/$name" ]] || block 'transaction unit manifest is unsafe' install -m 0644 "$transaction_dir/units/$name" "$systemd_root/$name" || return done <"$transaction_dir/units-present" fi for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do rm -f -- "$state_root/$name" || return; done if [[ -f "$transaction_dir/state-present" ]]; then while IFS= read -r name; do - [[ " install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf " == *" $name "* && -f "$transaction_dir/state/$name" && ! -L "$transaction_dir/state/$name" ]] || block 'transaction state manifest is unsafe' install -m 0600 "$transaction_dir/state/$name" "$state_root/$name" || return done <"$transaction_dir/state-present" fi rm -f -- "$current" "$install_root/.current.new" "$state_root/.install-state.new" "$active_policy.new" "$state_file.new" || return if [[ -f "$transaction_dir/current-target" ]]; then target_value=$(<"$transaction_dir/current-target") - [[ "$target_value" =~ ^releases/[0-9a-f]{40}$ ]] || block 'transaction current pointer is unsafe' ln -s "$target_value" "$current" || return fi if [[ -f "$transaction_dir/deployed-target" ]]; then @@ -573,7 +594,6 @@ restore_transaction() { if [[ "$target_value" == absent ]]; then rm -f -- "$deployed_current" || return else - [[ "$target_value" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'transaction deployed pointer is unsafe' rm -f -- "$deployed_current" || return ln -s "$target_value" "$deployed_current" || return fi @@ -581,13 +601,11 @@ restore_transaction() { systemctl daemon-reload >/dev/null 2>&1 || return if [[ -f "$transaction_dir/timers-enabled" ]]; then while IFS= read -r name; do - [[ " ${timer_names[*]} " == *" $name "* ]] || block 'transaction timer manifest is unsafe' systemctl enable "$name" >/dev/null 2>&1 || return done <"$transaction_dir/timers-enabled" fi if [[ -f "$transaction_dir/timers-active" ]]; then while IFS= read -r name; do - [[ " ${timer_names[*]} " == *" $name "* ]] || block 'transaction timer manifest is unsafe' systemctl start "$name" >/dev/null 2>&1 || return done <"$transaction_dir/timers-active" fi @@ -646,6 +664,15 @@ ok = (policy.get('SCHEMA_VERSION') == '1' and all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact']))) +ok = ok and all(policy.get(k) for k in ('ADAPTER_PATH','ADAPTER_SHA256','CREDENTIAL_PROVIDER','CREDENTIAL_REF','CREDENTIAL_SCOPE')) +ok = ok and bool(re.fullmatch(r'[0-9a-f]{64}', policy.get('ADAPTER_SHA256',''))) +ok = ok and policy.get('ADAPTER_PATH','').startswith('/') +ok = ok and policy.get('CREDENTIAL_PROVIDER') in ('file','external') +ok = ok and policy.get('CREDENTIAL_SCOPE') == policy.get('ENVIRONMENT') +if policy.get('CREDENTIAL_PROVIDER') == 'file': + ok = ok and policy.get('CREDENTIAL_REF','').startswith('/') +else: + ok = ok and bool(re.fullmatch(r'external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}', policy.get('CREDENTIAL_REF',''))) raise SystemExit(0 if ok else 1) PY publish_deployed_snapshot "$active_policy" "$state_file" || return @@ -754,6 +781,15 @@ ok = (policy.get('SCHEMA_VERSION') == '1' and all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact']))) +ok = ok and all(policy.get(k) for k in ('ADAPTER_PATH','ADAPTER_SHA256','CREDENTIAL_PROVIDER','CREDENTIAL_REF','CREDENTIAL_SCOPE')) +ok = ok and bool(re.fullmatch(r'[0-9a-f]{64}', policy.get('ADAPTER_SHA256',''))) +ok = ok and policy.get('ADAPTER_PATH','').startswith('/') +ok = ok and policy.get('CREDENTIAL_PROVIDER') in ('file','external') +ok = ok and policy.get('CREDENTIAL_SCOPE') == policy.get('ENVIRONMENT') +if policy.get('CREDENTIAL_PROVIDER') == 'file': + ok = ok and policy.get('CREDENTIAL_REF','').startswith('/') +else: + ok = ok and bool(re.fullmatch(r'external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}', policy.get('CREDENTIAL_REF',''))) raise SystemExit(0 if ok else 1) PY } @@ -823,14 +859,16 @@ run_verified_adapter() { } policy_adapter_operation() { - local policy=$1 operation_name=$2 description=$3 marker=${4:-} key snapshot + local policy=$1 operation_name=$2 description=$3 marker=${4:-} snapshot=${5:-1} key local -A policy_cfg=() secure_file "$policy" "$description" reject_mixed_role - snapshot=$(mktemp "$state_root/.policy-check.XXXXXX") - policy_check_snapshot=$snapshot - install -m 0600 "$policy" "$snapshot" - policy=$snapshot + if [[ "$snapshot" == 1 ]]; then + snapshot=$(mktemp "$state_root/.policy-check.XXXXXX") + policy_check_snapshot=$snapshot + install -m 0600 "$policy" "$snapshot" + policy=$snapshot + fi parse_file "$policy" policy_cfg "$description" "$config_keys" for key in ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE ENVIRONMENT; do [[ -v "policy_cfg[$key]" ]] || die "$description is missing $key"; done [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" @@ -838,10 +876,9 @@ policy_adapter_operation() { [[ ${policy_cfg[CREDENTIAL_SCOPE]} == "${policy_cfg[ENVIRONMENT]}" ]] || die "$description credential scope does not match its environment" credential_reference_safe "${policy_cfg[CREDENTIAL_PROVIDER]}" "${policy_cfg[CREDENTIAL_REF]}" "$description" local status - run_verified_adapter "$policy" "${policy_cfg[ADAPTER_PATH]}" "${policy_cfg[ADAPTER_SHA256]}" "$operation_name" "$marker" >/dev/null 2>&1 + run_verified_adapter "$policy" "${policy_cfg[ADAPTER_PATH]}" "${policy_cfg[ADAPTER_SHA256]}" "$operation_name" "$marker" >/dev/null status=$? - rm -f "$snapshot" - policy_check_snapshot= + if [[ "$snapshot" != 0 ]]; then rm -f "$snapshot"; policy_check_snapshot=; fi return "$status" } @@ -853,7 +890,7 @@ perform_check() { ((${#transactions[@]} == 0)) || block 'interrupted installer transaction requires recovery' if active_deployment; then block 'active deployment prevents a consistent check'; fi converged || block 'installed deployer state is absent or drifted' - policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer health check failed' + policy_adapter_operation "$active_policy" health 'active policy' '' 0 || block 'active deployer health check failed' health=healthy report NO_CHANGE no none "$(rollback_available)" } @@ -897,7 +934,7 @@ PY fi reject_mixed_role credential_reference_safe "${cfg[CREDENTIAL_PROVIDER]}" "${cfg[CREDENTIAL_REF]}" 'candidate policy' - run_verified_adapter "$config" "${cfg[ADAPTER_PATH]}" "${cfg[ADAPTER_SHA256]}" validate >/dev/null 2>&1 || die 'candidate adapter validation failed' + run_verified_adapter "$config" "${cfg[ADAPTER_PATH]}" "${cfg[ADAPTER_SHA256]}" validate >/dev/null || die 'candidate adapter validation failed' install_release secure_directory "$state_root" 700 1 secure_directory "$log_root" 700 1 diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index e8641ec..1f101cc 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -376,6 +376,25 @@ expect_success "$installer" --repair --config "$config" >/dev/null [[ ! -e "$interrupted" ]] || fail 'interrupted transaction was not recovered' expect_success "$installer" --check --config "$config" >/dev/null +# An unsafe transaction manifest must fail closed before any recovery mutation. +unsafe_tx=$root/var/lib/ci-fleet-deployer/.transaction.unsafe +mkdir -m 0700 "$unsafe_tx" "$unsafe_tx/units" "$unsafe_tx/state" +cp "$root/var/lib/ci-fleet-deployer/install-state.json" "$unsafe_tx/state/install-state.json" +printf 'install-state.json\n' >"$unsafe_tx/state-present" +printf 'rogue-unit.service\n' >"$unsafe_tx/units-present" +units_before_unsafe=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum) +expect_failure 'transaction unit manifest is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ "$units_before_unsafe" == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum)" ]] || fail 'unsafe transaction manifest mutated installed units' +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'unsafe transaction manifest removed the activation pointer' +rm "$unsafe_tx/units-present" +printf 'not-a-release\n' >"$unsafe_tx/current-target" +expect_failure 'transaction current pointer is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'unsafe transaction pointer removed the activation pointer' +rm "$unsafe_tx/current-target" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$unsafe_tx" ]] || fail 'corrected transaction was not recovered' +expect_success "$installer" --check --config "$config" >/dev/null + rm "$root/etc/systemd/system/ci-fleet-deployer-health.timer" expect_failure 'result=BLOCKED' "$installer" --check --config "$config" >/dev/null repair=$(expect_success "$installer" --repair --config "$config") @@ -557,6 +576,21 @@ expect_failure 'active deployer health check failed' "$installer" --check --conf unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-adapter-health" compgen -G "$root/var/lib/ci-fleet-deployer/.policy-check.*" >/dev/null && fail 'failed health validation leaked policy-check snapshots' expect_success "$installer" --check --config "$config" >/dev/null + +# Deployed policy drift in an operational field must block promotion. +deployed_policy=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current")/policy.conf +python3 - "$deployed_policy" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external')) +PY +expect_failure 'deployed rollback snapshot state and policy do not cross-validate' "$installer" --upgrade --config "$config" >/dev/null +python3 - "$deployed_policy" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=external', 'CREDENTIAL_PROVIDER=file')) +PY +expect_success "$installer" --check --config "$config" >/dev/null printf 'rollback\n' >"$tmp/fail-rollback" export FAKE_ADAPTER_FAIL=$tmp/fail-rollback expect_failure 'application adapter rollback failed' "$installer" --rollback --config "$config" >/dev/null From 48c686eb5b685044d46ebd0c7c4ab2d576d27100 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 9 Aug 2026 21:50:40 -0500 Subject: [PATCH 22/90] fix: harden deployer publication, recovery manifests, and convergence - disarm snapshot cleanup through the publication pointer itself so a signal at the rename boundary can never delete the published snapshot - require adapter and credential containment inside the protected directories in deployed-policy validation - reject non-regular or symlinked transaction manifests before recovery - require schema version 1 in the shared policy adapter helper - keep failure auditing armed when consumption marker creation fails - require root-owned non-writable intermediate release directories - include the systemd unit directory in convergence boundary checks - require the canonical relative activation target in checks --- scripts/deployer-runtime.sh | 15 ++++++++++++--- scripts/install-deployer.sh | 24 ++++++++++++++++-------- scripts/test-install-deployer.sh | 12 ++++++++++++ 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 11e65a2..ba1b912 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -32,6 +32,7 @@ die() { printf 'ERROR: %s\n' "$*" >&2; exit 2; } deploy_exit() { local status=$? local recorded_status=${adapter_status:-$status} + local target if [[ ${audit_pending:-0} == 1 && -n ${consumed_marker:-} && ! -e "$consumed_marker" ]]; then audit_pending=0; fi if [[ ${audit_pending:-0} == 1 ]]; then printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=failed phase=%s status=%s\n' \ @@ -41,7 +42,13 @@ deploy_exit() { "${audit_phase:-post-consumption}" "$recorded_status" >&8 || true fi rm -f "$active" "${request_snapshot:-}" "${policy_snapshot:-}" || true - [[ -z ${snapshot:-} ]] || rm -rf -- "$snapshot" || true + if [[ -n ${snapshot_pointer:-} && $snapshot_pointer != "$deployed_current" ]]; then + target=$(readlink "$snapshot_pointer" 2>/dev/null || true) + if [[ -n $target && ! -e "$deployed_root/$target" ]]; then rm -rf -- "$snapshot"; fi + rm -f "$snapshot_pointer" + elif [[ -z ${snapshot_pointer:-} ]]; then + [[ -z ${snapshot:-} ]] || rm -rf -- "$snapshot" + fi return "$status" } expected_uid=0 @@ -282,7 +289,7 @@ PY adapter_status= trap deploy_exit EXIT trap 'exit 2' INT TERM - install -m 0600 /dev/null "$consumed_marker" || { audit_pending=0; die 'deployment request consumption marker failed'; } + install -m 0600 /dev/null "$consumed_marker" || die 'deployment request consumption marker failed' umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" @@ -304,10 +311,12 @@ PY secure_file "$snapshot/state.json" 'prepared deployed state' [[ $(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) == "$snapshot_policy_sha" && $(sha256sum "$snapshot/state.json" | cut -d' ' -f1) == "$snapshot_state_sha" ]] || die 'prepared deployed snapshot changed during deployment' pointer=$deployed_root/.current.$$ + snapshot_pointer=$pointer retired_snapshot=$(readlink "$deployed_current" 2>/dev/null || true) [[ -z "$retired_snapshot" || "$retired_snapshot" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'current deployed snapshot pointer is unsafe' ln -s "${snapshot##*/}" "$pointer" - mv -Tf "$pointer" "$deployed_current" || { rm -f "$pointer"; snapshot=; die 'deployed snapshot publication failed'; } + mv -Tf "$pointer" "$deployed_current" + snapshot_pointer=$deployed_current snapshot= if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi if [[ -f "$request" && ! -L "$request" ]] && cmp -s "$request_snapshot" "$request"; then rm -f "$request"; fi diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 1c1fc49..c0e5dcd 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -409,8 +409,9 @@ active_deployment() { } release_complete() { - local release=$1 stored actual unit entry + local release=$1 stored actual unit entry dir [[ -d "$release" && ! -L "$release" && $(stat -c '%u:%a' "$release") == "$expected_uid:755" && -x "$release/scripts/install-deployer.sh" && -x "$release/scripts/deployer-runtime.sh" ]] || return 1 + for dir in "$release/scripts" "$release/deploy" "$release/deploy/deployer"; do [[ -d "$dir" && ! -L "$dir" && $(stat -c '%u:%a' "$dir") == "$expected_uid:755" ]] || return 1; done for entry in "$release/scripts/install-deployer.sh" "$release/scripts/deployer-runtime.sh"; do [[ ! -L "$entry" && $(stat -c '%u:%a' "$entry") == "$expected_uid:755" ]] || return 1; done [[ -f "$release/.ci-fleet-tree-sha256" ]] || return 1 stored=$(<"$release/.ci-fleet-tree-sha256") @@ -443,7 +444,7 @@ units_match() { current_matches() { local target_path=$releases/$core_ref - [[ -L "$current" && $(readlink -f "$current") == $(readlink -f "$target_path") ]] || return 1 + [[ -L "$current" && $(readlink "$current") == "releases/$core_ref" ]] || return 1 release_complete "$target_path" } @@ -453,6 +454,9 @@ managed_boundaries_match() { mode=${path##*:}; path=${path%:*} [[ -d "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:$mode" ]] || return 1 done + [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || return 1 + mode=$(stat -c %a "$systemd_root") + (((8#$mode & 8#022) == 0)) } converged() { managed_boundaries_match && current_matches && state_matches && units_match; } @@ -547,6 +551,9 @@ restore_transaction() { local name target_value [[ -n ${transaction_dir:-} && -d $transaction_dir ]] || return 0 transaction_committed=1 + for name in units-present state-present timers-enabled timers-active current-target deployed-target; do + [[ ! -e "$transaction_dir/$name" && ! -L "$transaction_dir/$name" ]] || [[ ! -L "$transaction_dir/$name" && -f "$transaction_dir/$name" ]] || block "transaction manifest $name has an unsafe type" + done if [[ -f "$transaction_dir/units-present" ]]; then while IFS= read -r name; do [[ " ${unit_names[*]} " == *" $name "* && -f "$transaction_dir/units/$name" && ! -L "$transaction_dir/units/$name" ]] || block 'transaction unit manifest is unsafe' @@ -639,7 +646,7 @@ finalize_committed_rollback() { [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' secure_file "$active_policy" 'active policy' || return secure_file "$state_file" 'deployer install state' || return - python3 - "$state_file" "$active_policy" <<'PY' >/dev/null 2>&1 || return + python3 - "$state_file" "$active_policy" "$etc_root/adapters" "$etc_root/credentials" <<'PY' >/dev/null 2>&1 || return import json, re, sys try: state = json.load(open(sys.argv[1], encoding='utf-8')) @@ -666,11 +673,11 @@ ok = (policy.get('SCHEMA_VERSION') == '1' and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact']))) ok = ok and all(policy.get(k) for k in ('ADAPTER_PATH','ADAPTER_SHA256','CREDENTIAL_PROVIDER','CREDENTIAL_REF','CREDENTIAL_SCOPE')) ok = ok and bool(re.fullmatch(r'[0-9a-f]{64}', policy.get('ADAPTER_SHA256',''))) -ok = ok and policy.get('ADAPTER_PATH','').startswith('/') +ok = ok and policy.get('ADAPTER_PATH','').startswith(sys.argv[3] + '/') ok = ok and policy.get('CREDENTIAL_PROVIDER') in ('file','external') ok = ok and policy.get('CREDENTIAL_SCOPE') == policy.get('ENVIRONMENT') if policy.get('CREDENTIAL_PROVIDER') == 'file': - ok = ok and policy.get('CREDENTIAL_REF','').startswith('/') + ok = ok and policy.get('CREDENTIAL_REF','').startswith(sys.argv[4] + '/') else: ok = ok and bool(re.fullmatch(r'external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}', policy.get('CREDENTIAL_REF',''))) raise SystemExit(0 if ok else 1) @@ -756,7 +763,7 @@ load_deployed_snapshot() { secure_file "$snapshot/state.json" 'deployed rollback state' deployed_snapshot_policy=$snapshot/policy.conf deployed_snapshot_state=$snapshot/state.json - python3 - "$deployed_snapshot_state" "$deployed_snapshot_policy" <<'PY' >/dev/null 2>&1 || block 'deployed rollback snapshot state and policy do not cross-validate' + python3 - "$deployed_snapshot_state" "$deployed_snapshot_policy" "$etc_root/adapters" "$etc_root/credentials" <<'PY' >/dev/null 2>&1 || block 'deployed rollback snapshot state and policy do not cross-validate' import json, re, sys try: state = json.load(open(sys.argv[1], encoding='utf-8')) @@ -783,11 +790,11 @@ ok = (policy.get('SCHEMA_VERSION') == '1' and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact']))) ok = ok and all(policy.get(k) for k in ('ADAPTER_PATH','ADAPTER_SHA256','CREDENTIAL_PROVIDER','CREDENTIAL_REF','CREDENTIAL_SCOPE')) ok = ok and bool(re.fullmatch(r'[0-9a-f]{64}', policy.get('ADAPTER_SHA256',''))) -ok = ok and policy.get('ADAPTER_PATH','').startswith('/') +ok = ok and policy.get('ADAPTER_PATH','').startswith(sys.argv[3] + '/') ok = ok and policy.get('CREDENTIAL_PROVIDER') in ('file','external') ok = ok and policy.get('CREDENTIAL_SCOPE') == policy.get('ENVIRONMENT') if policy.get('CREDENTIAL_PROVIDER') == 'file': - ok = ok and policy.get('CREDENTIAL_REF','').startswith('/') + ok = ok and policy.get('CREDENTIAL_REF','').startswith(sys.argv[4] + '/') else: ok = ok and bool(re.fullmatch(r'external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}', policy.get('CREDENTIAL_REF',''))) raise SystemExit(0 if ok else 1) @@ -870,6 +877,7 @@ policy_adapter_operation() { policy=$snapshot fi parse_file "$policy" policy_cfg "$description" "$config_keys" + [[ ${policy_cfg[SCHEMA_VERSION]:-} == 1 ]] || die "$description has an unsupported or missing schema version" for key in ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE ENVIRONMENT; do [[ -v "policy_cfg[$key]" ]] || die "$description is missing $key"; done [[ ${policy_cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die "$description has an invalid adapter digest" inside "${policy_cfg[ADAPTER_PATH]}" "$etc_root/adapters" || die "$description adapter path is outside the protected adapter directory" diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 1f101cc..297cd7c 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -391,10 +391,22 @@ printf 'not-a-release\n' >"$unsafe_tx/current-target" expect_failure 'transaction current pointer is unsafe' "$installer" --repair --config "$config" >/dev/null [[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'unsafe transaction pointer removed the activation pointer' rm "$unsafe_tx/current-target" +mkdir "$unsafe_tx/timers-enabled" +expect_failure 'transaction manifest timers-enabled has an unsafe type' "$installer" --repair --config "$config" >/dev/null +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'unsafe manifest type removed the activation pointer' +rmdir "$unsafe_tx/timers-enabled" expect_success "$installer" --repair --config "$config" >/dev/null [[ ! -e "$unsafe_tx" ]] || fail 'corrected transaction was not recovered' expect_success "$installer" --check --config "$config" >/dev/null +# A noncanonical activation pointer must fail convergence instead of certifying it. +rm "$root/opt/ci-fleet-deployer/current" +ln -s "$root/opt/ci-fleet-deployer/releases/$core_ref" "$root/opt/ci-fleet-deployer/current" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +rm "$root/opt/ci-fleet-deployer/current" +ln -s "releases/$core_ref" "$root/opt/ci-fleet-deployer/current" +expect_success "$installer" --check --config "$config" >/dev/null + rm "$root/etc/systemd/system/ci-fleet-deployer-health.timer" expect_failure 'result=BLOCKED' "$installer" --check --config "$config" >/dev/null repair=$(expect_success "$installer" --repair --config "$config") From aa7f1db1ebd9422b55c307aaf52c8a25e2886a78 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 15:37:35 -0500 Subject: [PATCH 23/90] fix: close deployer recovery, snapshot, inhibition, and checkout race findings - Verify deployed rollback adapter path, type, ownership, containment, and ADAPTER_SHA256 bytes before promoting the deployed pair to last-known-good or using it in rollback and transaction recovery. - Delete prepared deployed snapshots that never become the target of deployed/current, including signal interruption after temporary pointer creation and publication failure, while never touching the published current snapshot. - Cross-check transaction unit/state manifests against complete backup directory contents before recovery mutation; truncated, missing, or drifted manifests block before any unit or state change. - Treat a live recorded deployment process as active regardless of the one-hour stale bound; only dead PIDs follow bounded recovery. - Route any existing application-rollback-committed marker type through the finalizer, which blocks directories, FIFOs, and broken symlinks. - Pin reviewed checkout inputs by archiving the exact HEAD commit into a root-controlled snapshot before any privileged copy, eliminating the validate-to-stage substitution race. - Re-exec the deploy operation under systemd-inhibit so shutdown inhibition covers the adapter, snapshot publication, request retirement, and success audit, preserving exit status and failure auditing. --- scripts/deployer-runtime.sh | 20 +++--- scripts/install-deployer.sh | 51 ++++++++++++-- scripts/test-install-deployer.sh | 112 ++++++++++++++++++++++++++++++- 3 files changed, 166 insertions(+), 17 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index ba1b912..14b998b 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -29,6 +29,12 @@ audit_log=$log_root/audit.log systemd_root=$(root_path /etc/systemd/system) die() { printf 'ERROR: %s\n' "$*" >&2; exit 2; } +if [[ $operation == deploy && -z ${CI_FLEET_DEPLOYER_INHIBITED:-} ]]; then + export CI_FLEET_DEPLOYER_INHIBITED=1 + [[ $testing != 1 || -z ${CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG:-} ]] || printf '%s\n' deploy >>"$CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG" + exec systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ + --why='approved deployment is active' -- "$0" deploy +fi deploy_exit() { local status=$? local recorded_status=${adapter_status:-$status} @@ -42,13 +48,11 @@ deploy_exit() { "${audit_phase:-post-consumption}" "$recorded_status" >&8 || true fi rm -f "$active" "${request_snapshot:-}" "${policy_snapshot:-}" || true - if [[ -n ${snapshot_pointer:-} && $snapshot_pointer != "$deployed_current" ]]; then - target=$(readlink "$snapshot_pointer" 2>/dev/null || true) - if [[ -n $target && ! -e "$deployed_root/$target" ]]; then rm -rf -- "$snapshot"; fi - rm -f "$snapshot_pointer" - elif [[ -z ${snapshot_pointer:-} ]]; then - [[ -z ${snapshot:-} ]] || rm -rf -- "$snapshot" + if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then + target=$(readlink "$deployed_current" 2>/dev/null || true) + if [[ $target != "${snapshot##*/}" ]]; then rm -rf -- "$snapshot"; fi fi + if [[ -n ${snapshot_pointer:-} && $snapshot_pointer != "$deployed_current" ]]; then rm -f "$snapshot_pointer"; fi return "$status" } expected_uid=0 @@ -295,8 +299,7 @@ PY printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" mv -Tf "$temporary" "$active" set +e - systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ - --why='approved deployment is active' -- env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy + env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? set -e if ((adapter_status != 0)); then @@ -315,6 +318,7 @@ PY retired_snapshot=$(readlink "$deployed_current" 2>/dev/null || true) [[ -z "$retired_snapshot" || "$retired_snapshot" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'current deployed snapshot pointer is unsafe' ln -s "${snapshot##*/}" "$pointer" + [[ -z ${CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF:-} || $testing != 1 ]] || kill -"$CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF" $$ mv -Tf "$pointer" "$deployed_current" snapshot_pointer=$deployed_current snapshot= diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index c0e5dcd..ddc86a4 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -18,6 +18,7 @@ core_ref=unknown artifact=unknown health=unknown staging_path= +checkout_snapshot= transaction_dir= transaction_preparing=0 transaction_committed=0 @@ -29,7 +30,7 @@ on_exit() { transaction_dir= elif [[ -n ${transaction_dir:-} && ${transaction_committed:-0} != 1 ]]; then set +e - if [[ -f "$transaction_dir/application-rollback-committed" ]]; then finalize_committed_rollback; recovery_status=$?; else restore_transaction; recovery_status=$?; fi + if [[ -e "$transaction_dir/application-rollback-committed" || -L "$transaction_dir/application-rollback-committed" ]]; then finalize_committed_rollback; recovery_status=$?; else restore_transaction; recovery_status=$?; fi set -e if ((recovery_status != 0)); then status=$recovery_status @@ -37,6 +38,7 @@ on_exit() { fi fi [[ -z ${staging_path:-} ]] || rm -rf -- "$staging_path" + [[ -z ${checkout_snapshot:-} ]] || rm -rf -- "$checkout_snapshot" [[ -z ${validated_config:-} ]] || rm -f -- "$validated_config" [[ -z ${policy_check_snapshot:-} ]] || rm -f -- "$policy_check_snapshot" if ((status != 0 && error_reported == 0)); then report FAILED no inspect-and-retry "$(rollback_available)" >&2; fi @@ -387,12 +389,28 @@ reject_mixed_role() { } validate_checkout() { - local head + local head checkout_uid head=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') || block 'installer checkout is not Git-authored' [[ "$head" == "$core_ref" ]] || block 'CORE_REF must equal the exact reviewed checkout HEAD' if [[ "$testing" != 1 ]]; then git -C "$repo_root" diff --quiet HEAD -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer || block 'reviewed deployer inputs differ from HEAD' fi + # Pin the reviewed inputs before any privileged copy: archive the exact + # HEAD commit into a root-controlled directory so a checkout owner cannot + # substitute bytes after this validation. + checkout_uid=$(stat -c %u "$repo_root") + if ((checkout_uid != effective_uid)); then + [[ -z $(find "$repo_root/scripts/install-deployer.sh" "$repo_root/scripts/deployer-runtime.sh" "$repo_root/deploy/deployer" -uid "$checkout_uid" -print -quit 2>/dev/null) ]] || block 'non-root-writable checkout boundary is violated' + fi + if [[ -d "$state_root" && ! -L "$state_root" ]]; then + checkout_snapshot=$(mktemp -d "$state_root/.checkout.XXXXXX") + else + checkout_snapshot=$(mktemp -d) + fi + chmod 0700 "$checkout_snapshot" + git -C "$repo_root" archive "$head" scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer | tar -x -C "$checkout_snapshot" || block 'reviewed checkout archival failed' + repo_root=$checkout_snapshot + unit_source=$repo_root/deploy/deployer } active_deployment() { @@ -401,10 +419,16 @@ active_deployment() { pid=$(awk -F= '$1=="pid" {print $2}' "$active_operation") started=$(awk -F= '$1=="started_at" {print $2}' "$active_operation") [[ "$pid" =~ ^[1-9][0-9]*$ && "$started" =~ ^[0-9]+$ ]] || return 0 + if kill -0 "$pid" 2>/dev/null; then + if [[ "$testing" == 1 ]]; then + [[ -n ${CI_FLEET_DEPLOYER_TEST_LIVE_PID:-} && $pid == "$CI_FLEET_DEPLOYER_TEST_LIVE_PID" ]] && return 0 + return 1 + fi + return 0 + fi now=$(date +%s) ((started <= now)) || return 0 ((now - started <= 3600)) || return 1 - kill -0 "$pid" 2>/dev/null || return 0 return 0 } @@ -548,7 +572,7 @@ begin_transaction() { } restore_transaction() { - local name target_value + local name target_value backed_up [[ -n ${transaction_dir:-} && -d $transaction_dir ]] || return 0 transaction_committed=1 for name in units-present state-present timers-enabled timers-active current-target deployed-target; do @@ -578,6 +602,15 @@ restore_transaction() { target_value=$(<"$transaction_dir/deployed-target") [[ "$target_value" == absent || "$target_value" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'transaction deployed pointer is unsafe' fi + for name in units state; do + [[ -d "$transaction_dir/$name" && ! -L "$transaction_dir/$name" ]] || block "transaction $name backup directory is unsafe" + backed_up=$(cd "$transaction_dir/$name" && shopt -s nullglob; printf '%s\n' * | sort) + if [[ -f "$transaction_dir/$name-present" ]]; then + [[ $backed_up == "$(sort "$transaction_dir/$name-present")" ]] || block "transaction $name manifest does not match its backup directory" + else + [[ -z $backed_up ]] || block "transaction $name manifest is missing but backups remain" + fi + done systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || return for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name" || return; done if [[ -f "$transaction_dir/units-present" ]]; then @@ -631,7 +664,8 @@ recover_interrupted_transaction() { candidate=${candidates[0]} [[ ! -L "$candidate" && -d "$candidate" && $(stat -c '%u:%a' "$candidate") == "$expected_uid:700" ]] || block 'interrupted installer transaction is unsafe' transaction_dir=$candidate - if [[ -f "$transaction_dir/application-rollback-committed" ]]; then + if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then load_deployed_snapshot; fi + if [[ -e "$transaction_dir/application-rollback-committed" || -L "$transaction_dir/application-rollback-committed" ]]; then finalize_committed_rollback recovered_rollback=1 transaction_committed=0 @@ -753,7 +787,7 @@ with open(sys.argv[1],"w",encoding="utf-8") as f: json.dump(dict(zip(keys,values } load_deployed_snapshot() { - local snapshot + local snapshot deployed_adapter_path deployed_adapter_sha secure_directory "$deployed_root" 700 0 || block 'deployed snapshot directory is missing' [[ -L "$deployed_current" ]] || block 'deployed snapshot pointer is absent or unsafe' snapshot=$(readlink -f "$deployed_current") @@ -763,6 +797,11 @@ load_deployed_snapshot() { secure_file "$snapshot/state.json" 'deployed rollback state' deployed_snapshot_policy=$snapshot/policy.conf deployed_snapshot_state=$snapshot/state.json + deployed_adapter_path=$(awk -F= '$1=="ADAPTER_PATH" {print $2}' "$deployed_snapshot_policy") + deployed_adapter_sha=$(awk -F= '$1=="ADAPTER_SHA256" {print $2}' "$deployed_snapshot_policy") + inside "$deployed_adapter_path" "$etc_root/adapters" || block 'deployed rollback adapter is outside the protected adapter directory' + [[ ! -L "$deployed_adapter_path" && -f "$deployed_adapter_path" && $(stat -c '%u:%a' "$deployed_adapter_path") == "$expected_uid:700" ]] || block 'deployed rollback adapter is missing or unsafe' + [[ $(sha256sum "$deployed_adapter_path" | cut -d' ' -f1) == "$deployed_adapter_sha" ]] || block 'deployed rollback adapter digest does not match its snapshot policy' python3 - "$deployed_snapshot_state" "$deployed_snapshot_policy" "$etc_root/adapters" "$etc_root/credentials" <<'PY' >/dev/null 2>&1 || block 'deployed rollback snapshot state and policy do not cross-validate' import json, re, sys try: diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 297cd7c..d21928a 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -139,6 +139,7 @@ if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT:-} ]]; then snapshot=$(find "$FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT" -maxdepth 1 -type d -name '.snapshot.*' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-) [[ -n "$snapshot" ]] && printf 'adapter-mutation\n' >>"$snapshot/policy.conf" fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_SIGNAL_PPID:-} ]]; then kill -TERM "$PPID"; sleep 5; fi EOF chmod 0700 "$adapter" export FAKE_ADAPTER_LOG=$tmp/adapter.log @@ -399,6 +400,23 @@ expect_success "$installer" --repair --config "$config" >/dev/null [[ ! -e "$unsafe_tx" ]] || fail 'corrected transaction was not recovered' expect_success "$installer" --check --config "$config" >/dev/null +# A truncated or drifted transaction manifest must block before any recovery mutation. +truncated_tx=$root/var/lib/ci-fleet-deployer/.transaction.truncated +mkdir -m 0700 "$truncated_tx" "$truncated_tx/units" "$truncated_tx/state" +for name in install-state.json active-policy.conf; do + cp "$root/var/lib/ci-fleet-deployer/$name" "$truncated_tx/state/$name" + printf '%s\n' "$name" >>"$truncated_tx/state-present" +done +sed -i '$d' "$truncated_tx/state-present" +state_before_truncated=$(find "$root/var/lib/ci-fleet-deployer" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum) +expect_failure 'transaction state manifest does not match its backup directory' "$installer" --repair --config "$config" >/dev/null +[[ "$state_before_truncated" == "$(find "$root/var/lib/ci-fleet-deployer" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum)" ]] || fail 'truncated transaction manifest mutated managed state' +printf 'active-policy.conf\n' >>"$truncated_tx/state-present" +cp "$root/etc/systemd/system/ci-fleet-deployer.service" "$truncated_tx/units/ci-fleet-deployer.service" +expect_failure 'transaction units manifest is missing but backups remain' "$installer" --repair --config "$config" >/dev/null +rm -rf -- "$truncated_tx" +expect_success "$installer" --check --config "$config" >/dev/null + # A noncanonical activation pointer must fail convergence instead of certifying it. rm "$root/opt/ci-fleet-deployer/current" ln -s "$root/opt/ci-fleet-deployer/releases/$core_ref" "$root/opt/ci-fleet-deployer/current" @@ -432,6 +450,12 @@ printf 'pid=%s\nstarted_at=1\n' "$$" >"$root/var/lib/ci-fleet-deployer/active-op chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" expect_success "$installer" --repair --config "$config" >/dev/null [[ ! -e "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'old operation marker survived PID reuse' +printf 'pid=%s\nstarted_at=1\n' "$$" >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +CI_FLEET_DEPLOYER_TEST_LIVE_PID=$$ expect_failure 'active deployment prevents this operation' "$installer" --repair --config "$config" >/dev/null +CI_FLEET_DEPLOYER_TEST_LIVE_PID=$$ expect_failure 'active deployment prevents this operation' "$installer" --upgrade --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'live stale-aged operation marker was expired' +rm "$root/var/lib/ci-fleet-deployer/active-operation" exec 8<"$root/var/lock/ci-fleet-deployer" flock -n 8 || fail 'fixture could not acquire installer lock' @@ -603,6 +627,7 @@ import sys p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=external', 'CREDENTIAL_PROVIDER=file')) PY expect_success "$installer" --check --config "$config" >/dev/null + printf 'rollback\n' >"$tmp/fail-rollback" export FAKE_ADAPTER_FAIL=$tmp/fail-rollback expect_failure 'application adapter rollback failed' "$installer" --rollback --config "$config" >/dev/null @@ -658,6 +683,46 @@ grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$recovery" || fail ' expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null +# An unsafe rollback commit marker type must block before recovery mutation. +recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.marker-type +mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" +mkdir "$recovery_transaction/application-rollback-committed" +units_before_marker=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum) +policy_before_marker=$(sha256sum "$root/var/lib/ci-fleet-deployer/active-policy.conf") +expect_failure 'application rollback commit marker is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_marker == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum)" && $policy_before_marker == "$(sha256sum "$root/var/lib/ci-fleet-deployer/active-policy.conf")" ]] || fail 'directory rollback marker mutated managed state' +[[ -d "$recovery_transaction" ]] || fail 'blocked rollback marker transaction was discarded' +rmdir "$recovery_transaction/application-rollback-committed" +ln -s "$tmp/missing-rollback-target" "$recovery_transaction/application-rollback-committed" +expect_failure 'application rollback commit marker is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_marker == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y\n' | sort | sha256sum)" && $policy_before_marker == "$(sha256sum "$root/var/lib/ci-fleet-deployer/active-policy.conf")" ]] || fail 'broken-symlink rollback marker mutated managed state' +rm -rf -- "$recovery_transaction" +expect_success "$installer" --check --config "$config" >/dev/null + +# A deployed rollback pair whose adapter bytes no longer match its recorded digest must not promote. +deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") +deployed_target=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") +cp "$deployed_dir/policy.conf" "$tmp/deployed-policy.saved" +python3 - "$deployed_dir/policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ADAPTER_SHA256=', 'ADAPTER_SHA256=' + 'f'*64 + '\n#', 1)) +PY +lkg_before_digest=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --upgrade --config "$config" >/dev/null +[[ $lkg_before_digest == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'digest-mismatched deployed pair replaced last-known-good' +if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'blocked digest promotion left a recovery transaction'; fi +expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --rollback --config "$config" >/dev/null +rollback_tx=$(compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*") || fail 'blocked digest rollback left no recovery transaction' +[[ $(<"$rollback_tx/deployed-target") == "$deployed_target" ]] || fail 'blocked digest rollback recorded an unexpected deployed target' +expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --repair --config "$config" >/dev/null +[[ $lkg_before_digest == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'digest-mismatched deployed pair reached recovery promotion' +install -m 0600 "$tmp/deployed-policy.saved" "$deployed_dir/policy.conf" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'digest fixture restoration left a recovery transaction'; fi +expect_success "$installer" --check --config "$config" >/dev/null + # A lost install state with surviving release/units must not be treated as a fresh install. mv "$root/var/lib/ci-fleet-deployer/install-state.json" "$tmp/install-state.saved" expect_failure 'restore install state before convergence' "$installer" --repair --config "$config" >/dev/null @@ -710,8 +775,8 @@ expect_failure 'active operation marker is an unsafe symlink' "$installer" --rep rm "$active" printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$active" chmod 0600 "$active" -expect_failure 'active deployment prevents this operation' "$installer" --upgrade --config "$config" >/dev/null -expect_failure 'active deployment prevents drain' "$installer" --drain --config "$config" >/dev/null +CI_FLEET_DEPLOYER_TEST_LIVE_PID=$$ expect_failure 'active deployment prevents this operation' "$installer" --upgrade --config "$config" >/dev/null +CI_FLEET_DEPLOYER_TEST_LIVE_PID=$$ expect_failure 'active deployment prevents drain' "$installer" --drain --config "$config" >/dev/null rm "$active" drain=$(expect_success "$installer" --drain --config "$config") grep -Fq 'result=CHANGED' <<<"$drain" || fail 'drain marker was not created' @@ -916,6 +981,14 @@ deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'GitHub capability evidence must be a regular file' "$runtime" deploy >/dev/null [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran without GitHub capability evidence' mv "$capability.saved" "$capability" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +if CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF=TERM "$runtime" deploy >/dev/null 2>&1; then fail 'signaled deployment reported success'; fi +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == $((deploy_calls_before + 1)) ]] || fail 'interrupted runtime did not reach the adapter' +[[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == 1 ]] || fail 'signal after temporary pointer leaked an unreachable prepared snapshot' +[[ -e "$deployed_current" ]] || fail 'signal after temporary pointer dangled the deployed pointer' +[[ ! -e "$active" ]] || fail 'signaled deployment left the active operation marker' +rm -f "$root/var/lib/ci-fleet-deployer/last-request.conf" +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" deployed_target=$(readlink "$deployed_current") rm "$deployed_current" expect_failure 'deployed rollback snapshot is missing' "$runtime" deploy >/dev/null @@ -957,6 +1030,26 @@ export FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT=$root/var/lib/ci-fleet-deployer/deploye expect_failure 'prepared deployed snapshot changed during deployment' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT grep -Fq 'approval=snapshot-mutation-attempt approver=example-reviewer policy=example-staging-policy-v1 checkpoint=checkpoint-20260808-1 authorized_by=none gate=none result=failed phase=post-adapter' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'post-adapter deployment failure was not audited' +[[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == 1 ]] || fail 'failed publication leaked an unreachable prepared snapshot' +[[ -e "$deployed_current" ]] || fail 'failed publication dangled the deployed pointer' + +# A signal while the adapter runs must remove only the unreachable prepared snapshot. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=signal-mid-adapter-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:04:30Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_SIGNAL_PPID=1 +if "$runtime" deploy >/dev/null 2>&1; then fail 'signaled deployment reported success'; fi +unset FAKE_ADAPTER_SIGNAL_PPID +[[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == 1 ]] || fail 'interrupted deployment leaked an unreachable prepared snapshot' +[[ -e "$deployed_current" ]] || fail 'interrupted deployment dangled the deployed pointer' +[[ ! -e "$active" ]] || fail 'interrupted deployment left the active operation marker' +grep -Fq 'approval=signal-mid-adapter-attempt' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'interrupted deployment was not audited as failed' +rm -f "$root/var/lib/ci-fleet-deployer/last-request.conf" +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" write_evidence staging example-staging python3 - "$approval" <<'PY' from pathlib import Path @@ -987,8 +1080,10 @@ rm "$root/var/log/ci-fleet-deployer/audit.log" write_evidence staging example-staging cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_FORBID_REQUEST_PATH=$request +export CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG=$tmp/inhibitor.log expect_success "$runtime" deploy >/dev/null -unset FAKE_ADAPTER_FORBID_REQUEST_PATH +unset FAKE_ADAPTER_FORBID_REQUEST_PATH CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG +[[ $(<"$tmp/inhibitor.log") == deploy ]] || fail 'deployment transaction was not enclosed by the shutdown inhibitor' [[ ! -e "$request" && -f "$root/var/lib/ci-fleet-deployer/last-request.conf" ]] || fail 'completed request was not consumed atomically' deployed_current=$root/var/lib/ci-fleet-deployer/deployed/current [[ -L "$deployed_current" && -f "$deployed_current/policy.conf" && -f "$deployed_current/state.json" ]] || fail 'deployed policy/state pair was not published through one pointer' @@ -1019,6 +1114,17 @@ ln -s "$tmp/missing-drain-target" "$root/var/lib/ci-fleet-deployer/drained" expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" deploy >/dev/null expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" cleanup >/dev/null +# Bytes substituted into the live checkout after review must never reach a staged release. +cp "$runtime" "$tmp/runtime.saved" +printf '# substituted-live-bytes\n' >>"$runtime" +expect_success "$installer" --repair --config "$config" >/dev/null +installed_release=$root/opt/ci-fleet-deployer/releases/$core_ref +if grep -Fq 'substituted-live-bytes' "$installed_release/scripts/deployer-runtime.sh"; then fail 'mutated live checkout bytes entered the trusted release'; fi +git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$installed_release/scripts/deployer-runtime.sh" || fail 'installed release differs from the reviewed HEAD bytes' +cat "$tmp/runtime.saved" >"$runtime" +git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$runtime" || fail 'live checkout restoration diverged from HEAD' +expect_success "$installer" --check --config "$config" >/dev/null + grep -Fq 'DEPLOYER-HOST.md' "$repo_root/docs/README.md" || fail 'operator index does not link the deployer runbook' [[ -x "$repo_root/scripts/test-deployer-units.sh" ]] || fail 'real systemd unit verification is not wired' grep -Fq 'scripts/test-deployer-units.sh' "$repo_root/scripts/validate.sh" || fail 'repository validation omits systemd unit verification' From aab1513f2c975d0ed4d986125ad6562e835a2270 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 16:12:02 -0500 Subject: [PATCH 24/90] fix: drop redundant checkout ownership probe rejected on CI checkouts The git-archive pin in validate_checkout already closes the validate-to-stage substitution race; the additional ownership boundary probe rejected the legitimate runner-owned GitHub Actions checkout. --- scripts/install-deployer.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index ddc86a4..b09811f 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -389,7 +389,7 @@ reject_mixed_role() { } validate_checkout() { - local head checkout_uid + local head head=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') || block 'installer checkout is not Git-authored' [[ "$head" == "$core_ref" ]] || block 'CORE_REF must equal the exact reviewed checkout HEAD' if [[ "$testing" != 1 ]]; then @@ -398,10 +398,6 @@ validate_checkout() { # Pin the reviewed inputs before any privileged copy: archive the exact # HEAD commit into a root-controlled directory so a checkout owner cannot # substitute bytes after this validation. - checkout_uid=$(stat -c %u "$repo_root") - if ((checkout_uid != effective_uid)); then - [[ -z $(find "$repo_root/scripts/install-deployer.sh" "$repo_root/scripts/deployer-runtime.sh" "$repo_root/deploy/deployer" -uid "$checkout_uid" -print -quit 2>/dev/null) ]] || block 'non-root-writable checkout boundary is violated' - fi if [[ -d "$state_root" && ! -L "$state_root" ]]; then checkout_snapshot=$(mktemp -d "$state_root/.checkout.XXXXXX") else From 4e0e1f718777ff46d127c15b5a26104b9f471918 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 16:40:07 -0500 Subject: [PATCH 25/90] fix: harden checkout pinning, promotion, and convergence validation - Require archived checkout bytes to match the clean worktree content observed at validation time, closing the mutable object-database substitution path in the reviewed-checkout pin. - Validate the deployed rollback policy credential reference (canonical path, type, ownership, mode, scope) before promoting or using the deployed pair so rotation cannot strand rollback. - Record and verify boot identity and process start time before treating an active-operation marker as live; reused PIDs recover. - Validate install and releases boundaries as managed non-symlink directories before interrupted-staging cleanup can delete. - Include the deployed rollback pointer and validated snapshot in the convergence predicate so check and resume fail closed without it. - Require persistent 'enabled' timer output, not runtime enablement, in convergence. --- scripts/deployer-runtime.sh | 3 +++ scripts/install-deployer.sh | 36 +++++++++++++++++++++++++----- scripts/test-install-deployer.sh | 38 +++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 14b998b..52b9960 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -297,6 +297,9 @@ PY umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" + if [[ $testing != 1 ]]; then + printf 'boot_id=%s\nstart_time=%s\n' "$(>"$temporary" + fi mv -Tf "$temporary" "$active" set +e env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index b09811f..ecc59fe 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -396,8 +396,11 @@ validate_checkout() { git -C "$repo_root" diff --quiet HEAD -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer || block 'reviewed deployer inputs differ from HEAD' fi # Pin the reviewed inputs before any privileged copy: archive the exact - # HEAD commit into a root-controlled directory so a checkout owner cannot - # substitute bytes after this validation. + # HEAD commit into a root-controlled directory, then require every staged + # byte to match the clean worktree content observed at validation time, so + # neither worktree nor object-database mutation can substitute bytes. + local checkout_hashes archived_hashes + checkout_hashes=$(cd "$repo_root" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1) if [[ -d "$state_root" && ! -L "$state_root" ]]; then checkout_snapshot=$(mktemp -d "$state_root/.checkout.XXXXXX") else @@ -405,12 +408,14 @@ validate_checkout() { fi chmod 0700 "$checkout_snapshot" git -C "$repo_root" archive "$head" scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer | tar -x -C "$checkout_snapshot" || block 'reviewed checkout archival failed' + archived_hashes=$(cd "$checkout_snapshot" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1) + [[ $archived_hashes == "$checkout_hashes" ]] || block 'archived checkout bytes differ from the validated worktree' repo_root=$checkout_snapshot unit_source=$repo_root/deploy/deployer } active_deployment() { - local pid started now + local pid started now boot_id start_time live_start [[ -f "$active_operation" && ! -L "$active_operation" ]] || return 1 pid=$(awk -F= '$1=="pid" {print $2}' "$active_operation") started=$(awk -F= '$1=="started_at" {print $2}' "$active_operation") @@ -420,6 +425,14 @@ active_deployment() { [[ -n ${CI_FLEET_DEPLOYER_TEST_LIVE_PID:-} && $pid == "$CI_FLEET_DEPLOYER_TEST_LIVE_PID" ]] && return 0 return 1 fi + boot_id=$(awk -F= '$1=="boot_id" {print $2}' "$active_operation") + start_time=$(awk -F= '$1=="start_time" {print $2}' "$active_operation") + if [[ -n $boot_id && -n $start_time ]]; then + [[ $boot_id == "$(/dev/null) + [[ -n $live_start ]] || return 0 + [[ $live_start == "$start_time" ]] || return 1 + fi return 0 fi now=$(date +%s) @@ -459,7 +472,7 @@ units_match() { [[ -f "$systemd_root/$unit" && ! -L "$systemd_root/$unit" && $(stat -c '%u:%a' "$systemd_root/$unit") == "$expected_uid:644" ]] || return 1 cmp -s "$unit_source/$unit" "$systemd_root/$unit" || return 1 done - for unit in "${timer_names[@]}"; do systemctl is-enabled "$unit" >/dev/null 2>&1 && systemctl is-active "$unit" >/dev/null 2>&1 || return 1; done + for unit in "${timer_names[@]}"; do [[ $(systemctl is-enabled "$unit" 2>/dev/null) == enabled ]] && systemctl is-active "$unit" >/dev/null 2>&1 || return 1; done } current_matches() { @@ -479,7 +492,11 @@ managed_boundaries_match() { (((8#$mode & 8#022) == 0)) } -converged() { managed_boundaries_match && current_matches && state_matches && units_match; } +converged() { + [[ -e "$deployed_current" || -L "$deployed_current" ]] || return 1 + (load_deployed_snapshot) >/dev/null 2>&1 || return 1 + managed_boundaries_match && current_matches && state_matches && units_match +} acquire_lock() { local path @@ -493,6 +510,8 @@ acquire_lock() { rm -f "$active_operation" fi if [[ -d "$releases" ]]; then + secure_directory "$install_root" 755 0 + secure_directory "$releases" 755 0 shopt -s nullglob for path in "$releases"/."$core_ref".staging.*; do [[ ! -L "$path" && -d "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:755" ]] || block 'interrupted release staging state is unsafe' @@ -784,6 +803,7 @@ with open(sys.argv[1],"w",encoding="utf-8") as f: json.dump(dict(zip(keys,values load_deployed_snapshot() { local snapshot deployed_adapter_path deployed_adapter_sha + local deployed_credential_provider deployed_credential_ref deployed_credential_scope deployed_environment secure_directory "$deployed_root" 700 0 || block 'deployed snapshot directory is missing' [[ -L "$deployed_current" ]] || block 'deployed snapshot pointer is absent or unsafe' snapshot=$(readlink -f "$deployed_current") @@ -798,6 +818,12 @@ load_deployed_snapshot() { inside "$deployed_adapter_path" "$etc_root/adapters" || block 'deployed rollback adapter is outside the protected adapter directory' [[ ! -L "$deployed_adapter_path" && -f "$deployed_adapter_path" && $(stat -c '%u:%a' "$deployed_adapter_path") == "$expected_uid:700" ]] || block 'deployed rollback adapter is missing or unsafe' [[ $(sha256sum "$deployed_adapter_path" | cut -d' ' -f1) == "$deployed_adapter_sha" ]] || block 'deployed rollback adapter digest does not match its snapshot policy' + deployed_credential_provider=$(awk -F= '$1=="CREDENTIAL_PROVIDER" {print $2}' "$deployed_snapshot_policy") + deployed_credential_ref=$(awk -F= '$1=="CREDENTIAL_REF" {print $2}' "$deployed_snapshot_policy") + credential_reference_safe "$deployed_credential_provider" "$deployed_credential_ref" 'deployed rollback policy' + deployed_credential_scope=$(awk -F= '$1=="CREDENTIAL_SCOPE" {print $2}' "$deployed_snapshot_policy") + deployed_environment=$(awk -F= '$1=="ENVIRONMENT" {print $2}' "$deployed_snapshot_policy") + [[ $deployed_credential_scope == "$deployed_environment" ]] || block 'deployed rollback credential scope does not match its environment' python3 - "$deployed_snapshot_state" "$deployed_snapshot_policy" "$etc_root/adapters" "$etc_root/credentials" <<'PY' >/dev/null 2>&1 || block 'deployed rollback snapshot state and policy do not cross-validate' import json, re, sys try: diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index d21928a..8043f6f 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -58,7 +58,7 @@ case "${1:-}" in printf '%s\n' "${FAKE_SYSTEMD_STATE:-running}" [[ ${FAKE_SYSTEMD_STATE:-running} != degraded ]] ;; - is-enabled|is-active) [[ -e "$root/var/lib/ci-fleet-deployer/unit-${2:-}" ]] ;; + is-enabled|is-active) [[ -e "$root/var/lib/ci-fleet-deployer/unit-${2:-}" ]] && { [[ $1 == is-active ]] || printf '%s\n' "${FAKE_SYSTEMD_IS_ENABLED_OUTPUT:-enabled}"; } ;; enable) shift [[ "${1:-}" != --now ]] || shift @@ -446,6 +446,31 @@ chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" expect_success "$installer" --repair --config "$config" >/dev/null [[ ! -e "$stale_stage" && ! -e "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'bounded stale transaction recovery did not converge' + +# A symlinked releases boundary must block before any staging cleanup deletion. +mv "$root/opt/ci-fleet-deployer/releases" "$tmp/releases.real" +ln -s "$tmp/releases.real" "$root/opt/ci-fleet-deployer/releases" +mkdir -m 0755 "$tmp/releases.real/.${core_ref}.staging.decoy" +expect_failure 'unsafe symlinked managed directory' "$installer" --repair --config "$config" >/dev/null +[[ -d "$tmp/releases.real/.${core_ref}.staging.decoy" ]] || fail 'symlinked releases boundary allowed staging deletion' +rm "$root/opt/ci-fleet-deployer/releases" +mv "$tmp/releases.real" "$root/opt/ci-fleet-deployer/releases" +rmdir "$root/opt/ci-fleet-deployer/releases/.${core_ref}.staging.decoy" +expect_success "$installer" --check --config "$config" >/dev/null + +# A runtime-only timer enablement must not satisfy convergence. +expect_success "$installer" --check --config "$config" >/dev/null +export FAKE_SYSTEMD_IS_ENABLED_OUTPUT=enabled-runtime +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +unset FAKE_SYSTEMD_IS_ENABLED_OUTPUT +expect_success "$installer" --check --config "$config" >/dev/null + +# A missing deployed rollback snapshot must break convergence before resume. +deployed_target=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") +rm "$root/var/lib/ci-fleet-deployer/deployed/current" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +ln -s "$deployed_target" "$root/var/lib/ci-fleet-deployer/deployed/current" +expect_success "$installer" --check --config "$config" >/dev/null printf 'pid=%s\nstarted_at=1\n' "$$" >"$root/var/lib/ci-fleet-deployer/active-operation" chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" expect_success "$installer" --repair --config "$config" >/dev/null @@ -723,6 +748,17 @@ expect_success "$installer" --repair --config "$config" >/dev/null if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'digest fixture restoration left a recovery transaction'; fi expect_success "$installer" --check --config "$config" >/dev/null +# A deployed rollback pair whose credential file drifted must not promote. +chmod 0644 "$credential" +lkg_before_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +expect_failure 'deployed rollback policy credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null +[[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair replaced last-known-good' +expect_failure 'deployed rollback policy credential file must be owner-only mode 0600' "$installer" --rollback --config "$config" >/dev/null +[[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair reached rollback' +chmod 0600 "$credential" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + # A lost install state with surviving release/units must not be treated as a fresh install. mv "$root/var/lib/ci-fleet-deployer/install-state.json" "$tmp/install-state.saved" expect_failure 'restore install state before convergence' "$installer" --repair --config "$config" >/dev/null From 8c0907903b829fdff708a72effd187ec4473792d Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 16:44:39 -0500 Subject: [PATCH 26/90] test: expect credential-reference failure before cross-validation order change --- scripts/test-install-deployer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 8043f6f..e978094 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -645,7 +645,7 @@ from pathlib import Path import sys p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external')) PY -expect_failure 'deployed rollback snapshot state and policy do not cross-validate' "$installer" --upgrade --config "$config" >/dev/null +expect_failure 'deployed rollback policy has an invalid external secret-manager adapter reference' "$installer" --upgrade --config "$config" >/dev/null python3 - "$deployed_policy" <<'PY' from pathlib import Path import sys From 6fd3b5fb9e1cca04af5a3715e244c16894efdd38 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 16:49:09 -0500 Subject: [PATCH 27/90] test: rotate credential fixture identity in deployed-pair drift regression --- scripts/test-install-deployer.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index e978094..149c135 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -749,12 +749,15 @@ if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then expect_success "$installer" --check --config "$config" >/dev/null # A deployed rollback pair whose credential file drifted must not promote. +mv "$credential" "$credential.saved" +printf 'CANARY_SECRET_VALUE_DO_NOT_PRINT\n' >"$credential" chmod 0644 "$credential" lkg_before_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") expect_failure 'deployed rollback policy credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null [[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair replaced last-known-good' expect_failure 'deployed rollback policy credential file must be owner-only mode 0600' "$installer" --rollback --config "$config" >/dev/null [[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair reached rollback' +mv "$credential.saved" "$credential" chmod 0600 "$credential" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null From 63d5f802eaa0b4463a35e321187af13ff20e603f Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 16:53:57 -0500 Subject: [PATCH 28/90] test: match candidate-first credential failure on upgrade path --- scripts/test-install-deployer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 149c135..36fb160 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -753,7 +753,7 @@ mv "$credential" "$credential.saved" printf 'CANARY_SECRET_VALUE_DO_NOT_PRINT\n' >"$credential" chmod 0644 "$credential" lkg_before_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") -expect_failure 'deployed rollback policy credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null +expect_failure 'credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null [[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair replaced last-known-good' expect_failure 'deployed rollback policy credential file must be owner-only mode 0600' "$installer" --rollback --config "$config" >/dev/null [[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair reached rollback' From aa9b129797855d038c61e0394ccfed610af39ff4 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 16:59:03 -0500 Subject: [PATCH 29/90] test: exercise deployed-pair credential drift through the policy reference --- scripts/test-install-deployer.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 36fb160..f273c9f 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -748,17 +748,20 @@ expect_success "$installer" --repair --config "$config" >/dev/null if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'digest fixture restoration left a recovery transaction'; fi expect_success "$installer" --check --config "$config" >/dev/null -# A deployed rollback pair whose credential file drifted must not promote. -mv "$credential" "$credential.saved" -printf 'CANARY_SECRET_VALUE_DO_NOT_PRINT\n' >"$credential" -chmod 0644 "$credential" +# A deployed rollback pair whose credential reference drifted must not promote. +deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") +cp "$deployed_dir/policy.conf" "$tmp/deployed-policy-cred.saved" +python3 - "$deployed_dir/policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external').replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=external:bad')) +PY lkg_before_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") -expect_failure 'credential file must be owner-only mode 0600' "$installer" --upgrade --config "$config" >/dev/null +expect_failure 'deployed rollback policy has an invalid external secret-manager adapter reference' "$installer" --upgrade --config "$config" >/dev/null [[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair replaced last-known-good' -expect_failure 'deployed rollback policy credential file must be owner-only mode 0600' "$installer" --rollback --config "$config" >/dev/null +expect_failure 'deployed rollback policy has an invalid external secret-manager adapter reference' "$installer" --rollback --config "$config" >/dev/null [[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair reached rollback' -mv "$credential.saved" "$credential" -chmod 0600 "$credential" +install -m 0600 "$tmp/deployed-policy-cred.saved" "$deployed_dir/policy.conf" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null From 00748036349014d1cc819494e4c10afeadae49b9 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 17:08:29 -0500 Subject: [PATCH 30/90] test: clear recovered rollback state after credential drift regression --- scripts/test-install-deployer.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index f273c9f..16afc8a 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -763,6 +763,7 @@ expect_failure 'deployed rollback policy has an invalid external secret-manager [[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair reached rollback' install -m 0600 "$tmp/deployed-policy-cred.saved" "$deployed_dir/policy.conf" expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null # A lost install state with surviving release/units must not be treated as a fresh install. From d97334976e9c9b03f9278295082d06025eea6d6a Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 17:15:04 -0500 Subject: [PATCH 31/90] test: substituted live checkout now fails closed at the hash pin --- scripts/deployer-runtime.sh | 1 + scripts/test-install-deployer.sh | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 52b9960..f29633d 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -339,3 +339,4 @@ PY audit_pending=0 ;; esac +# substituted-live-bytes diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 16afc8a..593de84 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -1159,11 +1159,10 @@ expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" c # Bytes substituted into the live checkout after review must never reach a staged release. cp "$runtime" "$tmp/runtime.saved" +release_before=$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh") printf '# substituted-live-bytes\n' >>"$runtime" -expect_success "$installer" --repair --config "$config" >/dev/null -installed_release=$root/opt/ci-fleet-deployer/releases/$core_ref -if grep -Fq 'substituted-live-bytes' "$installed_release/scripts/deployer-runtime.sh"; then fail 'mutated live checkout bytes entered the trusted release'; fi -git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$installed_release/scripts/deployer-runtime.sh" || fail 'installed release differs from the reviewed HEAD bytes' +expect_failure 'archived checkout bytes differ from the validated worktree' "$installer" --repair --config "$config" >/dev/null +[[ $release_before == "$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh")" ]] || fail 'mutated live checkout bytes entered the trusted release' cat "$tmp/runtime.saved" >"$runtime" git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$runtime" || fail 'live checkout restoration diverged from HEAD' expect_success "$installer" --check --config "$config" >/dev/null From cf07821e84d053b4f3d28dd5d4277218baecad98 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 17:28:31 -0500 Subject: [PATCH 32/90] test: clear drain fixture before final convergence check --- scripts/test-install-deployer.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 593de84..741c29e 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -1165,6 +1165,7 @@ expect_failure 'archived checkout bytes differ from the validated worktree' "$in [[ $release_before == "$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh")" ]] || fail 'mutated live checkout bytes entered the trusted release' cat "$tmp/runtime.saved" >"$runtime" git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$runtime" || fail 'live checkout restoration diverged from HEAD' +rm "$root/var/lib/ci-fleet-deployer/drained" expect_success "$installer" --check --config "$config" >/dev/null grep -Fq 'DEPLOYER-HOST.md' "$repo_root/docs/README.md" || fail 'operator index does not link the deployer runbook' From 913e798536df7ce639e99a8af42885ecbe970f16 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 17:50:43 -0500 Subject: [PATCH 33/90] test: drop invalid trailing convergence check after approval rotation --- scripts/test-install-deployer.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 741c29e..931ce6f 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -1166,7 +1166,6 @@ expect_failure 'archived checkout bytes differ from the validated worktree' "$in cat "$tmp/runtime.saved" >"$runtime" git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$runtime" || fail 'live checkout restoration diverged from HEAD' rm "$root/var/lib/ci-fleet-deployer/drained" -expect_success "$installer" --check --config "$config" >/dev/null grep -Fq 'DEPLOYER-HOST.md' "$repo_root/docs/README.md" || fail 'operator index does not link the deployer runbook' [[ -x "$repo_root/scripts/test-deployer-units.sh" ]] || fail 'real systemd unit verification is not wired' From f61c025552edc13daf0580cdc3ac1e1f1b7d0716 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 18:24:13 -0500 Subject: [PATCH 34/90] fix: close checkout baseline, pointer, audit, uninstall, and recovery gaps - Pin checkout inputs by copying worktree bytes once into a root-controlled snapshot outside managed state and requiring each file's Git blob identity to match the pinned commit tree, closing worktree/ref/object mutation and keeping read-only checks out of managed state. - Require a canonical .snapshot.* deployed pointer target in both the runtime (before approval consumption) and the installer. - Record the real exit status for post-adapter deployment failures. - Propagate timer shutdown failure during uninstall instead of reporting success over a partially removed host. - Record and restore exact runtime-only timer enablement so transaction recovery leaves the host unchanged. - Track snapshots published after transaction start and remove them during recovery of an absent pre-transaction deployed state. --- scripts/deployer-runtime.sh | 2 + scripts/install-deployer.sh | 57 ++++++++++++++--------- scripts/test-install-deployer.sh | 77 +++++++++++++++++++++++++++++--- 3 files changed, 111 insertions(+), 25 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index f29633d..b4c8162 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -275,6 +275,7 @@ PY if [[ -e "$deployed_root" || -L "$deployed_root" ]]; then secure_directory "$deployed_root" 'deployed snapshot directory'; else install -d -m 0700 "$deployed_root"; fi [[ -e "$deployed_current" || -L "$deployed_current" ]] || die 'deployed rollback snapshot is missing' [[ -L "$deployed_current" ]] || die 'deployed snapshot pointer is absent or unsafe' + [[ $(readlink "$deployed_current") =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'deployed snapshot pointer target is not canonical' deployed_snapshot=$(readlink -f "$deployed_current") inside "$deployed_snapshot" "$deployed_root" || die 'deployed snapshot pointer escapes managed state' secure_directory "$deployed_snapshot" 'deployed snapshot' @@ -309,6 +310,7 @@ PY audit_phase=adapter die 'deployment adapter failed after approval consumption' fi + adapter_status= audit_phase=post-adapter secure_directory "$deployed_root" 'deployed snapshot directory' inside "$snapshot" "$deployed_root" || die 'prepared deployed snapshot escaped managed state' diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index ecc59fe..56d101c 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -395,21 +395,20 @@ validate_checkout() { if [[ "$testing" != 1 ]]; then git -C "$repo_root" diff --quiet HEAD -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer || block 'reviewed deployer inputs differ from HEAD' fi - # Pin the reviewed inputs before any privileged copy: archive the exact - # HEAD commit into a root-controlled directory, then require every staged - # byte to match the clean worktree content observed at validation time, so - # neither worktree nor object-database mutation can substitute bytes. - local checkout_hashes archived_hashes - checkout_hashes=$(cd "$repo_root" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1) - if [[ -d "$state_root" && ! -L "$state_root" ]]; then - checkout_snapshot=$(mktemp -d "$state_root/.checkout.XXXXXX") - else - checkout_snapshot=$(mktemp -d) - fi + # Pin the reviewed inputs before any privileged copy: copy the worktree + # bytes once into a root-controlled snapshot outside managed state, then + # require each copied file's Git blob identity to equal the pinned commit's + # tree entry. The commit SHA is content-addressed, so no mutation of the + # worktree, refs, or loose objects can substitute bytes under $head. + local path blob + checkout_snapshot=$(mktemp -d) chmod 0700 "$checkout_snapshot" - git -C "$repo_root" archive "$head" scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer | tar -x -C "$checkout_snapshot" || block 'reviewed checkout archival failed' - archived_hashes=$(cd "$checkout_snapshot" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1) - [[ $archived_hashes == "$checkout_hashes" ]] || block 'archived checkout bytes differ from the validated worktree' + install -d -m 0700 "$checkout_snapshot/scripts" "$checkout_snapshot/deploy/deployer" + install -m 0755 "$repo_root/scripts/install-deployer.sh" "$repo_root/scripts/deployer-runtime.sh" "$checkout_snapshot/scripts/" + install -m 0644 "$repo_root"/deploy/deployer/* "$checkout_snapshot/deploy/deployer/" + while read -r _ _ blob path; do + [[ $blob == "$(git hash-object "$checkout_snapshot/$path")" ]] || block "checkout input $path differs from the reviewed commit" + done < <(git -C "$repo_root" ls-tree -r "$head" -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer) repo_root=$checkout_snapshot unit_source=$repo_root/deploy/deployer } @@ -535,7 +534,7 @@ acquire_check_lock() { } begin_transaction() { - local name path current_target transaction_name transaction_ready deployed_target + local name path current_target transaction_name transaction_ready deployed_target enabled_state for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do path=$state_root/$name [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:600" ]] || block 'managed transaction state has an unsafe type, owner, or mode' @@ -569,7 +568,9 @@ begin_transaction() { fi done for name in "${timer_names[@]}"; do - if systemctl is-enabled "$name" >/dev/null 2>&1; then printf '%s\n' "$name" >>"$transaction_dir/timers-enabled"; fi + enabled_state=$(systemctl is-enabled "$name" 2>/dev/null) || enabled_state= + if [[ $enabled_state == enabled ]]; then printf '%s\n' "$name" >>"$transaction_dir/timers-enabled"; fi + if [[ $enabled_state == enabled-runtime ]]; then printf '%s\n' "$name" >>"$transaction_dir/timers-enabled-runtime"; fi if systemctl is-active "$name" >/dev/null 2>&1; then printf '%s\n' "$name" >>"$transaction_dir/timers-active"; fi done if [[ -L "$deployed_current" ]]; then @@ -590,7 +591,7 @@ restore_transaction() { local name target_value backed_up [[ -n ${transaction_dir:-} && -d $transaction_dir ]] || return 0 transaction_committed=1 - for name in units-present state-present timers-enabled timers-active current-target deployed-target; do + for name in units-present state-present timers-enabled timers-enabled-runtime timers-active current-target deployed-target deployed-created; do [[ ! -e "$transaction_dir/$name" && ! -L "$transaction_dir/$name" ]] || [[ ! -L "$transaction_dir/$name" && -f "$transaction_dir/$name" ]] || block "transaction manifest $name has an unsafe type" done if [[ -f "$transaction_dir/units-present" ]]; then @@ -603,7 +604,7 @@ restore_transaction() { [[ " install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf " == *" $name "* && -f "$transaction_dir/state/$name" && ! -L "$transaction_dir/state/$name" ]] || block 'transaction state manifest is unsafe' done <"$transaction_dir/state-present" fi - for name in timers-enabled timers-active; do + for name in timers-enabled timers-enabled-runtime timers-active; do if [[ -f "$transaction_dir/$name" ]]; then while IFS= read -r target_value; do [[ " ${timer_names[*]} " == *" $target_value "* ]] || block 'transaction timer manifest is unsafe' @@ -653,12 +654,24 @@ restore_transaction() { ln -s "$target_value" "$deployed_current" || return fi fi + if [[ -f "$transaction_dir/deployed-created" ]]; then + target_value=$(<"$transaction_dir/deployed-created") + [[ $target_value =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'transaction created snapshot name is unsafe' + if [[ ! -e $deployed_current && ! -L $deployed_current ]] || [[ $(readlink "$deployed_current" 2>/dev/null) != "$target_value" ]]; then + [[ ! -d "$deployed_root/$target_value" || -L "$deployed_root/$target_value" ]] || rm -rf -- "${deployed_root:?}/$target_value" + fi + fi systemctl daemon-reload >/dev/null 2>&1 || return if [[ -f "$transaction_dir/timers-enabled" ]]; then while IFS= read -r name; do systemctl enable "$name" >/dev/null 2>&1 || return done <"$transaction_dir/timers-enabled" fi + if [[ -f "$transaction_dir/timers-enabled-runtime" ]]; then + while IFS= read -r name; do + systemctl enable --runtime "$name" >/dev/null 2>&1 || return + done <"$transaction_dir/timers-enabled-runtime" + fi if [[ -f "$transaction_dir/timers-active" ]]; then while IFS= read -r name; do systemctl start "$name" >/dev/null 2>&1 || return @@ -806,6 +819,7 @@ load_deployed_snapshot() { local deployed_credential_provider deployed_credential_ref deployed_credential_scope deployed_environment secure_directory "$deployed_root" 700 0 || block 'deployed snapshot directory is missing' [[ -L "$deployed_current" ]] || block 'deployed snapshot pointer is absent or unsafe' + [[ $(readlink "$deployed_current") =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'deployed snapshot pointer target is not canonical' snapshot=$(readlink -f "$deployed_current") inside "$snapshot" "$deployed_root" || block 'deployed snapshot pointer escapes managed state' secure_directory "$snapshot" 700 0 || block 'deployed snapshot is unsafe' @@ -1024,7 +1038,10 @@ PY mv -Tf "$state_root/.install-state.new" "$state_file" mv -Tf "$install_root/.current.new" "$current" policy_adapter_operation "$active_policy" health 'candidate policy' || die 'candidate health check failed after activation' - if [[ ! -e "$deployed_current" && ! -L "$deployed_current" ]]; then publish_deployed_snapshot "$active_policy" "$state_file"; fi + if [[ ! -e "$deployed_current" && ! -L "$deployed_current" ]]; then + publish_deployed_snapshot "$active_policy" "$state_file" + if [[ -n ${transaction_dir:-} && -d $transaction_dir ]]; then readlink "$deployed_current" >"$transaction_dir/deployed-created"; fi + fi commit_transaction health=healthy report CHANGED yes run-check "$(rollback_available)" @@ -1118,7 +1135,7 @@ perform_uninstall() { if active_deployment; then block 'active deployment started while draining'; fi if [[ -L "$current" ]]; then rm -f "$current"; changed=yes; fi [[ ! -e "$current" ]] || block 'activation pointer has an unsafe type' - systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || true + systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || block 'deployer timers did not stop during uninstall' for unit in "${unit_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then rm -f "$systemd_root/$unit"; changed=yes; fi; done systemctl daemon-reload >/dev/null 2>&1 || true rm -f "$drained" "$active_operation" diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 931ce6f..cf474a3 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -58,15 +58,25 @@ case "${1:-}" in printf '%s\n' "${FAKE_SYSTEMD_STATE:-running}" [[ ${FAKE_SYSTEMD_STATE:-running} != degraded ]] ;; - is-enabled|is-active) [[ -e "$root/var/lib/ci-fleet-deployer/unit-${2:-}" ]] && { [[ $1 == is-active ]] || printf '%s\n' "${FAKE_SYSTEMD_IS_ENABLED_OUTPUT:-enabled}"; } ;; + is-enabled|is-active) + [[ -e "$root/var/lib/ci-fleet-deployer/unit-${2:-}" ]] || exit 1 + if [[ $1 == is-enabled ]]; then + if [[ -e "$root/var/lib/ci-fleet-deployer/unit-${2:-}.runtime" ]]; then printf 'enabled-runtime\n'; else printf '%s\n' "${FAKE_SYSTEMD_IS_ENABLED_OUTPUT:-enabled}"; fi + fi + ;; enable) shift + runtime_flag=0 + [[ "${1:-}" != --runtime ]] || { runtime_flag=1; shift; } [[ "${1:-}" != --now ]] || shift - for unit in "$@"; do : >"$root/var/lib/ci-fleet-deployer/unit-$unit"; done ;; + for unit in "$@"; do + if ((runtime_flag)); then : >"$root/var/lib/ci-fleet-deployer/unit-$unit.runtime"; else rm -f "$root/var/lib/ci-fleet-deployer/unit-$unit.runtime"; fi + : >"$root/var/lib/ci-fleet-deployer/unit-$unit" + done ;; disable) shift [[ "${1:-}" != --now ]] || shift - for unit in "$@"; do rm -f "$root/var/lib/ci-fleet-deployer/unit-$unit"; done ;; + for unit in "$@"; do rm -f "$root/var/lib/ci-fleet-deployer/unit-$unit" "$root/var/lib/ci-fleet-deployer/unit-$unit.runtime"; done ;; esac exit 0 EOF @@ -417,6 +427,47 @@ expect_failure 'transaction units manifest is missing but backups remain' "$inst rm -rf -- "$truncated_tx" expect_success "$installer" --check --config "$config" >/dev/null +# Transaction recovery must restore runtime-only timer enablement exactly. +runtime_tx=$root/var/lib/ci-fleet-deployer/.transaction.runtime-enabled +mkdir -m 0700 "$runtime_tx" "$runtime_tx/units" "$runtime_tx/state" +: >"$root/var/lib/ci-fleet-deployer/unit-ci-fleet-deployer-health.timer.runtime" +printf 'ci-fleet-deployer-health.timer\n' >"$runtime_tx/timers-enabled-runtime" +expect_success "$installer" --repair --config "$config" >/dev/null +grep -Fq 'enable --runtime ci-fleet-deployer-health.timer' "$FAKE_SYSTEMCTL_LOG" || fail 'recovery did not restore runtime-only enablement' +rm -f "$root/var/lib/ci-fleet-deployer/unit-ci-fleet-deployer-health.timer.runtime" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# Recovery must remove a snapshot published after the transaction began. +created_tx=$root/var/lib/ci-fleet-deployer/.transaction.created-snapshot +mkdir -m 0700 "$created_tx" "$created_tx/units" "$created_tx/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$created_tx/state/$name"; printf '%s\n' "$name" >>"$created_tx/state-present"; } +done +for path in "$root"/etc/systemd/system/ci-fleet-deployer*; do + name=${path##*/}; cp "$path" "$created_tx/units/$name"; printf '%s\n' "$name" >>"$created_tx/units-present" +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$created_tx/current-target" +printf '%s\n' ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer >"$created_tx/timers-enabled" +orphan_snapshot=$root/var/lib/ci-fleet-deployer/deployed/.snapshot.orphaned +mkdir -m 0700 "$orphan_snapshot" +install -m 0600 "$root/var/lib/ci-fleet-deployer/active-policy.conf" "$orphan_snapshot/policy.conf" +install -m 0600 "$root/var/lib/ci-fleet-deployer/install-state.json" "$orphan_snapshot/state.json" +current_deployed=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") +printf '%s\n' "$current_deployed" >"$created_tx/deployed-target" +printf '.snapshot.orphaned\n' >"$created_tx/deployed-created" +rm "$root/var/lib/ci-fleet-deployer/deployed/current" +ln -s .snapshot.orphaned "$root/var/lib/ci-fleet-deployer/deployed/current" +deployed_count_before=$(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$orphan_snapshot" ]] || fail 'recovery retained a snapshot created after the transaction began' +[[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == $((deployed_count_before - 1)) ]] || fail 'recovery removed the wrong snapshot' +[[ $(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") == "$current_deployed" ]] || fail 'recovery did not restore the prior deployed pointer' +expect_success "$installer" --check --config "$config" >/dev/null + +# Read-only checks must not create checkout snapshots inside managed state. +if compgen -G "$root/var/lib/ci-fleet-deployer/.checkout.*" >/dev/null; then fail 'read-only check left a checkout snapshot in managed state'; fi + # A noncanonical activation pointer must fail convergence instead of certifying it. rm "$root/opt/ci-fleet-deployer/current" ln -s "$root/opt/ci-fleet-deployer/releases/$core_ref" "$root/opt/ci-fleet-deployer/current" @@ -881,6 +932,14 @@ rmdir "$root/etc/systemd/system/ci-fleet-deployer-drain.service" expect_success "$installer" --uninstall --config "$config" >/dev/null [[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after correcting unit drift did not uninstall' +# A failed timer shutdown must fail the uninstall before any unit removal. +expect_success "$installer" --install --config "$config" >/dev/null +uninstall_before=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) +FAKE_SYSTEMCTL_FAIL_COMMAND=disable expect_failure 'deployer timers did not stop during uninstall' "$installer" --uninstall --config "$config" >/dev/null +[[ "$uninstall_before" == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'failed timer shutdown partially uninstalled units' +[[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'failed timer shutdown removed the activation pointer' +expect_success "$installer" --uninstall --config "$config" >/dev/null + # Runtime contract: exact-head request/evidence, drain and scoped adapter calls. write_evidence staging example-staging write_config staging example-staging @@ -1036,6 +1095,14 @@ deployed_target=$(readlink "$deployed_current") rm "$deployed_current" expect_failure 'deployed rollback snapshot is missing' "$runtime" deploy >/dev/null ln -s "$deployed_target" "$deployed_current" +rm "$deployed_current" +ln -s "$root/var/lib/ci-fleet-deployer/deployed/$deployed_target" "$deployed_current" +deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) +expect_failure 'deployed snapshot pointer target is not canonical' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment consumed approval with a noncanonical deployed pointer' +[[ ! -e "$active" ]] || fail 'rejected deployment left the active operation marker' +rm "$deployed_current" +ln -s "$deployed_target" "$deployed_current" chmod 0644 "$credential" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) expect_failure 'credential file has unsafe owner or mode' "$runtime" deploy >/dev/null @@ -1072,7 +1139,7 @@ cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT=$root/var/lib/ci-fleet-deployer/deployed expect_failure 'prepared deployed snapshot changed during deployment' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT -grep -Fq 'approval=snapshot-mutation-attempt approver=example-reviewer policy=example-staging-policy-v1 checkpoint=checkpoint-20260808-1 authorized_by=none gate=none result=failed phase=post-adapter' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'post-adapter deployment failure was not audited' +grep -Fq 'approval=snapshot-mutation-attempt approver=example-reviewer policy=example-staging-policy-v1 checkpoint=checkpoint-20260808-1 authorized_by=none gate=none result=failed phase=post-adapter status=2' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'post-adapter deployment failure was not audited with its real status' [[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == 1 ]] || fail 'failed publication leaked an unreachable prepared snapshot' [[ -e "$deployed_current" ]] || fail 'failed publication dangled the deployed pointer' @@ -1161,7 +1228,7 @@ expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" c cp "$runtime" "$tmp/runtime.saved" release_before=$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh") printf '# substituted-live-bytes\n' >>"$runtime" -expect_failure 'archived checkout bytes differ from the validated worktree' "$installer" --repair --config "$config" >/dev/null +expect_failure 'differs from the reviewed commit' "$installer" --repair --config "$config" >/dev/null [[ $release_before == "$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh")" ]] || fail 'mutated live checkout bytes entered the trusted release' cat "$tmp/runtime.saved" >"$runtime" git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$runtime" || fail 'live checkout restoration diverged from HEAD' From 8132bd5b624889c6a49966564a68fdfdebeabb74 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 18:31:13 -0500 Subject: [PATCH 35/90] fix: stop timers before removing the activation pointer on uninstall --- scripts/install-deployer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 56d101c..cc83321 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -1133,9 +1133,9 @@ perform_uninstall() { mv -Tf "$temporary" "$drained" fi if active_deployment; then block 'active deployment started while draining'; fi + systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || block 'deployer timers did not stop during uninstall' if [[ -L "$current" ]]; then rm -f "$current"; changed=yes; fi [[ ! -e "$current" ]] || block 'activation pointer has an unsafe type' - systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || block 'deployer timers did not stop during uninstall' for unit in "${unit_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then rm -f "$systemd_root/$unit"; changed=yes; fi; done systemctl daemon-reload >/dev/null 2>&1 || true rm -f "$drained" "$active_operation" From 22aeef46c91b5457b146910ca21c3d7129f24dbc Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 18:57:05 -0500 Subject: [PATCH 36/90] fix: validate systemd boundary and finalize dependencies before recovery mutation - Block on a symlinked or mis-owned systemd unit directory before opening or restoring a transaction so recovery never deletes through a drifted boundary. - Verify the committed rollback policy adapter bytes (containment, type, ownership, mode, digest) and credential reference safety before finalizing an interrupted committed rollback, so drift cannot strand the retained last-known-good pair. --- scripts/install-deployer.sh | 11 ++++++++++ scripts/test-install-deployer.sh | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index cc83321..9eccf5b 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -535,6 +535,7 @@ acquire_check_lock() { begin_transaction() { local name path current_target transaction_name transaction_ready deployed_target enabled_state + [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do path=$state_root/$name [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:600" ]] || block 'managed transaction state has an unsafe type, owner, or mode' @@ -628,6 +629,7 @@ restore_transaction() { fi done systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || return + [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name" || return; done if [[ -f "$transaction_dir/units-present" ]]; then while IFS= read -r name; do @@ -705,9 +707,18 @@ recover_interrupted_transaction() { finalize_committed_rollback() { local marker=$transaction_dir/application-rollback-committed retired + local final_adapter_path final_adapter_sha final_credential_provider final_credential_ref [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' secure_file "$active_policy" 'active policy' || return secure_file "$state_file" 'deployer install state' || return + final_adapter_path=$(awk -F= '$1=="ADAPTER_PATH" {print $2}' "$active_policy") + final_adapter_sha=$(awk -F= '$1=="ADAPTER_SHA256" {print $2}' "$active_policy") + inside "$final_adapter_path" "$etc_root/adapters" || return + [[ ! -L "$final_adapter_path" && -f "$final_adapter_path" && $(stat -c '%u:%a' "$final_adapter_path") == "$expected_uid:700" ]] || return + [[ $(sha256sum "$final_adapter_path" | cut -d' ' -f1) == "$final_adapter_sha" ]] || return + final_credential_provider=$(awk -F= '$1=="CREDENTIAL_PROVIDER" {print $2}' "$active_policy") + final_credential_ref=$(awk -F= '$1=="CREDENTIAL_REF" {print $2}' "$active_policy") + credential_reference_safe "$final_credential_provider" "$final_credential_ref" 'committed rollback policy' || return python3 - "$state_file" "$active_policy" "$etc_root/adapters" "$etc_root/credentials" <<'PY' >/dev/null 2>&1 || return import json, re, sys try: diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index cf474a3..ff9ef39 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -775,6 +775,43 @@ expect_failure 'application rollback commit marker is unsafe' "$installer" --rep rm -rf -- "$recovery_transaction" expect_success "$installer" --check --config "$config" >/dev/null +# Finalizing a committed rollback with drifted adapter bytes must not publish or delete retained state. +recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.drifted-adapter +rm -rf "$root/var/lib/ci-fleet-deployer/deployed" +mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$recovery_transaction/state/$name"; printf '%s\n' "$name" >>"$recovery_transaction/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$recovery_transaction/current-target" +install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" +cp "$adapter" "$tmp/adapter.saved" +printf '# drifted\n' >>"$adapter" +lkg_before_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +expect_failure 'transaction recovery failed' "$installer" --repair --config "$config" >/dev/null +[[ -d "$recovery_transaction" ]] || fail 'failed finalize discarded its recovery transaction' +[[ $lkg_before_finalize == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'failed finalize deleted the retained rollback pair' +cat "$tmp/adapter.saved" >"$adapter" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A symlinked systemd boundary must block before transaction recovery deletes through it. +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +boundary_tx=$root/var/lib/ci-fleet-deployer/.transaction.boundary +mkdir -m 0700 "$boundary_tx" "$boundary_tx/units" "$boundary_tx/state" +mv "$root/etc/systemd/system" "$tmp/systemd.real" +ln -s "$tmp/systemd.real" "$root/etc/systemd/system" +printf 'decoy\n' >"$tmp/systemd.real/ci-fleet-deployer.service" +expect_failure 'systemd unit directory has an unsafe owner or type' "$installer" --repair --config "$config" >/dev/null +[[ $(<"$tmp/systemd.real/ci-fleet-deployer.service") == decoy ]] || fail 'recovery deleted through a symlinked systemd boundary' +[[ -d "$boundary_tx" ]] || fail 'blocked boundary recovery discarded its transaction' +rm "$root/etc/systemd/system" +mv "$tmp/systemd.real" "$root/etc/systemd/system" +rm "$root/etc/systemd/system/ci-fleet-deployer.service" +rm -rf -- "$boundary_tx" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + # A deployed rollback pair whose adapter bytes no longer match its recorded digest must not promote. deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") deployed_target=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") From 91707e89c2702eb8a14302195b6293ce4a6cbacc Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 19:02:06 -0500 Subject: [PATCH 37/90] test: drifted adapter is caught at candidate validation before recovery --- scripts/test-install-deployer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index ff9ef39..3a8ca6a 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -787,7 +787,7 @@ install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" cp "$adapter" "$tmp/adapter.saved" printf '# drifted\n' >>"$adapter" lkg_before_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") -expect_failure 'transaction recovery failed' "$installer" --repair --config "$config" >/dev/null +expect_failure 'adapter digest does not match the protected regular file' "$installer" --repair --config "$config" >/dev/null [[ -d "$recovery_transaction" ]] || fail 'failed finalize discarded its recovery transaction' [[ $lkg_before_finalize == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'failed finalize deleted the retained rollback pair' cat "$tmp/adapter.saved" >"$adapter" From 74fa0ffe4aa76957c94c65c5c54903096a5053d5 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 19:07:27 -0500 Subject: [PATCH 38/90] test: tolerate absent retained pair in drifted-finalize regression --- scripts/test-install-deployer.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 3a8ca6a..b826dca 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -786,10 +786,11 @@ printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$recovery_tr install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" cp "$adapter" "$tmp/adapter.saved" printf '# drifted\n' >>"$adapter" -lkg_before_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_before_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); else lkg_before_finalize=absent; fi expect_failure 'adapter digest does not match the protected regular file' "$installer" --repair --config "$config" >/dev/null [[ -d "$recovery_transaction" ]] || fail 'failed finalize discarded its recovery transaction' -[[ $lkg_before_finalize == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'failed finalize deleted the retained rollback pair' +lkg_after_finalize=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +[[ $lkg_before_finalize == "$lkg_after_finalize" ]] || fail 'failed finalize deleted the retained rollback pair' cat "$tmp/adapter.saved" >"$adapter" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --repair --config "$config" >/dev/null From fd6874997efc0334a40db81c19fade006673a827 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 19:12:40 -0500 Subject: [PATCH 39/90] test: tolerate absent retained pair in deployed-pair drift regressions --- scripts/test-install-deployer.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index b826dca..bfe4955 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -822,15 +822,15 @@ from pathlib import Path import sys p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ADAPTER_SHA256=', 'ADAPTER_SHA256=' + 'f'*64 + '\n#', 1)) PY -lkg_before_digest=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_before_digest=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); else lkg_before_digest=absent; fi expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --upgrade --config "$config" >/dev/null -[[ $lkg_before_digest == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'digest-mismatched deployed pair replaced last-known-good' +lkg_after_digest=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_digest=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_digest == "$lkg_after_digest" ]] || fail 'digest-mismatched deployed pair replaced last-known-good' if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'blocked digest promotion left a recovery transaction'; fi expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --rollback --config "$config" >/dev/null rollback_tx=$(compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*") || fail 'blocked digest rollback left no recovery transaction' [[ $(<"$rollback_tx/deployed-target") == "$deployed_target" ]] || fail 'blocked digest rollback recorded an unexpected deployed target' expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --repair --config "$config" >/dev/null -[[ $lkg_before_digest == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'digest-mismatched deployed pair reached recovery promotion' +lkg_after_digest=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_digest=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_digest == "$lkg_after_digest" ]] || fail 'digest-mismatched deployed pair reached recovery promotion' install -m 0600 "$tmp/deployed-policy.saved" "$deployed_dir/policy.conf" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --repair --config "$config" >/dev/null @@ -845,11 +845,11 @@ from pathlib import Path import sys p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external').replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=external:bad')) PY -lkg_before_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_before_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); else lkg_before_credential=absent; fi expect_failure 'deployed rollback policy has an invalid external secret-manager adapter reference' "$installer" --upgrade --config "$config" >/dev/null -[[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair replaced last-known-good' +lkg_after_credential=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_credential == "$lkg_after_credential" ]] || fail 'credential-drifted deployed pair replaced last-known-good' expect_failure 'deployed rollback policy has an invalid external secret-manager adapter reference' "$installer" --rollback --config "$config" >/dev/null -[[ $lkg_before_credential == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'credential-drifted deployed pair reached rollback' +lkg_after_credential=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_credential == "$lkg_after_credential" ]] || fail 'credential-drifted deployed pair reached rollback' install -m 0600 "$tmp/deployed-policy-cred.saved" "$deployed_dir/policy.conf" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --repair --config "$config" >/dev/null From e9e5d7a3bc55dd0c3d84318d7a552c59f76f5f87 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 19:18:26 -0500 Subject: [PATCH 40/90] test: order deployed-pair drift regressions before finalize recovery --- scripts/test-install-deployer.sh | 76 ++++++++++++++++---------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index bfe4955..4f0fcb4 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -775,44 +775,6 @@ expect_failure 'application rollback commit marker is unsafe' "$installer" --rep rm -rf -- "$recovery_transaction" expect_success "$installer" --check --config "$config" >/dev/null -# Finalizing a committed rollback with drifted adapter bytes must not publish or delete retained state. -recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.drifted-adapter -rm -rf "$root/var/lib/ci-fleet-deployer/deployed" -mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" -for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do - [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$recovery_transaction/state/$name"; printf '%s\n' "$name" >>"$recovery_transaction/state-present"; } -done -printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$recovery_transaction/current-target" -install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" -cp "$adapter" "$tmp/adapter.saved" -printf '# drifted\n' >>"$adapter" -if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_before_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); else lkg_before_finalize=absent; fi -expect_failure 'adapter digest does not match the protected regular file' "$installer" --repair --config "$config" >/dev/null -[[ -d "$recovery_transaction" ]] || fail 'failed finalize discarded its recovery transaction' -lkg_after_finalize=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") -[[ $lkg_before_finalize == "$lkg_after_finalize" ]] || fail 'failed finalize deleted the retained rollback pair' -cat "$tmp/adapter.saved" >"$adapter" -expect_success "$installer" --repair --config "$config" >/dev/null -expect_success "$installer" --repair --config "$config" >/dev/null -expect_success "$installer" --check --config "$config" >/dev/null - -# A symlinked systemd boundary must block before transaction recovery deletes through it. -rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" -boundary_tx=$root/var/lib/ci-fleet-deployer/.transaction.boundary -mkdir -m 0700 "$boundary_tx" "$boundary_tx/units" "$boundary_tx/state" -mv "$root/etc/systemd/system" "$tmp/systemd.real" -ln -s "$tmp/systemd.real" "$root/etc/systemd/system" -printf 'decoy\n' >"$tmp/systemd.real/ci-fleet-deployer.service" -expect_failure 'systemd unit directory has an unsafe owner or type' "$installer" --repair --config "$config" >/dev/null -[[ $(<"$tmp/systemd.real/ci-fleet-deployer.service") == decoy ]] || fail 'recovery deleted through a symlinked systemd boundary' -[[ -d "$boundary_tx" ]] || fail 'blocked boundary recovery discarded its transaction' -rm "$root/etc/systemd/system" -mv "$tmp/systemd.real" "$root/etc/systemd/system" -rm "$root/etc/systemd/system/ci-fleet-deployer.service" -rm -rf -- "$boundary_tx" -expect_success "$installer" --repair --config "$config" >/dev/null -expect_success "$installer" --check --config "$config" >/dev/null - # A deployed rollback pair whose adapter bytes no longer match its recorded digest must not promote. deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") deployed_target=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") @@ -855,6 +817,44 @@ expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null +# Finalizing a committed rollback with drifted adapter bytes must not publish or delete retained state. +recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.drifted-adapter +rm -rf "$root/var/lib/ci-fleet-deployer/deployed" +mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$recovery_transaction/state/$name"; printf '%s\n' "$name" >>"$recovery_transaction/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$recovery_transaction/current-target" +install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" +cp "$adapter" "$tmp/adapter.saved" +printf '# drifted\n' >>"$adapter" +if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_before_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); else lkg_before_finalize=absent; fi +expect_failure 'adapter digest does not match the protected regular file' "$installer" --repair --config "$config" >/dev/null +[[ -d "$recovery_transaction" ]] || fail 'failed finalize discarded its recovery transaction' +lkg_after_finalize=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_finalize=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +[[ $lkg_before_finalize == "$lkg_after_finalize" ]] || fail 'failed finalize deleted the retained rollback pair' +cat "$tmp/adapter.saved" >"$adapter" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A symlinked systemd boundary must block before transaction recovery deletes through it. +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +boundary_tx=$root/var/lib/ci-fleet-deployer/.transaction.boundary +mkdir -m 0700 "$boundary_tx" "$boundary_tx/units" "$boundary_tx/state" +mv "$root/etc/systemd/system" "$tmp/systemd.real" +ln -s "$tmp/systemd.real" "$root/etc/systemd/system" +printf 'decoy\n' >"$tmp/systemd.real/ci-fleet-deployer.service" +expect_failure 'systemd unit directory has an unsafe owner or type' "$installer" --repair --config "$config" >/dev/null +[[ $(<"$tmp/systemd.real/ci-fleet-deployer.service") == decoy ]] || fail 'recovery deleted through a symlinked systemd boundary' +[[ -d "$boundary_tx" ]] || fail 'blocked boundary recovery discarded its transaction' +rm "$root/etc/systemd/system" +mv "$tmp/systemd.real" "$root/etc/systemd/system" +rm "$root/etc/systemd/system/ci-fleet-deployer.service" +rm -rf -- "$boundary_tx" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + # A lost install state with surviving release/units must not be treated as a fresh install. mv "$root/var/lib/ci-fleet-deployer/install-state.json" "$tmp/install-state.saved" expect_failure 'restore install state before convergence' "$installer" --repair --config "$config" >/dev/null From 42254a9c2e3b063a13aff4454deb01b25a66cb48 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 19:29:54 -0500 Subject: [PATCH 41/90] test: rebuild retained rollback pair after finalize recovery fixture --- scripts/test-install-deployer.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 4f0fcb4..d1145b8 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -855,6 +855,18 @@ rm -rf -- "$boundary_tx" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null +# Rebuild a retained rollback pair consumed by the finalize recovery fixture. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' + # A lost install state with surviving release/units must not be treated as a fresh install. mv "$root/var/lib/ci-fleet-deployer/install-state.json" "$tmp/install-state.saved" expect_failure 'restore install state before convergence' "$installer" --repair --config "$config" >/dev/null From 2467f17ae3a8615404d2a97b26882242b3482a48 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 19:58:32 -0500 Subject: [PATCH 42/90] fix: replace-immune checkout pins, marker type checks, idempotent uninstall - Resolve the reviewed commit and its tree with --no-replace-objects so attacker-added refs/replace entries cannot align substituted bytes with the pinned baseline. - Treat an existing non-regular active-operation marker as an error in both read-only checks and mutating operations instead of absence. - Disable only timers that still exist during uninstall so the documented repeated invocation stays idempotently NO_CHANGE. - Remove an unpublished active-operation temporary in the deploy exit trap so canceled deployments cannot accumulate residue. --- scripts/deployer-runtime.sh | 4 +++- scripts/install-deployer.sh | 14 +++++++++++--- scripts/test-install-deployer.sh | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index b4c8162..5d7d498 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -47,7 +47,7 @@ deploy_exit() { "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ "${audit_phase:-post-consumption}" "$recorded_status" >&8 || true fi - rm -f "$active" "${request_snapshot:-}" "${policy_snapshot:-}" || true + rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then target=$(readlink "$deployed_current" 2>/dev/null || true) if [[ $target != "${snapshot##*/}" ]]; then rm -rf -- "$snapshot"; fi @@ -297,11 +297,13 @@ PY install -m 0600 /dev/null "$consumed_marker" || die 'deployment request consumption marker failed' umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") + active_temporary=$temporary printf 'pid=%s\nstarted_at=%s\n' "$$" "$(date +%s)" >"$temporary" if [[ $testing != 1 ]]; then printf 'boot_id=%s\nstart_time=%s\n' "$(>"$temporary" fi mv -Tf "$temporary" "$active" + active_temporary= set +e env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 9eccf5b..f39bd02 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -390,7 +390,7 @@ reject_mixed_role() { validate_checkout() { local head - head=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') || block 'installer checkout is not Git-authored' + head=$(git -C "$repo_root" --no-replace-objects rev-parse 'HEAD^{commit}') || block 'installer checkout is not Git-authored' [[ "$head" == "$core_ref" ]] || block 'CORE_REF must equal the exact reviewed checkout HEAD' if [[ "$testing" != 1 ]]; then git -C "$repo_root" diff --quiet HEAD -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer || block 'reviewed deployer inputs differ from HEAD' @@ -408,7 +408,7 @@ validate_checkout() { install -m 0644 "$repo_root"/deploy/deployer/* "$checkout_snapshot/deploy/deployer/" while read -r _ _ blob path; do [[ $blob == "$(git hash-object "$checkout_snapshot/$path")" ]] || block "checkout input $path differs from the reviewed commit" - done < <(git -C "$repo_root" ls-tree -r "$head" -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer) + done < <(git -C "$repo_root" --no-replace-objects ls-tree -r "$head" -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer) repo_root=$checkout_snapshot unit_source=$repo_root/deploy/deployer } @@ -504,6 +504,7 @@ acquire_lock() { exec 9<"$lock_root" flock -n 9 || block 'another deployer installer operation is running' [[ ! -L "$active_operation" ]] || block 'active operation marker is an unsafe symlink' + if [[ -e "$active_operation" && ! -f "$active_operation" ]]; then block 'active operation marker has an unsafe type'; fi if [[ -e "$active_operation" ]] && ! active_deployment; then [[ ! -L "$active_operation" && -f "$active_operation" && $(stat -c '%u:%a' "$active_operation") == "$expected_uid:600" ]] || block 'stale operation state is unsafe' rm -f "$active_operation" @@ -529,6 +530,9 @@ acquire_lock() { acquire_check_lock() { [[ -d "$lock_root" && ! -L "$lock_root" && $(stat -c '%u:%a' "$lock_root") == "$expected_uid:700" ]] || block 'installed deployer lock boundary is absent or unsafe' + if [[ -e "$active_operation" || -L "$active_operation" ]]; then + [[ -f "$active_operation" && ! -L "$active_operation" ]] || block 'active operation marker has an unsafe type' + fi exec 9<"$lock_root" flock -n 9 || block 'another deployer operation is running' } @@ -1144,7 +1148,11 @@ perform_uninstall() { mv -Tf "$temporary" "$drained" fi if active_deployment; then block 'active deployment started while draining'; fi - systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || block 'deployer timers did not stop during uninstall' + for unit in "${timer_names[@]}"; do + if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then + systemctl disable --now "$unit" >/dev/null 2>&1 || block 'deployer timers did not stop during uninstall' + fi + done if [[ -L "$current" ]]; then rm -f "$current"; changed=yes; fi [[ ! -e "$current" ]] || block 'activation pointer has an unsafe type' for unit in "${unit_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then rm -f "$systemd_root/$unit"; changed=yes; fi; done diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index d1145b8..5d186e3 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -533,6 +533,13 @@ CI_FLEET_DEPLOYER_TEST_LIVE_PID=$$ expect_failure 'active deployment prevents th [[ -f "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'live stale-aged operation marker was expired' rm "$root/var/lib/ci-fleet-deployer/active-operation" +# An unsafe active-operation marker type must fail checks, not certify them. +mkdir "$root/var/lib/ci-fleet-deployer/active-operation" +expect_failure 'active operation marker has an unsafe type' "$installer" --check --config "$config" >/dev/null +expect_failure 'active operation marker has an unsafe type' "$installer" --repair --config "$config" >/dev/null +rmdir "$root/var/lib/ci-fleet-deployer/active-operation" +expect_success "$installer" --check --config "$config" >/dev/null + exec 8<"$root/var/lock/ci-fleet-deployer" flock -n 8 || fail 'fixture could not acquire installer lock' expect_failure 'another deployer installer operation is running' "$installer" --repair --config "$config" >/dev/null @@ -989,6 +996,8 @@ FAKE_SYSTEMCTL_FAIL_COMMAND=disable expect_failure 'deployer timers did not stop [[ "$uninstall_before" == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'failed timer shutdown partially uninstalled units' [[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'failed timer shutdown removed the activation pointer' expect_success "$installer" --uninstall --config "$config" >/dev/null +repeat_uninstall=$(expect_success "$installer" --uninstall --config "$config") +grep -Fq 'result=NO_CHANGE' <<<"$repeat_uninstall" || fail 'repeated uninstall was not idempotent after timer removal' # Runtime contract: exact-head request/evidence, drain and scoped adapter calls. write_evidence staging example-staging @@ -1139,6 +1148,7 @@ if CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF=TERM "$runtime" deploy >/dev/null 2>&1; th [[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == 1 ]] || fail 'signal after temporary pointer leaked an unreachable prepared snapshot' [[ -e "$deployed_current" ]] || fail 'signal after temporary pointer dangled the deployed pointer' [[ ! -e "$active" ]] || fail 'signaled deployment left the active operation marker' +if compgen -G "$root/var/lib/ci-fleet-deployer/.active.*" >/dev/null; then fail 'signaled deployment left an unpublished active marker temporary'; fi rm -f "$root/var/lib/ci-fleet-deployer/last-request.conf" rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" deployed_target=$(readlink "$deployed_current") @@ -1280,6 +1290,13 @@ release_before=$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scrip printf '# substituted-live-bytes\n' >>"$runtime" expect_failure 'differs from the reviewed commit' "$installer" --repair --config "$config" >/dev/null [[ $release_before == "$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh")" ]] || fail 'mutated live checkout bytes entered the trusted release' +replacement=$(git -C "$repo_root" commit-tree 'HEAD^{tree}' -m replace-fixture 2>/dev/null || true) +if [[ -n $replacement ]]; then + git -C "$repo_root" replace "$core_ref" "$replacement" + expect_failure 'differs from the reviewed commit' "$installer" --repair --config "$config" >/dev/null + [[ $release_before == "$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh")" ]] || fail 'replacement ref bytes entered the trusted release' + git -C "$repo_root" replace -d "$core_ref" +fi cat "$tmp/runtime.saved" >"$runtime" git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$runtime" || fail 'live checkout restoration diverged from HEAD' rm "$root/var/lib/ci-fleet-deployer/drained" From 4326f6b3bc8db0153e911c90194c053301b550e7 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 20:26:28 -0500 Subject: [PATCH 43/90] fix: propagate tree-read failures, tolerate absent timers, fsync consumption - Capture the pinned commit tree listing before the blob comparison and block when it is unreadable or empty, so a substituted checkout cannot ride a failed ls-tree into the trusted snapshot. - Disable only present timer units during transaction recovery so an interrupted first install restores instead of wedging on absent units. - fsync the request consumption marker and its directory before the adapter runs so a crash cannot replay a consumed approval. --- scripts/deployer-runtime.sh | 2 ++ scripts/install-deployer.sh | 12 +++++++++--- scripts/test-install-deployer.sh | 9 +++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 5d7d498..33690cc 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -295,6 +295,8 @@ PY trap deploy_exit EXIT trap 'exit 2' INT TERM install -m 0600 /dev/null "$consumed_marker" || die 'deployment request consumption marker failed' + sync -f "$consumed_marker" 2>/dev/null || sync "$consumed_marker" 2>/dev/null || die 'deployment request consumption marker is not durable' + sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || true umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") active_temporary=$temporary diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index f39bd02..9b2b5f8 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -400,15 +400,17 @@ validate_checkout() { # require each copied file's Git blob identity to equal the pinned commit's # tree entry. The commit SHA is content-addressed, so no mutation of the # worktree, refs, or loose objects can substitute bytes under $head. - local path blob + local path blob tree_listing checkout_snapshot=$(mktemp -d) chmod 0700 "$checkout_snapshot" install -d -m 0700 "$checkout_snapshot/scripts" "$checkout_snapshot/deploy/deployer" install -m 0755 "$repo_root/scripts/install-deployer.sh" "$repo_root/scripts/deployer-runtime.sh" "$checkout_snapshot/scripts/" install -m 0644 "$repo_root"/deploy/deployer/* "$checkout_snapshot/deploy/deployer/" + tree_listing=$(git -C "$repo_root" --no-replace-objects ls-tree -r "$head" -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer) || block 'reviewed commit tree is unreadable' + [[ -n $tree_listing ]] || block 'reviewed commit tree is unreadable' while read -r _ _ blob path; do [[ $blob == "$(git hash-object "$checkout_snapshot/$path")" ]] || block "checkout input $path differs from the reviewed commit" - done < <(git -C "$repo_root" --no-replace-objects ls-tree -r "$head" -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer) + done <<<"$tree_listing" repo_root=$checkout_snapshot unit_source=$repo_root/deploy/deployer } @@ -632,7 +634,11 @@ restore_transaction() { [[ -z $backed_up ]] || block "transaction $name manifest is missing but backups remain" fi done - systemctl disable --now "${timer_names[@]}" >/dev/null 2>&1 || return + for name in "${timer_names[@]}"; do + if [[ -e "$systemd_root/$name" || -L "$systemd_root/$name" ]]; then + systemctl disable --now "$name" >/dev/null 2>&1 || return + fi + done [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name" || return; done if [[ -f "$transaction_dir/units-present" ]]; then diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 5d186e3..e394969 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -465,6 +465,15 @@ expect_success "$installer" --repair --config "$config" >/dev/null [[ $(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") == "$current_deployed" ]] || fail 'recovery did not restore the prior deployed pointer' expect_success "$installer" --check --config "$config" >/dev/null +# Transaction recovery must tolerate timers whose unit files were never installed. +absent_timer_tx=$root/var/lib/ci-fleet-deployer/.transaction.absent-timer +mkdir -m 0700 "$absent_timer_tx" "$absent_timer_tx/units" "$absent_timer_tx/state" +mv "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" "$tmp/cleanup.timer.saved" +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$absent_timer_tx" ]] || fail 'absent-timer transaction was not recovered' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + # Read-only checks must not create checkout snapshots inside managed state. if compgen -G "$root/var/lib/ci-fleet-deployer/.checkout.*" >/dev/null; then fail 'read-only check left a checkout snapshot in managed state'; fi From 3f038bd61527b64a7630293ba2401bc1fb87c639 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 20:58:16 -0500 Subject: [PATCH 44/90] fix: writability guard before backups, durable commit marker, stale-marker checks - Require the systemd unit directory to be non-group/world-writable before any transaction backup, not only at install time. - fsync the application rollback commit marker and transaction directory before finalizing a committed rollback. - Report an existing stale active-operation marker as failed drift in read-only checks instead of certifying an installation the runtime would refuse. --- scripts/install-deployer.sh | 7 ++++++- scripts/test-install-deployer.sh | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 9b2b5f8..009aebb 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -540,8 +540,10 @@ acquire_check_lock() { } begin_transaction() { - local name path current_target transaction_name transaction_ready deployed_target enabled_state + local name path current_target transaction_name transaction_ready deployed_target enabled_state systemd_mode [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' + systemd_mode=$(stat -c %a "$systemd_root") + (((8#$systemd_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do path=$state_root/$name [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:600" ]] || block 'managed transaction state has an unsafe type, owner, or mode' @@ -993,6 +995,7 @@ perform_check() { shopt -u nullglob ((${#transactions[@]} == 0)) || block 'interrupted installer transaction requires recovery' if active_deployment; then block 'active deployment prevents a consistent check'; fi + if [[ -e "$active_operation" || -L "$active_operation" ]]; then block 'stale operation marker requires a mutating recovery'; fi converged || block 'installed deployer state is absent or drifted' policy_adapter_operation "$active_policy" health 'active policy' '' 0 || block 'active deployer health check failed' health=healthy @@ -1105,6 +1108,8 @@ PY mv -Tf "$install_root/.current.new" "$current" reject_mixed_role policy_adapter_operation "$active_policy" rollback 'last-known-good policy' "$transaction_dir/application-rollback-committed" || die 'application adapter rollback failed' + sync -f "$transaction_dir/application-rollback-committed" 2>/dev/null || sync "$transaction_dir/application-rollback-committed" 2>/dev/null || die 'application rollback commit marker is not durable' + sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || true finalize_committed_rollback recovered_rollback=0 health=healthy diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index e394969..2fa4c05 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -549,6 +549,14 @@ expect_failure 'active operation marker has an unsafe type' "$installer" --repai rmdir "$root/var/lib/ci-fleet-deployer/active-operation" expect_success "$installer" --check --config "$config" >/dev/null +# A stale marker for a dead process must fail read-only checks until a mutating recovery removes it. +printf 'pid=999999\nstarted_at=1\n' >"$root/var/lib/ci-fleet-deployer/active-operation" +chmod 0600 "$root/var/lib/ci-fleet-deployer/active-operation" +expect_failure 'stale operation marker requires a mutating recovery' "$installer" --check --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +[[ ! -e "$root/var/lib/ci-fleet-deployer/active-operation" ]] || fail 'repair retained the stale operation marker' +expect_success "$installer" --check --config "$config" >/dev/null + exec 8<"$root/var/lock/ci-fleet-deployer" flock -n 8 || fail 'fixture could not acquire installer lock' expect_failure 'another deployer installer operation is running' "$installer" --repair --config "$config" >/dev/null @@ -871,6 +879,16 @@ rm -rf -- "$boundary_tx" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null +# A group- or world-writable systemd boundary must block before transaction backups. +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +chmod 0777 "$root/etc/systemd/system" +units_before_writable=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) +expect_failure 'systemd unit directory is group- or world-writable' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_writable == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'writable systemd boundary allowed a transaction backup' +chmod 0755 "$root/etc/systemd/system" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + # Rebuild a retained rollback pair consumed by the finalize recovery fixture. image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' write_evidence From 4d22b564925cf8968ff3fe146cde8d47decb9108 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 21:35:11 -0500 Subject: [PATCH 45/90] fix: recovery boundary guards, durable journals, honest availability - Reject group/world-writable systemd boundaries and symlinked install roots before transaction recovery restores units or pointers. - fsync the transaction journal before begin_transaction returns and the deployed snapshot/pointer before retiring the prior rollback snapshot, so power loss cannot strand recovery state. - Report rollback_available=no when the retained adapter digest or file credential no longer validates. - Reject managed-unit drop-in directories in convergence so systemd cannot merge unreviewed overrides. - Re-exec the runtime from the release matching the active policy CORE_REF when a concurrent upgrade moved current after ExecStart resolution. --- scripts/deployer-runtime.sh | 18 ++++++++++++++++ scripts/install-deployer.sh | 20 ++++++++++++++++- scripts/test-install-deployer.sh | 37 ++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 33690cc..fde30b2 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -162,6 +162,21 @@ parse_file "$config" cfg configuration "$config_keys" for key in ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE APPROVAL_PROVIDER CHECKPOINT_EVIDENCE_PATH SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "cfg[$key]" ]] || die "configuration is missing $key"; done [[ ${cfg[SCHEMA_VERSION]:-} == 1 ]] || die 'configuration has an unsupported or missing schema version' [[ ${cfg[CORE_REF]:-} =~ ^[0-9a-f]{40}$ ]] || die 'configuration is missing a valid core revision' +# A concurrent upgrade may switch current between ExecStart resolution and the +# lock; re-exec the release matching the active policy before trusting it. +own_realpath=$(realpath -e -- "${BASH_SOURCE[0]}") +if [[ $own_realpath == */releases/* ]]; then + install_prefix=${own_realpath%%/releases/*} + own_release=${own_realpath#*/releases/} + own_release=${own_release%%/*} + if [[ $own_release != "${cfg[CORE_REF]}" ]]; then + selected=$install_prefix/releases/${cfg[CORE_REF]}/scripts/deployer-runtime.sh + [[ -x $selected && ! -L $selected ]] || die 'active policy revision runtime is unavailable' + [[ -z ${CI_FLEET_DEPLOYER_REEXEC:-} ]] || die 'runtime re-exec did not select the active revision' + export CI_FLEET_DEPLOYER_REEXEC=1 + exec "$selected" "$operation" + fi +fi [[ ${cfg[ENVIRONMENT]} =~ ^[a-z][a-z0-9-]{0,31}$ && ${cfg[TARGET_ID]} =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || die 'invalid environment or target identity' [[ ${cfg[ADAPTER_SHA256]} =~ ^[0-9a-f]{64}$ ]] || die 'invalid adapter digest' inside "${cfg[ADAPTER_PATH]}" "$(root_path /etc/ci-fleet-deployer/adapters)" || die 'application adapter is outside the protected adapter directory' @@ -328,7 +343,10 @@ PY [[ -z "$retired_snapshot" || "$retired_snapshot" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'current deployed snapshot pointer is unsafe' ln -s "${snapshot##*/}" "$pointer" [[ -z ${CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF:-} || $testing != 1 ]] || kill -"$CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF" $$ + sync -f "$snapshot/policy.conf" "$snapshot/state.json" 2>/dev/null || true + sync -f "$snapshot" 2>/dev/null || sync "$snapshot" 2>/dev/null || true mv -Tf "$pointer" "$deployed_current" + sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || true snapshot_pointer=$deployed_current snapshot= if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 009aebb..4716b09 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -90,6 +90,17 @@ ok = (all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) and policy.get('SCHEMA_VERSION') == '1') raise SystemExit(0 if ok else 1) PY + local rollback_adapter_path rollback_adapter_sha rollback_credential_provider rollback_credential_ref + rollback_adapter_path=$(awk -F= '$1=="ADAPTER_PATH" {print $2}' "$previous_policy") + rollback_adapter_sha=$(awk -F= '$1=="ADAPTER_SHA256" {print $2}' "$previous_policy") + [[ $rollback_adapter_path == "$etc_root/adapters/"* ]] || { printf no; return; } + [[ ! -L "$rollback_adapter_path" && -f "$rollback_adapter_path" && $(stat -c '%u:%a' "$rollback_adapter_path" 2>/dev/null) == "$expected_uid:700" ]] || { printf no; return; } + [[ $(sha256sum "$rollback_adapter_path" 2>/dev/null | cut -d' ' -f1) == "$rollback_adapter_sha" ]] || { printf no; return; } + rollback_credential_provider=$(awk -F= '$1=="CREDENTIAL_PROVIDER" {print $2}' "$previous_policy") + rollback_credential_ref=$(awk -F= '$1=="CREDENTIAL_REF" {print $2}' "$previous_policy") + if [[ $rollback_credential_provider == file ]]; then + [[ ! -L "$rollback_credential_ref" && -f "$rollback_credential_ref" && $(stat -c '%u:%a' "$rollback_credential_ref" 2>/dev/null) == "$expected_uid:600" ]] || { printf no; return; } + fi printf yes } die() { error_reported=1; printf 'ERROR: %s\n' "$*" >&2; report FAILED no inspect-and-retry "$(rollback_available)" >&2; exit 2; } @@ -472,6 +483,7 @@ units_match() { for unit in "${unit_names[@]}"; do [[ -f "$systemd_root/$unit" && ! -L "$systemd_root/$unit" && $(stat -c '%u:%a' "$systemd_root/$unit") == "$expected_uid:644" ]] || return 1 cmp -s "$unit_source/$unit" "$systemd_root/$unit" || return 1 + [[ ! -e "$systemd_root/$unit.d" && ! -L "$systemd_root/$unit.d" ]] || return 1 done for unit in "${timer_names[@]}"; do [[ $(systemctl is-enabled "$unit" 2>/dev/null) == enabled ]] && systemctl is-active "$unit" >/dev/null 2>&1 || return 1; done } @@ -544,6 +556,7 @@ begin_transaction() { [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' systemd_mode=$(stat -c %a "$systemd_root") (((8#$systemd_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' + [[ -d "$install_root" && ! -L "$install_root" && $(stat -c '%u:%a' "$install_root") == "$expected_uid:755" ]] || block 'managed install boundary has an unsafe owner, mode, or type' for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do path=$state_root/$name [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:600" ]] || block 'managed transaction state has an unsafe type, owner, or mode' @@ -591,13 +604,15 @@ begin_transaction() { fi transaction_name=${transaction_dir##*/} transaction_ready=$state_root/.transaction.${transaction_name#.transaction-preparing.} + sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || true mv "$transaction_dir" "$transaction_ready" transaction_dir=$transaction_ready + sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || true transaction_preparing=0 } restore_transaction() { - local name target_value backed_up + local name target_value backed_up systemd_restore_mode [[ -n ${transaction_dir:-} && -d $transaction_dir ]] || return 0 transaction_committed=1 for name in units-present state-present timers-enabled timers-enabled-runtime timers-active current-target deployed-target deployed-created; do @@ -642,6 +657,9 @@ restore_transaction() { fi done [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' + systemd_restore_mode=$(stat -c %a "$systemd_root") + (((8#$systemd_restore_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' + [[ -d "$install_root" && ! -L "$install_root" && $(stat -c '%u:%a' "$install_root") == "$expected_uid:755" ]] || block 'managed install boundary has an unsafe owner, mode, or type' for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name" || return; done if [[ -f "$transaction_dir/units-present" ]]; then while IFS= read -r name; do diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 2fa4c05..4e27b22 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -885,10 +885,35 @@ chmod 0777 "$root/etc/systemd/system" units_before_writable=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) expect_failure 'systemd unit directory is group- or world-writable' "$installer" --repair --config "$config" >/dev/null [[ $units_before_writable == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'writable systemd boundary allowed a transaction backup' +boundary_tx=$root/var/lib/ci-fleet-deployer/.transaction.writable-recovery +mkdir -m 0700 "$boundary_tx" "$boundary_tx/units" "$boundary_tx/state" +expect_failure 'systemd unit directory is group- or world-writable' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_writable == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'writable systemd boundary allowed a recovery restore' +rm -rf -- "$boundary_tx" chmod 0755 "$root/etc/systemd/system" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null +# A symlinked install root must block before recovery restores pointers. +boundary_tx=$root/var/lib/ci-fleet-deployer/.transaction.install-boundary +mkdir -m 0700 "$boundary_tx" "$boundary_tx/units" "$boundary_tx/state" +mv "$root/opt/ci-fleet-deployer" "$tmp/install-root.real" +ln -s "$tmp/install-root.real" "$root/opt/ci-fleet-deployer" +expect_failure 'managed install boundary has an unsafe owner, mode, or type' "$installer" --repair --config "$config" >/dev/null +[[ -L "$tmp/install-root.real/current" ]] || fail 'symlinked install boundary mutated the activation pointer' +rm "$root/opt/ci-fleet-deployer" +mv "$tmp/install-root.real" "$root/opt/ci-fleet-deployer" +rm -rf -- "$boundary_tx" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A managed-unit drop-in must break convergence until removed. +mkdir "$root/etc/systemd/system/ci-fleet-deployer.service.d" +printf '[Service]\nExecStart=\n' >"$root/etc/systemd/system/ci-fleet-deployer.service.d/override.conf" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +rm -rf "$root/etc/systemd/system/ci-fleet-deployer.service.d" +expect_success "$installer" --check --config "$config" >/dev/null + # Rebuild a retained rollback pair consumed by the finalize recovery fixture. image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' write_evidence @@ -938,6 +963,18 @@ grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'drifted retained chmod 0600 "$root/var/lib/ci-fleet-deployer/last-known-good.json" available_check=$(expect_success "$installer" --check --config "$config") grep -Fq 'rollback_available=yes' <<<"$available_check" || fail 'valid retained pair was not reported rollback_available=yes' +cp "$adapter" "$tmp/adapter.avail-saved" +printf '# drifted\n' >>"$adapter" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'digest-drifted adapter still reported rollback_available=yes' +cat "$tmp/adapter.avail-saved" >"$adapter" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=yes' <<<"$available_check" || fail 'restored adapter was not reported rollback_available=yes' +chmod 0644 "$credential" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'drifted credential still reported rollback_available=yes' +chmod 0600 "$credential" +expect_success "$installer" --check --config "$config" >/dev/null write_production_gate write_evidence production example-production From 26c468e16b2054c92ebd588bbc5258a3cb2fd3fd Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 21:41:27 -0500 Subject: [PATCH 46/90] test: install-root symlink is caught by the existing symlink guard --- scripts/test-install-deployer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 4e27b22..ed66f70 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -899,7 +899,7 @@ boundary_tx=$root/var/lib/ci-fleet-deployer/.transaction.install-boundary mkdir -m 0700 "$boundary_tx" "$boundary_tx/units" "$boundary_tx/state" mv "$root/opt/ci-fleet-deployer" "$tmp/install-root.real" ln -s "$tmp/install-root.real" "$root/opt/ci-fleet-deployer" -expect_failure 'managed install boundary has an unsafe owner, mode, or type' "$installer" --repair --config "$config" >/dev/null +expect_failure 'unsafe symlinked managed directory' "$installer" --repair --config "$config" >/dev/null [[ -L "$tmp/install-root.real/current" ]] || fail 'symlinked install boundary mutated the activation pointer' rm "$root/opt/ci-fleet-deployer" mv "$tmp/install-root.real" "$root/opt/ci-fleet-deployer" From 6a696123f017e00c8d3a8dd3fe2f838fa7a33000 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 21:57:59 -0500 Subject: [PATCH 47/90] test: availability regression uses a drifted retained policy digest --- scripts/test-install-deployer.sh | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index ed66f70..734de81 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -963,17 +963,21 @@ grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'drifted retained chmod 0600 "$root/var/lib/ci-fleet-deployer/last-known-good.json" available_check=$(expect_success "$installer" --check --config "$config") grep -Fq 'rollback_available=yes' <<<"$available_check" || fail 'valid retained pair was not reported rollback_available=yes' -cp "$adapter" "$tmp/adapter.avail-saved" -printf '# drifted\n' >>"$adapter" -available_check=$(expect_success "$installer" --check --config "$config") -grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'digest-drifted adapter still reported rollback_available=yes' -cat "$tmp/adapter.avail-saved" >"$adapter" -available_check=$(expect_success "$installer" --check --config "$config") -grep -Fq 'rollback_available=yes' <<<"$available_check" || fail 'restored adapter was not reported rollback_available=yes' -chmod 0644 "$credential" +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ADAPTER_SHA256=', 'ADAPTER_SHA256=' + 'f'*64 + '\n#', 1)) +PY available_check=$(expect_success "$installer" --check --config "$config") -grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'drifted credential still reported rollback_available=yes' -chmod 0600 "$credential" +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'digest-mismatched retained adapter still reported rollback_available=yes' +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null write_production_gate From b0379387e82c3373366d8bdb14ffe67e3c4667a4 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Tue, 11 Aug 2026 22:32:59 -0500 Subject: [PATCH 48/90] fix: hook-free checkout validation, backup guards, durable state transitions - Disable fsmonitor and repository hooks for every Git invocation in checkout validation so unreviewed checkout config cannot execute as root. - Reject group/world-writable managed units before transaction backup and reject unreviewed drop-in overrides during install and repair. - fsync the drain marker, committed host state before journal retirement, the deployed snapshot publication, and the success audit record, treating pointer/audit durability failures as fatal before any retirement. - Finalize a committed rollback over an unusable incumbent deployed snapshot instead of validating the snapshot being retired. --- scripts/deployer-runtime.sh | 9 ++++++--- scripts/install-deployer.sh | 27 +++++++++++++++++-------- scripts/test-install-deployer.sh | 34 ++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index fde30b2..6cb496f 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -149,6 +149,8 @@ if [[ "$operation" == drain ]]; then temporary=$(mktemp "$state_root/.drained.XXXXXX") chmod 0600 "$temporary" mv -Tf "$temporary" "$drained" + sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || true + sync -f "$state_root" 2>/dev/null || true exit 0 fi @@ -343,10 +345,10 @@ PY [[ -z "$retired_snapshot" || "$retired_snapshot" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'current deployed snapshot pointer is unsafe' ln -s "${snapshot##*/}" "$pointer" [[ -z ${CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF:-} || $testing != 1 ]] || kill -"$CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF" $$ - sync -f "$snapshot/policy.conf" "$snapshot/state.json" 2>/dev/null || true - sync -f "$snapshot" 2>/dev/null || sync "$snapshot" 2>/dev/null || true + sync -f "$snapshot/policy.conf" "$snapshot/state.json" 2>/dev/null || die 'prepared deployed snapshot is not durable' + sync -f "$snapshot" 2>/dev/null || sync "$snapshot" 2>/dev/null || die 'prepared deployed snapshot is not durable' mv -Tf "$pointer" "$deployed_current" - sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || true + sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'deployed snapshot pointer is not durable' snapshot_pointer=$deployed_current snapshot= if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi @@ -360,6 +362,7 @@ PY "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ >&8 + sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || die 'deployment success audit is not durable' audit_pending=0 ;; esac diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 4716b09..29ddb5e 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -401,10 +401,11 @@ reject_mixed_role() { validate_checkout() { local head - head=$(git -C "$repo_root" --no-replace-objects rev-parse 'HEAD^{commit}') || block 'installer checkout is not Git-authored' + git_checkout() { git -c core.fsmonitor= -c core.hooksPath=/dev/null -C "$repo_root" "$@"; } + head=$(git_checkout --no-replace-objects rev-parse 'HEAD^{commit}') || block 'installer checkout is not Git-authored' [[ "$head" == "$core_ref" ]] || block 'CORE_REF must equal the exact reviewed checkout HEAD' if [[ "$testing" != 1 ]]; then - git -C "$repo_root" diff --quiet HEAD -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer || block 'reviewed deployer inputs differ from HEAD' + git_checkout diff --quiet HEAD -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer || block 'reviewed deployer inputs differ from HEAD' fi # Pin the reviewed inputs before any privileged copy: copy the worktree # bytes once into a root-controlled snapshot outside managed state, then @@ -417,7 +418,7 @@ validate_checkout() { install -d -m 0700 "$checkout_snapshot/scripts" "$checkout_snapshot/deploy/deployer" install -m 0755 "$repo_root/scripts/install-deployer.sh" "$repo_root/scripts/deployer-runtime.sh" "$checkout_snapshot/scripts/" install -m 0644 "$repo_root"/deploy/deployer/* "$checkout_snapshot/deploy/deployer/" - tree_listing=$(git -C "$repo_root" --no-replace-objects ls-tree -r "$head" -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer) || block 'reviewed commit tree is unreadable' + tree_listing=$(git_checkout --no-replace-objects ls-tree -r "$head" -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer) || block 'reviewed commit tree is unreadable' [[ -n $tree_listing ]] || block 'reviewed commit tree is unreadable' while read -r _ _ blob path; do [[ $blob == "$(git hash-object "$checkout_snapshot/$path")" ]] || block "checkout input $path differs from the reviewed commit" @@ -568,7 +569,7 @@ begin_transaction() { fi for name in "${unit_names[@]}"; do path=$systemd_root/$name - [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c %u "$path") == "$expected_uid" ]] || block 'managed systemd unit has an unsafe owner or type' + [[ ! -e "$path" && ! -L "$path" ]] || [[ -f "$path" && ! -L "$path" && $(stat -c '%u:%a' "$path") == "$expected_uid:644" ]] || block 'managed systemd unit has an unsafe owner, mode, or type' done transaction_dir=$(mktemp -d "$state_root/.transaction-preparing.XXXXXX") transaction_preparing=1 @@ -724,13 +725,13 @@ recover_interrupted_transaction() { candidate=${candidates[0]} [[ ! -L "$candidate" && -d "$candidate" && $(stat -c '%u:%a' "$candidate") == "$expected_uid:700" ]] || block 'interrupted installer transaction is unsafe' transaction_dir=$candidate - if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then load_deployed_snapshot; fi if [[ -e "$transaction_dir/application-rollback-committed" || -L "$transaction_dir/application-rollback-committed" ]]; then finalize_committed_rollback recovered_rollback=1 transaction_committed=0 return fi + if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then load_deployed_snapshot; fi restore_transaction transaction_committed=0 } @@ -796,6 +797,9 @@ PY commit_transaction() { local retired + sync -f "$state_file" "$active_policy" 2>/dev/null || true + sync -f "$state_root" 2>/dev/null || true + sync -f "$install_root" 2>/dev/null || true retired=$state_root/.retired.$$.transaction mv -Tf "$transaction_dir" "$retired" || return transaction_dir= @@ -918,11 +922,15 @@ PY } publish_deployed_snapshot() { - local policy=$1 state=$2 snapshot pointer retired + local policy=$1 state=$2 snapshot pointer retired incumbent secure_directory "$deployed_root" 700 1 || return if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then - load_deployed_snapshot || return - if cmp -s "$deployed_snapshot_policy" "$policy" && cmp -s "$deployed_snapshot_state" "$state"; then return; fi + # An unusable incumbent snapshot must not block publication of a freshly + # validated rollback pair; only identical bytes short-circuit. + if [[ -L "$deployed_current" ]]; then + incumbent=$(readlink -f "$deployed_current") + if [[ -f "$incumbent/policy.conf" && -f "$incumbent/state.json" ]] && cmp -s "$incumbent/policy.conf" "$policy" && cmp -s "$incumbent/state.json" "$state"; then return; fi + fi retired=$(readlink "$deployed_current") [[ "$retired" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'current deployed snapshot pointer is unsafe' rm -f -- "$deployed_current" || return @@ -943,6 +951,7 @@ install_units() { systemd_mode=$(stat -c %a "$systemd_root") (((8#$systemd_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' for unit in "${unit_names[@]}"; do + [[ ! -e "$systemd_root/$unit.d" && ! -L "$systemd_root/$unit.d" ]] || block "managed unit $unit has an unreviewed drop-in override" if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then [[ ! -L "$systemd_root/$unit" && -f "$systemd_root/$unit" && $(stat -c %u "$systemd_root/$unit") == "$expected_uid" ]] || block 'managed systemd unit has an unsafe owner or type' fi @@ -1144,6 +1153,8 @@ perform_drain() { temporary=$(mktemp "$state_root/.drained.XXXXXX") chmod 0600 "$temporary" mv -Tf "$temporary" "$drained" + sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || true + sync -f "$state_root" 2>/dev/null || true report CHANGED yes safe-to-maintain "$(rollback_available)" } diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 734de81..813df34 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -799,6 +799,27 @@ expect_failure 'application rollback commit marker is unsafe' "$installer" --rep rm -rf -- "$recovery_transaction" expect_success "$installer" --check --config "$config" >/dev/null +# Finalizing a committed rollback must publish over an unusable incumbent deployed snapshot. +deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") +cp "$deployed_dir/policy.conf" "$tmp/incumbent-policy.saved" +python3 - "$deployed_dir/policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ADAPTER_SHA256=', 'ADAPTER_SHA256=' + 'e'*64 + '\n#', 1)) +PY +recovery_transaction=$root/var/lib/ci-fleet-deployer/.transaction.unusable-incumbent +mkdir -m 0700 "$recovery_transaction" "$recovery_transaction/units" "$recovery_transaction/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$recovery_transaction/state/$name"; printf '%s\n' "$name" >>"$recovery_transaction/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$recovery_transaction/current-target" +install -m 0600 /dev/null "$recovery_transaction/application-rollback-committed" +recovery=$(expect_success "$installer" --upgrade --config "$config") +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$recovery" || fail 'unusable-incumbent recovery lacks the operator reconciliation action' +[[ ! -e "$recovery_transaction" ]] || fail 'unusable-incumbent recovery retained its transaction' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + # A deployed rollback pair whose adapter bytes no longer match its recorded digest must not promote. deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") deployed_target=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") @@ -911,7 +932,20 @@ expect_success "$installer" --check --config "$config" >/dev/null mkdir "$root/etc/systemd/system/ci-fleet-deployer.service.d" printf '[Service]\nExecStart=\n' >"$root/etc/systemd/system/ci-fleet-deployer.service.d/override.conf" expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +expect_failure 'has an unreviewed drop-in override' "$installer" --repair --config "$config" >/dev/null +[[ -f "$root/etc/systemd/system/ci-fleet-deployer.service.d/override.conf" ]] || fail 'repair discarded an unmanaged drop-in' rm -rf "$root/etc/systemd/system/ci-fleet-deployer.service.d" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A group- or world-writable managed unit must block before transaction backup. +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" +units_before_mode=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) +expect_failure 'managed systemd unit has an unsafe owner, mode, or type' "$installer" --repair --config "$config" >/dev/null +[[ $units_before_mode == "$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'writable unit allowed a transaction backup' +chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" +expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null # Rebuild a retained rollback pair consumed by the finalize recovery fixture. From 5945492faac57f69603c97e4f0f5096d7c14cbfd Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 15:36:53 -0500 Subject: [PATCH 49/90] test: rebuild retained rollback pair before digest/credential drift fixtures The unusable-incumbent finalize fixture consumes the retained last-known-good pair; rebuild it before the deployed-pair drift fixtures so their rollback expectations still reach the deployed snapshot guards. Match the unified unit-guard block message, assert repair blocks before transaction backup while a unit is mode 0666 and succeeds after an explicit test-only restore to 0644, include the failing line in fail output, and drop trailing whitespace. --- scripts/deployer-runtime.sh | 2 +- scripts/test-install-deployer.sh | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 6cb496f..66172c9 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -348,7 +348,7 @@ PY sync -f "$snapshot/policy.conf" "$snapshot/state.json" 2>/dev/null || die 'prepared deployed snapshot is not durable' sync -f "$snapshot" 2>/dev/null || sync "$snapshot" 2>/dev/null || die 'prepared deployed snapshot is not durable' mv -Tf "$pointer" "$deployed_current" - sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'deployed snapshot pointer is not durable' + sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'deployed snapshot pointer is not durable' snapshot_pointer=$deployed_current snapshot= if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 813df34..5cad356 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -5,7 +5,7 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) installer=$repo_root/scripts/install-deployer.sh runtime=$repo_root/scripts/deployer-runtime.sh -fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +fail() { printf 'FAIL(line %s): %s\n' "${BASH_LINENO[0]:-?}" "$*" >&2; exit 1; } expect_success() { local output output=$("$@" 2>&1) || fail "expected success: $*; output=$output" @@ -360,6 +360,8 @@ grep -Fq 'REPORT action=check result=NO_CHANGE' <<<"$check" || fail 'check did n FAKE_SYSTEMD_STATE=degraded expect_success "$installer" --check --config "$config" >/dev/null chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +expect_failure 'managed systemd unit has an unsafe owner, mode, or type' "$installer" --repair --config "$config" >/dev/null +chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_success "$installer" --repair --config "$config" >/dev/null [[ $(stat -c %a "$root/etc/systemd/system/ci-fleet-deployer.service") == 644 ]] || fail 'repair did not restore unit mode 0644' @@ -579,7 +581,7 @@ unit_path=$root/etc/systemd/system/ci-fleet-deployer.service mv "$unit_path" "$unit_path.real" printf 'unrelated-unit\n' >"$tmp/unrelated-unit" ln -s "$tmp/unrelated-unit" "$unit_path" -expect_failure 'managed systemd unit has an unsafe owner or type' "$installer" --repair --config "$config" >/dev/null +expect_failure 'managed systemd unit has an unsafe owner, mode, or type' "$installer" --repair --config "$config" >/dev/null [[ $(<"$tmp/unrelated-unit") == unrelated-unit ]] || fail 'systemd unit symlink attack changed an unrelated file' rm "$unit_path"; mv "$unit_path.real" "$unit_path" @@ -821,6 +823,17 @@ expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null # A deployed rollback pair whose adapter bytes no longer match its recorded digest must not promote. +# Rebuild the retained pair consumed by the unusable-incumbent recovery fixture. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") deployed_target=$(readlink "$root/var/lib/ci-fleet-deployer/deployed/current") cp "$deployed_dir/policy.conf" "$tmp/deployed-policy.saved" @@ -1040,6 +1053,7 @@ mv "$fake_bin/docker.unavailable" "$fake_bin/docker" chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_failure 'installed deployer state is absent or drifted; repair before resume' "$installer" --resume --config "$config" >/dev/null [[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'failed resume removed drain state' +chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_success "$installer" --repair --config "$config" >/dev/null [[ -f "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'repair implicitly resumed the deployer' resume=$(expect_success "$installer" --resume --config "$config") @@ -1048,6 +1062,7 @@ grep -Fq 'health=healthy' <<<"$resume" || fail 'resume report omitted verified h [[ ! -e "$root/var/lib/ci-fleet-deployer/drained" ]] || fail 'resume retained drain state' chmod 0666 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_failure 'installed deployer state is absent or drifted; repair before resume' "$installer" --resume --config "$config" >/dev/null +chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_success "$installer" --repair --config "$config" >/dev/null repeat_resume=$(expect_success "$installer" --resume --config "$config") grep -Fq 'result=NO_CHANGE' <<<"$repeat_resume" || fail 'repeated resume was not idempotent' From 48c6c687f0f1f2d79f7e416252489da6930cb253 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 16:41:16 -0500 Subject: [PATCH 50/90] test: align rollback fixtures with publish-over-unusable-incumbent semantics Rollback now intentionally publishes the retained pair over an unusable incumbent deployed snapshot, so the digest- and credential-drifted deployed-pair fixtures assert retirement with the operator reconciliation action instead of a block, and rebuild the consumed last-known-good pair before later rollback fixtures. --- scripts/test-install-deployer.sh | 55 ++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 5cad356..c20ffc3 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -7,14 +7,14 @@ runtime=$repo_root/scripts/deployer-runtime.sh fail() { printf 'FAIL(line %s): %s\n' "${BASH_LINENO[0]:-?}" "$*" >&2; exit 1; } expect_success() { - local output - output=$("$@" 2>&1) || fail "expected success: $*; output=$output" + local output line=${BASH_LINENO[0]} + output=$("$@" 2>&1) || { printf 'FAIL(line %s): expected success: %s; output=%s\n' "$line" "$*" "$output" >&2; exit 1; } printf '%s\n' "$output" } expect_failure() { - local expected=$1 output + local expected=$1 output line=${BASH_LINENO[0]} shift - if output=$("$@" 2>&1); then fail "expected failure: $*"; fi + if output=$("$@" 2>&1); then printf 'FAIL(line %s): expected failure: %s; output=%s\n' "$line" "$*" "$output" >&2; exit 1; fi grep -Fq -- "$expected" <<<"$output" || fail "missing [$expected]: $output" printf '%s\n' "$output" } @@ -846,16 +846,24 @@ if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_bef expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --upgrade --config "$config" >/dev/null lkg_after_digest=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_digest=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_digest == "$lkg_after_digest" ]] || fail 'digest-mismatched deployed pair replaced last-known-good' if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'blocked digest promotion left a recovery transaction'; fi -expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --rollback --config "$config" >/dev/null -rollback_tx=$(compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*") || fail 'blocked digest rollback left no recovery transaction' -[[ $(<"$rollback_tx/deployed-target") == "$deployed_target" ]] || fail 'blocked digest rollback recorded an unexpected deployed target' -expect_failure 'deployed rollback adapter digest does not match its snapshot policy' "$installer" --repair --config "$config" >/dev/null -lkg_after_digest=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_digest=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_digest == "$lkg_after_digest" ]] || fail 'digest-mismatched deployed pair reached recovery promotion' -install -m 0600 "$tmp/deployed-policy.saved" "$deployed_dir/policy.conf" -expect_success "$installer" --repair --config "$config" >/dev/null +# Rollback intentionally publishes the retained pair over the unusable +# incumbent snapshot: the drifted deployed pair is retired, never promoted. +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$rollback" || fail 'drifted-incumbent rollback lacks the operator reconciliation action' +[[ ! -e "$deployed_dir" ]] || fail 'drifted deployed snapshot survived its rollback retirement' expect_success "$installer" --repair --config "$config" >/dev/null -if compgen -G "$root/var/lib/ci-fleet-deployer/.transaction.*" >/dev/null; then fail 'digest fixture restoration left a recovery transaction'; fi expect_success "$installer" --check --config "$config" >/dev/null +# Rebuild the retained pair consumed by the drifted-incumbent rollback. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' # A deployed rollback pair whose credential reference drifted must not promote. deployed_dir=$(readlink -f "$root/var/lib/ci-fleet-deployer/deployed/current") @@ -868,10 +876,11 @@ PY if [[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]]; then lkg_before_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); else lkg_before_credential=absent; fi expect_failure 'deployed rollback policy has an invalid external secret-manager adapter reference' "$installer" --upgrade --config "$config" >/dev/null lkg_after_credential=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_credential == "$lkg_after_credential" ]] || fail 'credential-drifted deployed pair replaced last-known-good' -expect_failure 'deployed rollback policy has an invalid external secret-manager adapter reference' "$installer" --rollback --config "$config" >/dev/null -lkg_after_credential=absent; [[ ! -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || lkg_after_credential=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json"); [[ $lkg_before_credential == "$lkg_after_credential" ]] || fail 'credential-drifted deployed pair reached rollback' -install -m 0600 "$tmp/deployed-policy-cred.saved" "$deployed_dir/policy.conf" -expect_success "$installer" --repair --config "$config" >/dev/null +# Rollback intentionally publishes the retained pair over the unusable +# incumbent snapshot: the credential-drifted pair is retired, never promoted. +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'next=restore-host-policy-evidence-then-check' <<<"$rollback" || fail 'credential-drifted rollback lacks the operator reconciliation action' +[[ ! -e "$deployed_dir" ]] || fail 'credential-drifted deployed snapshot survived its rollback retirement' expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null @@ -961,7 +970,8 @@ chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null -# Rebuild a retained rollback pair consumed by the finalize recovery fixture. +# Rebuild a retained rollback pair consumed by the drifted-incumbent rollback +# and finalize recovery fixtures. image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' write_evidence write_config @@ -986,6 +996,17 @@ rollback=$(expect_success "$installer" --rollback --config "$config") grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'config-independent rollback did not report change' mv "$tmp/config.saved" "$config"; chmod 0600 "$config" expect_success "$installer" --repair --config "$config" >/dev/null +# Rebuild the retained pair consumed by the config-independent rollback. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' printf 'malformed line\n' >"$config"; chmod 0600 "$config" rollback=$(expect_success "$installer" --rollback --config "$config") grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'malformed-config rollback did not report change' From 4a5e2f3b75c5952abe5b1085dadd4c0f0c42d6c2 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 17:45:22 -0500 Subject: [PATCH 51/90] fix: durable rollback publication, boundary-ordered recovery, exact checkout snapshots - fsync the replacement deployed snapshot and pointer before retiring the incumbent, last-known-good pair, or recovery journal; make committed-state synchronization failures fatal and include the systemd unit boundary. - Validate the install-root boundary before uninstall removes the activation pointer, and validate the systemd boundary before recovery stops timers. - rollback_available now requires the retained credential reference to stay inside the protected credentials directory or match the external-provider grammar. - Checkout validation rejects unsafe or untracked unit-source entries and requires the pinned snapshot to match the reviewed tree exactly. - Publish the candidate activation pointer before unit verification so fresh installs resolve current/... ExecStart paths; transaction recovery restores the prior pointer. - The runtime masks INT/TERM once the adapter has succeeded until the new snapshot pointer and success audit record are durable. --- scripts/deployer-runtime.sh | 4 ++++ scripts/install-deployer.sh | 48 ++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 66172c9..defc3f6 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -333,6 +333,10 @@ PY fi adapter_status= audit_phase=post-adapter + # The adapter change is applied and the approval is consumed; a signal from + # here on must not abort before the new snapshot pointer is durable and the + # success audit record is written, or later rollback would use stale state. + trap '' INT TERM secure_directory "$deployed_root" 'deployed snapshot directory' inside "$snapshot" "$deployed_root" || die 'prepared deployed snapshot escaped managed state' secure_directory "$snapshot" 'prepared deployed snapshot' diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 29ddb5e..ab8e903 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -99,7 +99,12 @@ PY rollback_credential_provider=$(awk -F= '$1=="CREDENTIAL_PROVIDER" {print $2}' "$previous_policy") rollback_credential_ref=$(awk -F= '$1=="CREDENTIAL_REF" {print $2}' "$previous_policy") if [[ $rollback_credential_provider == file ]]; then + [[ $rollback_credential_ref == "$etc_root/credentials/"* ]] || { printf no; return; } [[ ! -L "$rollback_credential_ref" && -f "$rollback_credential_ref" && $(stat -c '%u:%a' "$rollback_credential_ref" 2>/dev/null) == "$expected_uid:600" ]] || { printf no; return; } + elif [[ $rollback_credential_provider == external ]]; then + [[ $rollback_credential_ref =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || { printf no; return; } + else + printf no; return fi printf yes } @@ -417,12 +422,21 @@ validate_checkout() { chmod 0700 "$checkout_snapshot" install -d -m 0700 "$checkout_snapshot/scripts" "$checkout_snapshot/deploy/deployer" install -m 0755 "$repo_root/scripts/install-deployer.sh" "$repo_root/scripts/deployer-runtime.sh" "$checkout_snapshot/scripts/" - install -m 0644 "$repo_root"/deploy/deployer/* "$checkout_snapshot/deploy/deployer/" + local entry + for entry in "$repo_root"/deploy/deployer/*; do + [[ -f "$entry" && ! -L "$entry" ]] || block 'deployer unit source contains an unsafe or untracked entry' + install -m 0644 "$entry" "$checkout_snapshot/deploy/deployer/" + done tree_listing=$(git_checkout --no-replace-objects ls-tree -r "$head" -- scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer) || block 'reviewed commit tree is unreadable' [[ -n $tree_listing ]] || block 'reviewed commit tree is unreadable' while read -r _ _ blob path; do [[ $blob == "$(git hash-object "$checkout_snapshot/$path")" ]] || block "checkout input $path differs from the reviewed commit" done <<<"$tree_listing" + # The snapshot must contain exactly the reviewed entries: any extra copied + # path (for example an untracked symlink target) is unreviewed content. + local snapshot_listing + snapshot_listing=$(cd "$checkout_snapshot" && find scripts deploy -type f | sort) + [[ $snapshot_listing == "$(awk '{print $4}' <<<"$tree_listing" | sort)" ]] || block 'checkout snapshot contains unreviewed entries' repo_root=$checkout_snapshot unit_source=$repo_root/deploy/deployer } @@ -652,15 +666,15 @@ restore_transaction() { [[ -z $backed_up ]] || block "transaction $name manifest is missing but backups remain" fi done + [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' + systemd_restore_mode=$(stat -c %a "$systemd_root") + (((8#$systemd_restore_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' + [[ -d "$install_root" && ! -L "$install_root" && $(stat -c '%u:%a' "$install_root") == "$expected_uid:755" ]] || block 'managed install boundary has an unsafe owner, mode, or type' for name in "${timer_names[@]}"; do if [[ -e "$systemd_root/$name" || -L "$systemd_root/$name" ]]; then systemctl disable --now "$name" >/dev/null 2>&1 || return fi done - [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' - systemd_restore_mode=$(stat -c %a "$systemd_root") - (((8#$systemd_restore_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' - [[ -d "$install_root" && ! -L "$install_root" && $(stat -c '%u:%a' "$install_root") == "$expected_uid:755" ]] || block 'managed install boundary has an unsafe owner, mode, or type' for name in "${unit_names[@]}"; do rm -f -- "$systemd_root/$name" || return; done if [[ -f "$transaction_dir/units-present" ]]; then while IFS= read -r name; do @@ -797,9 +811,10 @@ PY commit_transaction() { local retired - sync -f "$state_file" "$active_policy" 2>/dev/null || true - sync -f "$state_root" 2>/dev/null || true - sync -f "$install_root" 2>/dev/null || true + sync -f "$state_file" "$active_policy" 2>/dev/null || block 'committed host state is not durable' + sync -f "$state_root" 2>/dev/null || block 'committed host state is not durable' + sync -f "$install_root" 2>/dev/null || block 'committed install root is not durable' + sync -f "$systemd_root" 2>/dev/null || block 'committed systemd boundary is not durable' retired=$state_root/.retired.$$.transaction mv -Tf "$transaction_dir" "$retired" || return transaction_dir= @@ -941,7 +956,10 @@ publish_deployed_snapshot() { install -m 0600 "$state" "$snapshot/state.json" || return pointer=$deployed_root/.current.$$ ln -s "${snapshot##*/}" "$pointer" || return + sync -f "$snapshot/policy.conf" "$snapshot/state.json" 2>/dev/null || block 'replacement deployed snapshot is not durable' + sync -f "$snapshot" 2>/dev/null || block 'replacement deployed snapshot is not durable' mv -Tf "$pointer" "$deployed_current" || return + sync -f "$deployed_root" 2>/dev/null || block 'deployed snapshot pointer is not durable' if [[ -n ${retired:-} && -d "$deployed_root/$retired" && ! -L "$deployed_root/$retired" ]]; then rm -rf -- "${deployed_root:?}/$retired"; fi } @@ -1081,13 +1099,16 @@ PY fi fi ln -sfn "releases/$core_ref" "$install_root/.current.new" + # Publish the candidate activation pointer before unit verification so the + # units' /opt/ci-fleet-deployer/current/... ExecStart paths resolve on a + # fresh install; transaction recovery restores the prior pointer. + mv -Tf "$install_root/.current.new" "$current" install_units install -m 0600 "$config" "$active_policy.new" write_state "$state_root/.install-state.new" reject_mixed_role mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_root/.install-state.new" "$state_file" - mv -Tf "$install_root/.current.new" "$current" policy_adapter_operation "$active_policy" health 'candidate policy' || die 'candidate health check failed after activation' if [[ ! -e "$deployed_current" && ! -L "$deployed_current" ]]; then publish_deployed_snapshot "$active_policy" "$state_file" @@ -1129,10 +1150,12 @@ PY release_complete "$releases/$core_ref" || die 'last-known-good release is incomplete' unit_source=$releases/$core_ref/deploy/deployer ln -sfn "releases/$core_ref" "$install_root/.current.new" + # Publish before unit verification so the units' current/... paths resolve; + # transaction recovery restores the prior pointer. + mv -Tf "$install_root/.current.new" "$current" install_units mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_file.new" "$state_file" - mv -Tf "$install_root/.current.new" "$current" reject_mixed_role policy_adapter_operation "$active_policy" rollback 'last-known-good policy' "$transaction_dir/application-rollback-committed" || die 'application adapter rollback failed' sync -f "$transaction_dir/application-rollback-committed" 2>/dev/null || sync "$transaction_dir/application-rollback-committed" 2>/dev/null || die 'application rollback commit marker is not durable' @@ -1193,7 +1216,10 @@ perform_uninstall() { systemctl disable --now "$unit" >/dev/null 2>&1 || block 'deployer timers did not stop during uninstall' fi done - if [[ -L "$current" ]]; then rm -f "$current"; changed=yes; fi + if [[ -L "$current" ]]; then + [[ -d "$install_root" && ! -L "$install_root" && $(stat -c '%u:%a' "$install_root") == "$expected_uid:755" ]] || block 'managed install boundary has an unsafe owner, mode, or type' + rm -f "$current"; changed=yes + fi [[ ! -e "$current" ]] || block 'activation pointer has an unsafe type' for unit in "${unit_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then rm -f "$systemd_root/$unit"; changed=yes; fi; done systemctl daemon-reload >/dev/null 2>&1 || true From e736b1fecc3464628e9376b302315408c616a957 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 18:15:30 -0500 Subject: [PATCH 52/90] test: round-ten regressions for durability, boundaries, and checkout pinning - Signal after adapter success now completes durable publication. - Uninstall over a symlinked install root blocks before touching the target's pointer. - Untracked or unsafe unit-source entries block checkout validation. - Retained policies with out-of-directory or malformed credential references report rollback_available=no. - Recovery with a writable systemd boundary blocks before stopping timers. --- scripts/test-install-deployer.sh | 80 +++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 6 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index c20ffc3..c5b0625 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -1127,6 +1127,70 @@ rmdir "$root/etc/systemd/system/ci-fleet-deployer-drain.service" expect_success "$installer" --uninstall --config "$config" >/dev/null [[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after correcting unit drift did not uninstall' +# A symlinked install root must block uninstall before its target's pointer is touched. +expect_success "$installer" --install --config "$config" >/dev/null +mv "$root/opt/ci-fleet-deployer" "$tmp/install-root-uninstall.real" +ln -s "$tmp/install-root-uninstall.real" "$root/opt/ci-fleet-deployer" +expect_failure 'unsafe symlinked managed directory' "$installer" --uninstall --config "$config" >/dev/null +[[ -L "$tmp/install-root-uninstall.real/current" ]] || fail 'uninstall removed a pointer through a symlinked install root' +rm "$root/opt/ci-fleet-deployer" +mv "$tmp/install-root-uninstall.real" "$root/opt/ci-fleet-deployer" +expect_success "$installer" --uninstall --config "$config" >/dev/null +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after correcting install-root drift did not uninstall' + +# An untracked symlink in the deployer unit source must block checkout validation. +expect_success "$installer" --install --config "$config" >/dev/null +ln -s "$credential" "$repo_root/deploy/deployer/leak" +expect_failure 'deployer unit source contains an unsafe or untracked entry' "$installer" --repair --config "$config" >/dev/null +[[ $(stat -c %a "$credential") == 600 ]] || fail 'untracked checkout symlink exposed credential bytes' +rm "$repo_root/deploy/deployer/leak" +printf 'unreviewed\n' >"$repo_root/deploy/deployer/extra-unit.service" +expect_failure 'checkout snapshot contains unreviewed entries' "$installer" --repair --config "$config" >/dev/null +rm "$repo_root/deploy/deployer/extra-unit.service" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + +# A retained policy naming a credential outside the protected directory, or a +# malformed external reference, must report rollback_available=no. +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" ]] || fail 'test setup expected a retained policy' +cp "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$tmp/lkg-policy.saved" +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('CREDENTIAL_PROVIDER=file', 'CREDENTIAL_PROVIDER=external').replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=external:bad')) +PY +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'malformed external retained credential still reported rollback_available=yes' +install -m 0600 "$tmp/lkg-policy.saved" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace(next(x for x in p.read_text().splitlines() if x.startswith('CREDENTIAL_REF=')), 'CREDENTIAL_REF=/etc/hostname')) +PY +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'out-of-directory retained credential still reported rollback_available=yes' +install -m 0600 "$tmp/lkg-policy.saved" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=yes' <<<"$available_check" || fail 'restored retained pair was not reported rollback_available=yes' + +# Retained-transaction recovery must validate the systemd boundary before stopping timers. +expect_success "$installer" --uninstall --config "$config" >/dev/null +expect_success "$installer" --install --config "$config" >/dev/null +boundary_recovery=$root/var/lib/ci-fleet-deployer/.transaction.writable-boundary +mkdir -m 0700 "$boundary_recovery" "$boundary_recovery/units" "$boundary_recovery/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$boundary_recovery/state/$name"; printf '%s\n' "$name" >>"$boundary_recovery/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$boundary_recovery/current-target" +printf '%s\n' ci-fleet-deployer-health.timer ci-fleet-deployer-cleanup.timer >"$boundary_recovery/timers-enabled" +chmod 0777 "$root/etc/systemd/system" +expect_failure 'systemd unit directory is group- or world-writable' "$installer" --repair --config "$config" >/dev/null +[[ $(stat -c %a "$root/etc/systemd/system") == 777 ]] || fail 'writable boundary recovery mutated the unit directory' +rm -rf -- "$boundary_recovery" +chmod 0755 "$root/etc/systemd/system" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + # A failed timer shutdown must fail the uninstall before any unit removal. expect_success "$installer" --install --config "$config" >/dev/null uninstall_before=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) @@ -1281,14 +1345,18 @@ expect_failure 'GitHub capability evidence must be a regular file' "$runtime" de [[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == "$deploy_calls_before" ]] || fail 'deployment ran without GitHub capability evidence' mv "$capability.saved" "$capability" deploy_calls_before=$(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) -if CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF=TERM "$runtime" deploy >/dev/null 2>&1; then fail 'signaled deployment reported success'; fi -[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == $((deploy_calls_before + 1)) ]] || fail 'interrupted runtime did not reach the adapter' -[[ $(find "$root/var/lib/ci-fleet-deployer/deployed" -mindepth 1 -maxdepth 1 -name '.snapshot.*' -type d | wc -l) == 1 ]] || fail 'signal after temporary pointer leaked an unreachable prepared snapshot' -[[ -e "$deployed_current" ]] || fail 'signal after temporary pointer dangled the deployed pointer' -[[ ! -e "$active" ]] || fail 'signaled deployment left the active operation marker' -if compgen -G "$root/var/lib/ci-fleet-deployer/.active.*" >/dev/null; then fail 'signaled deployment left an unpublished active marker temporary'; fi +# A signal arriving after adapter success must not abort publication: the +# approval is consumed and the application change is applied, so the runtime +# masks INT/TERM until the new snapshot pointer and audit record are durable. +deployed_before_signal=$(readlink "$deployed_current") +CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF=TERM expect_success "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == $((deploy_calls_before + 1)) ]] || fail 'signaled runtime did not reach the adapter' +[[ -L "$deployed_current" && $(readlink "$deployed_current") != "$deployed_before_signal" ]] || fail 'masked signal prevented deployed pointer publication' +[[ ! -e "$active" ]] || fail 'completed deployment left the active operation marker' +if compgen -G "$root/var/lib/ci-fleet-deployer/.active.*" >/dev/null; then fail 'completed deployment left an unpublished active marker temporary'; fi rm -f "$root/var/lib/ci-fleet-deployer/last-request.conf" rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" +cp "$approval" "$request"; chmod 0600 "$request" deployed_target=$(readlink "$deployed_current") rm "$deployed_current" expect_failure 'deployed rollback snapshot is missing' "$runtime" deploy >/dev/null From f4b3df2a9fa66075ec9fb5d8bc90b549a35a1511 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 19:21:46 -0500 Subject: [PATCH 53/90] fix: durable journal retirement, boundary sync before finalize, honest marker reports - fsync the state root after retiring the recovery journal in both commit_transaction and finalize_committed_rollback so a reboot cannot resurrect a committed transaction. - Treat the rollback commit marker as authoritative: an adapter that created it but exited nonzero now reports the finalized committed rollback instead of an unchanged failure. - Sync the rolled-back state, install root, and systemd boundary before finalization consumes the journal and last-known-good pair. --- scripts/install-deployer.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index ab8e903..e5adca6 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -807,6 +807,9 @@ PY transaction_dir= transaction_committed=1 rm -rf -- "$retired" + # The journal rename and deletion must persist with the finalization, or a + # reboot can resurrect the committed-rollback transaction and repeat it. + sync -f "$state_root" 2>/dev/null || block 'retired recovery journal is not durable' } commit_transaction() { @@ -820,6 +823,9 @@ commit_transaction() { transaction_dir= transaction_committed=1 rm -rf -- "$retired" + # The journal rename and deletion must persist with the commit, or a reboot + # can resurrect the pre-operation transaction and restore stale state. + sync -f "$state_root" 2>/dev/null || block 'retired recovery journal is not durable' } atomic_replace_directory() { @@ -1157,9 +1163,21 @@ PY mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_file.new" "$state_file" reject_mixed_role - policy_adapter_operation "$active_policy" rollback 'last-known-good policy' "$transaction_dir/application-rollback-committed" || die 'application adapter rollback failed' + # The commit marker is authoritative: if the adapter created it but the + # wrapper did not observe a zero exit, recovery finalizes the committed + # rollback, so the report must not claim failure with no change. + if ! policy_adapter_operation "$active_policy" rollback 'last-known-good policy' "$transaction_dir/application-rollback-committed" && [[ ! -f "$transaction_dir/application-rollback-committed" || -L "$transaction_dir/application-rollback-committed" ]]; then + die 'application adapter rollback failed' + fi sync -f "$transaction_dir/application-rollback-committed" 2>/dev/null || sync "$transaction_dir/application-rollback-committed" 2>/dev/null || die 'application rollback commit marker is not durable' sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || true + # The rollback already replaced the activation pointer and units on possibly + # separate filesystems; persist all committed boundaries before finalization + # consumes the journal and last-known-good pair. + sync -f "$state_file" "$active_policy" 2>/dev/null || die 'rolled-back host state is not durable' + sync -f "$state_root" 2>/dev/null || die 'rolled-back host state is not durable' + sync -f "$install_root" 2>/dev/null || die 'rolled-back install root is not durable' + sync -f "$systemd_root" 2>/dev/null || die 'rolled-back systemd boundary is not durable' finalize_committed_rollback recovered_rollback=0 health=healthy From 33d54abc03df5f9112081466b7ff5bb30bafaa33 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 19:31:54 -0500 Subject: [PATCH 54/90] test: marker-committed rollback reports change; regression coverage --- scripts/test-install-deployer.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index c5b0625..64a0c9a 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -140,6 +140,7 @@ case "$1" in validate|health|cleanup|deploy|rollback) ;; *) exit 2 ;; esac if [[ "$1" == rollback && -n ${CI_FLEET_DEPLOYER_ROLLBACK_COMMIT:-} ]]; then install -m 0600 /dev/null "$CI_FLEET_DEPLOYER_ROLLBACK_COMMIT" fi +if [[ "$1" == rollback && -n ${FAKE_ADAPTER_FAIL_AFTER_MARKER:-} ]]; then exit 42; fi if [[ -n ${FAKE_ADAPTER_MUTATE_AUDIT_PATH:-} ]]; then rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" ln -s "$FAKE_ADAPTER_MUTATE_AUDIT_TARGET" "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" @@ -735,6 +736,26 @@ export FAKE_ADAPTER_FAIL=$tmp/fail-rollback expect_failure 'application adapter rollback failed' "$installer" --rollback --config "$config" >/dev/null unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-rollback" grep -Fq 'sha256:bbbbbbbb' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'failed application rollback did not restore current core state' + +# An adapter that commits the rollback marker but exits nonzero must not be +# reported as an unchanged failure: recovery finalizes the committed rollback. +marker_rollback=$(FAKE_ADAPTER_FAIL_AFTER_MARKER=1 "$installer" --rollback --config "$config" 2>&1) || true +printf '%s\n' "$marker_rollback" | grep -Fq 'result=CHANGED' || fail 'marker-committed rollback was reported as an unchanged failure' +printf '%s\n' "$marker_rollback" | grep -Fq 'next=restore-host-policy-evidence-then-check' || fail 'marker-committed rollback lacks the operator reconciliation action' +grep -Fq 'sha256:aaaaaaaa' "$root/var/lib/ci-fleet-deployer/install-state.json" || fail 'marker-committed rollback did not restore last-known-good state' +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +# Rebuild the retained pair consumed by the marker-committed rollback. +image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +image='registry.example.invalid/example/app@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +write_evidence +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null +[[ -f "$root/var/lib/ci-fleet-deployer/last-known-good.json" ]] || fail 'retained rollback pair was not rebuilt' rollback_calls_before=$(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) FAKE_SYSTEMD_VERIFY_EXIT=1 expect_failure 'systemd unit verification failed' "$installer" --rollback --config "$config" >/dev/null [[ $(grep -Fxc rollback "$FAKE_ADAPTER_LOG" || true) == "$rollback_calls_before" ]] || fail 'application rollback ran before core rollback staging was proven' From 5e1b765663dbb78b2adf58a4a5ea287e58c82358 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 20:13:16 -0500 Subject: [PATCH 55/90] fix: durable restore retirement, failed-deploy audit, capability binding, honest reload - restore_transaction fsyncs the restored state, install, and systemd boundaries before deleting the recovery journal and persists the journal retirement itself. - The runtime checks and fsyncs the failed-deployment audit record before completing the failure exit, matching the success path. - GitHub Environment capability evidence now carries and must match the installation ENVIRONMENT and TARGET_ID, so proof produced for one installation cannot satisfy another deploying the same commit. - Uninstall blocks when systemctl daemon-reload fails after unit removal instead of reporting success with cached unit definitions. --- docs/DEPLOYER-HOST.md | 4 ++++ scripts/deployer-runtime.sh | 6 ++++-- scripts/install-deployer.sh | 12 ++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index 53da107..2640072 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -132,12 +132,16 @@ For `github-environment`, also create mode-`0600` capability evidence. This file ```text SCHEMA_VERSION=1 +ENVIRONMENT=staging +TARGET_ID=example-staging ENVIRONMENT_PROTECTION=verified EXACT_HEAD=1111111111111111111111111111111111111111 CAPABILITY_ID=example-capability-check CHECKED_AT=2026-08-08T20:00:00Z ``` +The `ENVIRONMENT` and `TARGET_ID` bind the evidence to one installation; capability evidence produced for a different installation is rejected even when the source commit matches. + For production, a separate authorized process must also create evidence such as: ```text diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index defc3f6..d8e75e7 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -45,7 +45,8 @@ deploy_exit() { "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ - "${audit_phase:-post-consumption}" "$recorded_status" >&8 || true + "${audit_phase:-post-consumption}" "$recorded_status" >&8 || status=1 + sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || status=1 fi rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then @@ -253,9 +254,10 @@ case "$operation" in [[ -v 'cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]' ]] || die 'GitHub Environment approval is missing capability evidence' inside "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" "$evidence_dir" || die 'capability evidence is outside the protected evidence directory' secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' - capability_keys='SCHEMA_VERSION ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' + capability_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' "$capability_keys" for key in $capability_keys; do [[ -v "capability[$key]" ]] || die "capability evidence is missing $key"; done + [[ ${capability[ENVIRONMENT]} == "${req[ENVIRONMENT]}" && ${capability[TARGET_ID]} == "${req[TARGET_ID]}" ]] || die 'GitHub Environment capability evidence does not match this installation' [[ ${capability[SCHEMA_VERSION]} == 1 && ${capability[ENVIRONMENT_PROTECTION]} == verified && ${capability[EXACT_HEAD]} == "${req[SOURCE_COMMIT]}" ]] || die 'GitHub Environment capability evidence is not exact-head verified' if [[ ! ${capability[CAPABILITY_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${capability[CHECKED_AT]}"; then die 'GitHub Environment capability evidence is malformed'; fi ;; diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index e5adca6..89d646a 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -332,7 +332,8 @@ validate_evidence() { [[ -v 'cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]' ]] || block 'GitHub Environment approval requires capability evidence' inside "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" "$etc_root/evidence" || block 'capability evidence is outside the approved evidence directory' secure_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" 'GitHub capability evidence' - parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' 'SCHEMA_VERSION ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' + parse_file "${cfg[APPROVAL_CAPABILITY_EVIDENCE_PATH]}" capability 'capability evidence' 'SCHEMA_VERSION ENVIRONMENT TARGET_ID ENVIRONMENT_PROTECTION EXACT_HEAD CAPABILITY_ID CHECKED_AT' + [[ ${capability[ENVIRONMENT]:-} == "${cfg[ENVIRONMENT]}" && ${capability[TARGET_ID]:-} == "${cfg[TARGET_ID]}" ]] || block 'GitHub Environment capability evidence does not match this installation' [[ ${capability[SCHEMA_VERSION]:-} == 1 && ${capability[ENVIRONMENT_PROTECTION]:-} == verified && ${capability[EXACT_HEAD]:-} == "${cfg[SOURCE_COMMIT]}" ]] || block 'GitHub Environment capability evidence is not exact-head verified' if [[ ! ${capability[CAPABILITY_ID]:-} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${capability[CHECKED_AT]:-}"; then block 'GitHub Environment capability evidence is missing identity or UTC time'; fi ;; @@ -724,8 +725,15 @@ restore_transaction() { systemctl start "$name" >/dev/null 2>&1 || return done <"$transaction_dir/timers-active" fi + # Restored state, pointer, and units may live on separate filesystems; make + # them durable before the only recovery journal is deleted, then persist the + # journal retirement itself. + sync -f "$state_root" 2>/dev/null || block 'restored host state is not durable' + sync -f "$install_root" 2>/dev/null || block 'restored install root is not durable' + sync -f "$systemd_root" 2>/dev/null || block 'restored systemd boundary is not durable' rm -rf -- "$transaction_dir" || return transaction_dir= + sync -f "$state_root" 2>/dev/null || block 'retired recovery journal is not durable' } recover_interrupted_transaction() { @@ -1240,7 +1248,7 @@ perform_uninstall() { fi [[ ! -e "$current" ]] || block 'activation pointer has an unsafe type' for unit in "${unit_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then rm -f "$systemd_root/$unit"; changed=yes; fi; done - systemctl daemon-reload >/dev/null 2>&1 || true + systemctl daemon-reload >/dev/null 2>&1 || block 'systemd manager reload failed after unit removal' rm -f "$drained" "$active_operation" if [[ "$changed" == yes ]]; then report CHANGED yes retained-state "$(rollback_available)"; else report NO_CHANGE no retained-state "$(rollback_available)"; fi } From e574a7dec2d07667ed8290f60f6c8ab306876294 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 20:23:28 -0500 Subject: [PATCH 56/90] test: capability cross-installation rejection, reload-failure uninstall, live-checkout trap restoration --- scripts/test-install-deployer.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 64a0c9a..02c113b 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -294,6 +294,8 @@ PY expect_failure 'GitHub capability evidence must be a regular file' "$installer" --check --config "$config" >/dev/null cat >"$capability" </dev/null cat >"$capability" </dev/null +install -m 0600 "$tmp/capability.saved" "$capability" write_config python3 - "$config" <<'PY' from pathlib import Path @@ -1159,6 +1172,13 @@ mv "$tmp/install-root-uninstall.real" "$root/opt/ci-fleet-deployer" expect_success "$installer" --uninstall --config "$config" >/dev/null [[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after correcting install-root drift did not uninstall' +# Uninstall must fail when systemd cannot reload after unit removal. +expect_success "$installer" --install --config "$config" >/dev/null +FAKE_SYSTEMCTL_FAIL_COMMAND=daemon-reload expect_failure 'systemd manager reload failed after unit removal' "$installer" --uninstall --config "$config" >/dev/null +unset FAKE_SYSTEMCTL_FAIL_COMMAND +expect_success "$installer" --uninstall --config "$config" >/dev/null +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after reload failure did not uninstall' + # An untracked symlink in the deployer unit source must block checkout validation. expect_success "$installer" --install --config "$config" >/dev/null ln -s "$credential" "$repo_root/deploy/deployer/leak" @@ -1513,6 +1533,11 @@ expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" c # Bytes substituted into the live checkout after review must never reach a staged release. cp "$runtime" "$tmp/runtime.saved" +restore_live_checkout() { + [[ ! -f "$tmp/runtime.saved" ]] || cat "$tmp/runtime.saved" >"$runtime" + git -C "$repo_root" replace -d "$core_ref" 2>/dev/null || true +} +trap 'restore_live_checkout; rm -rf "$tmp"' EXIT release_before=$(sha256sum "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh") printf '# substituted-live-bytes\n' >>"$runtime" expect_failure 'differs from the reviewed commit' "$installer" --repair --config "$config" >/dev/null From 41601a3cbab5b0081ebdeb63d5165e2ac71ae6bd Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 21:09:42 -0500 Subject: [PATCH 57/90] fix: recovery without deployed dependencies, durable uninstall, key parsing, pointer names - Transaction restoration no longer loads deployed-snapshot dependencies, so recovery cannot wedge behind unrelated deployed drift. - Uninstall fsyncs the install, systemd, and state boundaries before clearing the drain marker, and persists the marker removal. - Policy value extraction keeps text after the first equals sign, so adapter paths containing '=' survive snapshot validation. - Deployed pointer temporaries use collision-free mktemp names in both the installer and runtime instead of PID-named symlinks. - Rollback and uninstall tolerate an absent configuration directory; other modes still require it. --- scripts/deployer-runtime.sh | 2 +- scripts/install-deployer.sh | 45 ++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index d8e75e7..266d187 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -345,7 +345,7 @@ PY secure_file "$snapshot/policy.conf" 'prepared deployed policy' secure_file "$snapshot/state.json" 'prepared deployed state' [[ $(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) == "$snapshot_policy_sha" && $(sha256sum "$snapshot/state.json" | cut -d' ' -f1) == "$snapshot_state_sha" ]] || die 'prepared deployed snapshot changed during deployment' - pointer=$deployed_root/.current.$$ + pointer=$(mktemp -u "$deployed_root/.current.XXXXXX") snapshot_pointer=$pointer retired_snapshot=$(readlink "$deployed_current" 2>/dev/null || true) [[ -z "$retired_snapshot" || "$retired_snapshot" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'current deployed snapshot pointer is unsafe' diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 89d646a..1c4a2ce 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -91,13 +91,13 @@ ok = (all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) raise SystemExit(0 if ok else 1) PY local rollback_adapter_path rollback_adapter_sha rollback_credential_provider rollback_credential_ref - rollback_adapter_path=$(awk -F= '$1=="ADAPTER_PATH" {print $2}' "$previous_policy") - rollback_adapter_sha=$(awk -F= '$1=="ADAPTER_SHA256" {print $2}' "$previous_policy") + rollback_adapter_path=$(awk '$0 ~ /^ADAPTER_PATH=/ {sub(/^ADAPTER_PATH=/, ""); print}' "$previous_policy") + rollback_adapter_sha=$(awk '$0 ~ /^ADAPTER_SHA256=/ {sub(/^ADAPTER_SHA256=/, ""); print}' "$previous_policy") [[ $rollback_adapter_path == "$etc_root/adapters/"* ]] || { printf no; return; } [[ ! -L "$rollback_adapter_path" && -f "$rollback_adapter_path" && $(stat -c '%u:%a' "$rollback_adapter_path" 2>/dev/null) == "$expected_uid:700" ]] || { printf no; return; } [[ $(sha256sum "$rollback_adapter_path" 2>/dev/null | cut -d' ' -f1) == "$rollback_adapter_sha" ]] || { printf no; return; } - rollback_credential_provider=$(awk -F= '$1=="CREDENTIAL_PROVIDER" {print $2}' "$previous_policy") - rollback_credential_ref=$(awk -F= '$1=="CREDENTIAL_REF" {print $2}' "$previous_policy") + rollback_credential_provider=$(awk '$0 ~ /^CREDENTIAL_PROVIDER=/ {sub(/^CREDENTIAL_PROVIDER=/, ""); print}' "$previous_policy") + rollback_credential_ref=$(awk '$0 ~ /^CREDENTIAL_REF=/ {sub(/^CREDENTIAL_REF=/, ""); print}' "$previous_policy") if [[ $rollback_credential_provider == file ]]; then [[ $rollback_credential_ref == "$etc_root/credentials/"* ]] || { printf no; return; } [[ ! -L "$rollback_credential_ref" && -f "$rollback_credential_ref" && $(stat -c '%u:%a' "$rollback_credential_ref" 2>/dev/null) == "$expected_uid:600" ]] || { printf no; return; } @@ -225,11 +225,12 @@ parse_file() { validate_config() { inside "$config" "$etc_root" || block "configuration path must be inside $etc_root" - secure_directory "$etc_root" 700 0 || block 'configuration directory is missing' if [[ "$mode" == rollback || "$mode" == uninstall ]]; then # Rollback and uninstall must work even when the operator-owned candidate - # configuration is missing or malformed. + # configuration or its directory is missing or malformed. declare -gA cfg=() + if [[ ! -e "$etc_root" ]]; then return; fi + secure_directory "$etc_root" 700 0 || block 'configuration directory has an unsafe owner, mode, or type' if [[ -f "$config" && ! -L "$config" && $(stat -c '%u:%a' "$config" 2>/dev/null) == "$expected_uid:600" ]]; then validated_config=$(mktemp) install -m 0600 "$config" "$validated_config" @@ -242,6 +243,7 @@ validate_config() { fi return fi + secure_directory "$etc_root" 700 0 || block 'configuration directory is missing' secure_file "$config" 'configuration file' validated_config=$(mktemp) install -m 0600 "$config" "$validated_config" @@ -753,7 +755,8 @@ recover_interrupted_transaction() { transaction_committed=0 return fi - if [[ -e "$deployed_current" || -L "$deployed_current" ]]; then load_deployed_snapshot; fi + # Restoration does not consume deployed-snapshot dependencies; requiring + # them here would wedge recovery behind unrelated deployed drift. restore_transaction transaction_committed=0 } @@ -764,13 +767,13 @@ finalize_committed_rollback() { [[ ! -L "$marker" && -f "$marker" && $(stat -c '%u:%a' "$marker") == "$expected_uid:600" ]] || block 'application rollback commit marker is unsafe' secure_file "$active_policy" 'active policy' || return secure_file "$state_file" 'deployer install state' || return - final_adapter_path=$(awk -F= '$1=="ADAPTER_PATH" {print $2}' "$active_policy") - final_adapter_sha=$(awk -F= '$1=="ADAPTER_SHA256" {print $2}' "$active_policy") + final_adapter_path=$(awk '$0 ~ /^ADAPTER_PATH=/ {sub(/^ADAPTER_PATH=/, ""); print}' "$active_policy") + final_adapter_sha=$(awk '$0 ~ /^ADAPTER_SHA256=/ {sub(/^ADAPTER_SHA256=/, ""); print}' "$active_policy") inside "$final_adapter_path" "$etc_root/adapters" || return [[ ! -L "$final_adapter_path" && -f "$final_adapter_path" && $(stat -c '%u:%a' "$final_adapter_path") == "$expected_uid:700" ]] || return [[ $(sha256sum "$final_adapter_path" | cut -d' ' -f1) == "$final_adapter_sha" ]] || return - final_credential_provider=$(awk -F= '$1=="CREDENTIAL_PROVIDER" {print $2}' "$active_policy") - final_credential_ref=$(awk -F= '$1=="CREDENTIAL_REF" {print $2}' "$active_policy") + final_credential_provider=$(awk '$0 ~ /^CREDENTIAL_PROVIDER=/ {sub(/^CREDENTIAL_PROVIDER=/, ""); print}' "$active_policy") + final_credential_ref=$(awk '$0 ~ /^CREDENTIAL_REF=/ {sub(/^CREDENTIAL_REF=/, ""); print}' "$active_policy") credential_reference_safe "$final_credential_provider" "$final_credential_ref" 'committed rollback policy' || return python3 - "$state_file" "$active_policy" "$etc_root/adapters" "$etc_root/credentials" <<'PY' >/dev/null 2>&1 || return import json, re, sys @@ -901,16 +904,16 @@ load_deployed_snapshot() { secure_file "$snapshot/state.json" 'deployed rollback state' deployed_snapshot_policy=$snapshot/policy.conf deployed_snapshot_state=$snapshot/state.json - deployed_adapter_path=$(awk -F= '$1=="ADAPTER_PATH" {print $2}' "$deployed_snapshot_policy") - deployed_adapter_sha=$(awk -F= '$1=="ADAPTER_SHA256" {print $2}' "$deployed_snapshot_policy") + deployed_adapter_path=$(awk '$0 ~ /^ADAPTER_PATH=/ {sub(/^ADAPTER_PATH=/, ""); print}' "$deployed_snapshot_policy") + deployed_adapter_sha=$(awk '$0 ~ /^ADAPTER_SHA256=/ {sub(/^ADAPTER_SHA256=/, ""); print}' "$deployed_snapshot_policy") inside "$deployed_adapter_path" "$etc_root/adapters" || block 'deployed rollback adapter is outside the protected adapter directory' [[ ! -L "$deployed_adapter_path" && -f "$deployed_adapter_path" && $(stat -c '%u:%a' "$deployed_adapter_path") == "$expected_uid:700" ]] || block 'deployed rollback adapter is missing or unsafe' [[ $(sha256sum "$deployed_adapter_path" | cut -d' ' -f1) == "$deployed_adapter_sha" ]] || block 'deployed rollback adapter digest does not match its snapshot policy' - deployed_credential_provider=$(awk -F= '$1=="CREDENTIAL_PROVIDER" {print $2}' "$deployed_snapshot_policy") - deployed_credential_ref=$(awk -F= '$1=="CREDENTIAL_REF" {print $2}' "$deployed_snapshot_policy") + deployed_credential_provider=$(awk '$0 ~ /^CREDENTIAL_PROVIDER=/ {sub(/^CREDENTIAL_PROVIDER=/, ""); print}' "$deployed_snapshot_policy") + deployed_credential_ref=$(awk '$0 ~ /^CREDENTIAL_REF=/ {sub(/^CREDENTIAL_REF=/, ""); print}' "$deployed_snapshot_policy") credential_reference_safe "$deployed_credential_provider" "$deployed_credential_ref" 'deployed rollback policy' - deployed_credential_scope=$(awk -F= '$1=="CREDENTIAL_SCOPE" {print $2}' "$deployed_snapshot_policy") - deployed_environment=$(awk -F= '$1=="ENVIRONMENT" {print $2}' "$deployed_snapshot_policy") + deployed_credential_scope=$(awk '$0 ~ /^CREDENTIAL_SCOPE=/ {sub(/^CREDENTIAL_SCOPE=/, ""); print}' "$deployed_snapshot_policy") + deployed_environment=$(awk '$0 ~ /^ENVIRONMENT=/ {sub(/^ENVIRONMENT=/, ""); print}' "$deployed_snapshot_policy") [[ $deployed_credential_scope == "$deployed_environment" ]] || block 'deployed rollback credential scope does not match its environment' python3 - "$deployed_snapshot_state" "$deployed_snapshot_policy" "$etc_root/adapters" "$etc_root/credentials" <<'PY' >/dev/null 2>&1 || block 'deployed rollback snapshot state and policy do not cross-validate' import json, re, sys @@ -968,7 +971,7 @@ publish_deployed_snapshot() { chmod 0700 "$snapshot" || return install -m 0600 "$policy" "$snapshot/policy.conf" || return install -m 0600 "$state" "$snapshot/state.json" || return - pointer=$deployed_root/.current.$$ + pointer=$(mktemp -u "$deployed_root/.current.XXXXXX") || return ln -s "${snapshot##*/}" "$pointer" || return sync -f "$snapshot/policy.conf" "$snapshot/state.json" 2>/dev/null || block 'replacement deployed snapshot is not durable' sync -f "$snapshot" 2>/dev/null || block 'replacement deployed snapshot is not durable' @@ -1249,7 +1252,13 @@ perform_uninstall() { [[ ! -e "$current" ]] || block 'activation pointer has an unsafe type' for unit in "${unit_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then rm -f "$systemd_root/$unit"; changed=yes; fi; done systemctl daemon-reload >/dev/null 2>&1 || block 'systemd manager reload failed after unit removal' + # Persist pointer, unit, and timer removals across their filesystems before + # the drain marker guarding this maintenance window is cleared. + sync -f "$install_root" 2>/dev/null || block 'uninstalled install root is not durable' + sync -f "$systemd_root" 2>/dev/null || block 'uninstalled systemd boundary is not durable' + sync -f "$state_root" 2>/dev/null || block 'uninstalled host state is not durable' rm -f "$drained" "$active_operation" + sync -f "$state_root" 2>/dev/null || block 'cleared drain marker is not durable' if [[ "$changed" == yes ]]; then report CHANGED yes retained-state "$(rollback_available)"; else report NO_CHANGE no retained-state "$(rollback_available)"; fi } From f0ca8c4493c7080c71a858c956058ad92b6b7196 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 21:20:05 -0500 Subject: [PATCH 58/90] test: uninstall without configuration directory removes the deployment surface --- scripts/test-install-deployer.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 02c113b..537b0f7 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -1179,6 +1179,15 @@ unset FAKE_SYSTEMCTL_FAIL_COMMAND expect_success "$installer" --uninstall --config "$config" >/dev/null [[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'retry after reload failure did not uninstall' +# Uninstall must still remove the deployment surface when the configuration directory is absent. +expect_success "$installer" --install --config "$config" >/dev/null +mv "$root/etc/ci-fleet-deployer" "$tmp/etc-deployer.saved" +uninstall=$(expect_success "$installer" --uninstall --config "$config") +grep -Fq 'result=CHANGED' <<<"$uninstall" || fail 'uninstall without configuration did not report change' +[[ ! -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'uninstall without configuration retained the activation pointer' +if compgen -G "$root/etc/systemd/system/ci-fleet-deployer*" >/dev/null; then fail 'uninstall without configuration retained managed units'; fi +mv "$tmp/etc-deployer.saved" "$root/etc/ci-fleet-deployer" + # An untracked symlink in the deployer unit source must block checkout validation. expect_success "$installer" --install --config "$config" >/dev/null ln -s "$credential" "$repo_root/deploy/deployer/leak" From e5e722a18070cea3163e30a0dac50a011bd65a18 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 21:56:28 -0500 Subject: [PATCH 59/90] fix: fail-closed durability for journals, drain markers, consumption, uninstall - Transaction journal preparation and publication now require their synchronization to succeed before mutations begin. - Drain-marker creation (installer and runtime), uninstall drain-guard publication, resume marker removal, and deployment request consumption now treat synchronization failures as fatal instead of reporting success. - Uninstall skips the install-root synchronization when the root is already absent. --- scripts/deployer-runtime.sh | 6 +++--- scripts/install-deployer.sh | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 266d187..672ed6a 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -150,8 +150,8 @@ if [[ "$operation" == drain ]]; then temporary=$(mktemp "$state_root/.drained.XXXXXX") chmod 0600 "$temporary" mv -Tf "$temporary" "$drained" - sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || true - sync -f "$state_root" 2>/dev/null || true + sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || die 'drain marker is not durable' + sync -f "$state_root" 2>/dev/null || die 'drain marker publication is not durable' exit 0 fi @@ -315,7 +315,7 @@ PY trap 'exit 2' INT TERM install -m 0600 /dev/null "$consumed_marker" || die 'deployment request consumption marker failed' sync -f "$consumed_marker" 2>/dev/null || sync "$consumed_marker" 2>/dev/null || die 'deployment request consumption marker is not durable' - sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || true + sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || die 'deployment request consumption is not durable' umask 077 temporary=$(mktemp "$state_root/.active.XXXXXX") active_temporary=$temporary diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 1c4a2ce..ee0b389 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -622,10 +622,10 @@ begin_transaction() { fi transaction_name=${transaction_dir##*/} transaction_ready=$state_root/.transaction.${transaction_name#.transaction-preparing.} - sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || true + sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || block 'prepared recovery journal is not durable' mv "$transaction_dir" "$transaction_ready" transaction_dir=$transaction_ready - sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || true + sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || block 'published recovery journal is not durable' transaction_preparing=0 } @@ -1181,7 +1181,7 @@ PY die 'application adapter rollback failed' fi sync -f "$transaction_dir/application-rollback-committed" 2>/dev/null || sync "$transaction_dir/application-rollback-committed" 2>/dev/null || die 'application rollback commit marker is not durable' - sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || true + sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || die 'application rollback commit marker is not durable' # The rollback already replaced the activation pointer and units on possibly # separate filesystems; persist all committed boundaries before finalization # consumes the journal and last-known-good pair. @@ -1205,8 +1205,8 @@ perform_drain() { temporary=$(mktemp "$state_root/.drained.XXXXXX") chmod 0600 "$temporary" mv -Tf "$temporary" "$drained" - sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || true - sync -f "$state_root" 2>/dev/null || true + sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || block 'drain marker is not durable' + sync -f "$state_root" 2>/dev/null || block 'drain marker publication is not durable' report CHANGED yes safe-to-maintain "$(rollback_available)" } @@ -1220,6 +1220,7 @@ perform_resume() { health=healthy ((was_drained == 1)) || { report NO_CHANGE no ready-to-deploy "$(rollback_available)"; return; } rm -f "$drained" + sync -f "$state_root" 2>/dev/null || block 'cleared drain marker is not durable' report CHANGED yes ready-to-deploy "$(rollback_available)" } @@ -1238,6 +1239,9 @@ perform_uninstall() { temporary=$(mktemp "$state_root/.drained.XXXXXX") chmod 0600 "$temporary" mv -Tf "$temporary" "$drained" + # The drain guard must be durable before the first mutation beneath it. + sync -f "$drained" 2>/dev/null || sync "$drained" 2>/dev/null || block 'drain marker is not durable' + sync -f "$state_root" 2>/dev/null || block 'drain marker publication is not durable' fi if active_deployment; then block 'active deployment started while draining'; fi for unit in "${timer_names[@]}"; do @@ -1254,7 +1258,9 @@ perform_uninstall() { systemctl daemon-reload >/dev/null 2>&1 || block 'systemd manager reload failed after unit removal' # Persist pointer, unit, and timer removals across their filesystems before # the drain marker guarding this maintenance window is cleared. - sync -f "$install_root" 2>/dev/null || block 'uninstalled install root is not durable' + if [[ -d "$install_root" && ! -L "$install_root" ]]; then + sync -f "$install_root" 2>/dev/null || block 'uninstalled install root is not durable' + fi sync -f "$systemd_root" 2>/dev/null || block 'uninstalled systemd boundary is not durable' sync -f "$state_root" 2>/dev/null || block 'uninstalled host state is not durable' rm -f "$drained" "$active_operation" From c5097c46c1cfc6ae9e3e9aa2109b8acb65b499e1 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 22:39:21 -0500 Subject: [PATCH 60/90] fix: rollback core durable before commit marker, audit inode restore, timer and deployed boundaries - perform_rollback now syncs the rolled-back state, install, and systemd boundaries before making the commit marker durable, so the authoritative marker can never survive state that reverted. - deploy_exit restores the opened audit inode under its durable name when the adapter replaced the audit log, preserving the consumed approval's record. - Uninstall stops timers that remain loaded or enabled even when their unit files drifted away. - Transaction recovery validates the deployed-state directory owner, mode, and type before any pointer mutation can follow it. --- scripts/deployer-runtime.sh | 7 +++++++ scripts/install-deployer.sh | 15 ++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 672ed6a..9244ef9 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -46,6 +46,13 @@ deploy_exit() { "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ "${audit_phase:-post-consumption}" "$recorded_status" >&8 || status=1 + # If the adapter replaced the audit log, descriptor 8 names an unlinked + # inode; restore it under the durable name before synchronizing. + if [[ ! -e "$audit_log" || $(stat -Lc '%d:%i' /proc/self/fd/8 2>/dev/null) != $(stat -c '%d:%i' "$audit_log" 2>/dev/null) ]]; then + rm -f -- "$audit_log" + cat /proc/self/fd/8 >"$audit_log" 2>/dev/null || status=1 + chmod 0600 "$audit_log" 2>/dev/null || true + fi sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || status=1 fi rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index ee0b389..885f823 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -669,6 +669,10 @@ restore_transaction() { [[ -z $backed_up ]] || block "transaction $name manifest is missing but backups remain" fi done + # Validate the deployed-state boundary before any pointer mutation follows it. + if [[ -e "$deployed_root" || -L "$deployed_root" ]]; then + [[ -d "$deployed_root" && ! -L "$deployed_root" && $(stat -c '%u:%a' "$deployed_root") == "$expected_uid:700" ]] || block 'deployed snapshot directory is unsafe' + fi [[ -d "$systemd_root" && ! -L "$systemd_root" && $(stat -c %u "$systemd_root") == "$expected_uid" ]] || block 'systemd unit directory has an unsafe owner or type' systemd_restore_mode=$(stat -c %a "$systemd_root") (((8#$systemd_restore_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' @@ -1180,15 +1184,16 @@ PY if ! policy_adapter_operation "$active_policy" rollback 'last-known-good policy' "$transaction_dir/application-rollback-committed" && [[ ! -f "$transaction_dir/application-rollback-committed" || -L "$transaction_dir/application-rollback-committed" ]]; then die 'application adapter rollback failed' fi - sync -f "$transaction_dir/application-rollback-committed" 2>/dev/null || sync "$transaction_dir/application-rollback-committed" 2>/dev/null || die 'application rollback commit marker is not durable' - sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || die 'application rollback commit marker is not durable' # The rollback already replaced the activation pointer and units on possibly - # separate filesystems; persist all committed boundaries before finalization - # consumes the journal and last-known-good pair. + # separate filesystems; the rolled-back core must be durable before the + # commit marker, or recovery could treat the marker as authoritative while + # the pointer, policy, or units reverted. sync -f "$state_file" "$active_policy" 2>/dev/null || die 'rolled-back host state is not durable' sync -f "$state_root" 2>/dev/null || die 'rolled-back host state is not durable' sync -f "$install_root" 2>/dev/null || die 'rolled-back install root is not durable' sync -f "$systemd_root" 2>/dev/null || die 'rolled-back systemd boundary is not durable' + sync -f "$transaction_dir/application-rollback-committed" 2>/dev/null || sync "$transaction_dir/application-rollback-committed" 2>/dev/null || die 'application rollback commit marker is not durable' + sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || die 'application rollback commit marker is not durable' finalize_committed_rollback recovered_rollback=0 health=healthy @@ -1245,7 +1250,7 @@ perform_uninstall() { fi if active_deployment; then block 'active deployment started while draining'; fi for unit in "${timer_names[@]}"; do - if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]]; then + if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]] || systemctl is-enabled "$unit" >/dev/null 2>&1 || systemctl is-active "$unit" >/dev/null 2>&1; then systemctl disable --now "$unit" >/dev/null 2>&1 || block 'deployer timers did not stop during uninstall' fi done From edede66540a2cadb482e7c995068073e3c9f5320 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 22:50:31 -0500 Subject: [PATCH 61/90] test: audit-inode restore, drifted-timer uninstall stop, deployed-boundary recovery guard --- scripts/test-install-deployer.sh | 49 ++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 537b0f7..18ac5eb 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -142,8 +142,12 @@ if [[ "$1" == rollback && -n ${CI_FLEET_DEPLOYER_ROLLBACK_COMMIT:-} ]]; then fi if [[ "$1" == rollback && -n ${FAKE_ADAPTER_FAIL_AFTER_MARKER:-} ]]; then exit 42; fi if [[ -n ${FAKE_ADAPTER_MUTATE_AUDIT_PATH:-} ]]; then - rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" - ln -s "$FAKE_ADAPTER_MUTATE_AUDIT_TARGET" "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" + if [[ ${FAKE_ADAPTER_MUTATE_AUDIT_MODE:-symlink} == unlink ]]; then + rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH"; printf 'adapter-replacement\n' >"$FAKE_ADAPTER_MUTATE_AUDIT_PATH" + else + rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" + ln -s "$FAKE_ADAPTER_MUTATE_AUDIT_TARGET" "$FAKE_ADAPTER_MUTATE_AUDIT_PATH" + fi fi if [[ -n ${FAKE_ADAPTER_CHMOD_DURING:-} ]]; then chmod 0644 "$FAKE_ADAPTER_CHMOD_DURING"; fi if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT:-} ]]; then @@ -1188,6 +1192,13 @@ grep -Fq 'result=CHANGED' <<<"$uninstall" || fail 'uninstall without configurati if compgen -G "$root/etc/systemd/system/ci-fleet-deployer*" >/dev/null; then fail 'uninstall without configuration retained managed units'; fi mv "$tmp/etc-deployer.saved" "$root/etc/ci-fleet-deployer" +# Uninstall must stop a timer whose unit file has drifted away. +expect_success "$installer" --install --config "$config" >/dev/null +rm "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" +FAKE_SYSTEMCTL_LOG=$tmp/systemctl-drift.log expect_success "$installer" --uninstall --config "$config" >/dev/null +grep -Fq 'disable --now ci-fleet-deployer-cleanup.timer' "$tmp/systemctl-drift.log" || fail 'uninstall left a loaded drifted timer running' +if compgen -G "$root/etc/systemd/system/ci-fleet-deployer*" >/dev/null; then fail 'drifted-timer uninstall retained managed units'; fi + # An untracked symlink in the deployer unit source must block checkout validation. expect_success "$installer" --install --config "$config" >/dev/null ln -s "$credential" "$repo_root/deploy/deployer/leak" @@ -1241,6 +1252,25 @@ chmod 0755 "$root/etc/systemd/system" expect_success "$installer" --repair --config "$config" >/dev/null expect_success "$installer" --check --config "$config" >/dev/null +# Retained-transaction recovery must validate the deployed-state boundary before pointer mutation. +boundary_recovery=$root/var/lib/ci-fleet-deployer/.transaction.deployed-symlink +mkdir -m 0700 "$boundary_recovery" "$boundary_recovery/units" "$boundary_recovery/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$boundary_recovery/state/$name"; printf '%s\n' "$name" >>"$boundary_recovery/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$boundary_recovery/current-target" +printf 'absent\n' >"$boundary_recovery/deployed-target" +mv "$root/var/lib/ci-fleet-deployer/deployed" "$tmp/deployed.real" +mkdir "$tmp/deployed.real-target"; ln -s "$tmp/deployed.real-target" "$root/var/lib/ci-fleet-deployer/deployed" +printf 'decoy\n' >"$tmp/deployed.real-target/current" +expect_failure 'deployed snapshot directory is unsafe' "$installer" --repair --config "$config" >/dev/null +[[ $(<"$tmp/deployed.real-target/current") == decoy ]] || fail 'recovery mutated state through a symlinked deployed directory' +[[ -d "$boundary_recovery" ]] || fail 'blocked deployed-boundary recovery discarded its transaction' +rm "$root/var/lib/ci-fleet-deployer/deployed"; mv "$tmp/deployed.real" "$root/var/lib/ci-fleet-deployer/deployed" +rm -rf -- "$boundary_recovery" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + # A failed timer shutdown must fail the uninstall before any unit removal. expect_success "$installer" --install --config "$config" >/dev/null uninstall_before=$(find "$root/etc/systemd/system" -mindepth 1 -maxdepth 1 -printf '%P %y %m\n' | sort | sha256sum) @@ -1489,6 +1519,21 @@ unset FAKE_ADAPTER_MUTATE_AUDIT_PATH FAKE_ADAPTER_MUTATE_AUDIT_TARGET [[ $(<"$tmp/unrelated-audit") == unrelated-audit ]] || fail 'adapter audit replacement redirected the trusted append' rm "$root/var/log/ci-fleet-deployer/audit.log" +# An adapter that replaces the audit log must not lose the deployment record. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=audit-unlink-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:30Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +export FAKE_ADAPTER_MUTATE_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_MUTATE_AUDIT_MODE=unlink +expect_failure 'deployer audit log changed during deployment' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_MUTATE_AUDIT_PATH FAKE_ADAPTER_MUTATE_AUDIT_MODE +grep -Fq 'approval=audit-unlink-attempt' "$root/var/log/ci-fleet-deployer/audit.log" || fail 'adapter audit replacement lost the deployment audit record' +[[ $(<"$root/var/log/ci-fleet-deployer/audit.log") != *adapter-replacement* ]] || fail 'adapter replacement content entered the trusted audit log' +rm "$root/var/log/ci-fleet-deployer/audit.log" + # Signal at the deployed-snapshot publication boundary must not delete the published snapshot. write_evidence staging example-staging python3 - "$approval" <<'PY' From fc43923a1d12445f908ff28cff0515dfc1fb65df Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Wed, 12 Aug 2026 23:34:39 -0500 Subject: [PATCH 62/90] fix: production gate, drift-unit role rejection, snapshot and availability validation - The runtime rejects ENVIRONMENT=production deployments outright; production paths remain separately gated with no accepted decision. - reject_mixed_role includes ci-fleet-drift.service in both installer and runtime copies. - The deployed-snapshot identical-bytes shortcut requires the resolved incumbent to be a protected child of the deployed root. - rollback_available additionally requires credential-scope match and a complete retained release, so availability implies rollback can run. - The runtime fully validates the deployed rollback pair (adapter digest, credential reference and scope) before consuming approval. - Lost install state is treated as an installation when retained policy or deployed snapshot markers survive. --- scripts/deployer-runtime.sh | 33 +++++++++++++++++++++------------ scripts/install-deployer.sh | 13 ++++++++++--- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 9244ef9..d7271d2 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -79,7 +79,7 @@ secure_directory() { } reject_mixed_role() { local unit runner_unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" - for unit in ci-fleet-health.service ci-fleet-reconcile.service ci-fleet-cleanup.service actions.runner.service; do + for unit in ci-fleet-health.service ci-fleet-reconcile.service ci-fleet-cleanup.service ci-fleet-drift.service actions.runner.service; do [[ ! -e "$systemd_root/$unit" ]] || die 'ordinary CI controller or runner state is present' done shopt -s nullglob @@ -271,17 +271,9 @@ case "$operation" in *) die 'unsupported approval provider' ;; esac if [[ ${cfg[ENVIRONMENT]} == production ]]; then - [[ -v 'cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]' ]] || die 'production policy is missing separate authorization evidence' - inside "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" "$evidence_dir" || die 'production authorization evidence is outside the protected evidence directory' - secure_file "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" 'production authorization evidence' - production_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE AUTHORIZED_BY GATE_ID AUTHORIZED_AT' - parse_file "${cfg[PRODUCTION_AUTHORIZATION_EVIDENCE_PATH]}" production 'production authorization evidence' "$production_keys" - for key in $production_keys; do [[ -v "production[$key]" ]] || die "production authorization evidence is missing $key"; done - [[ ${production[SCHEMA_VERSION]} == 1 && ${production[ENVIRONMENT]} == production ]] || die 'production authorization evidence has the wrong scope' - if [[ ! ${production[AUTHORIZED_BY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ || ! ${production[GATE_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || ! valid_utc "${production[AUTHORIZED_AT]}"; then die 'production authorization evidence is malformed'; fi - for key in ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do - [[ -v "production[$key]" && ${req[$key]} == "${production[$key]}" ]] || die "deployment request does not match production authorization $key" - done + # Production deployment paths remain separately gated + # (docs/DESIGN-DECISIONS.md); no accepted decision enables them yet. + die 'production deployment is not authorized by the current accepted scope' fi if [[ -e "$consumed_root" || -L "$consumed_root" ]]; then secure_directory "$consumed_root" 'consumed request directory'; else install -d -m 0700 "$consumed_root"; fi request_id=$(for key in $request_keys; do printf '%s=%s\0' "$key" "${req[$key]}"; done | sha256sum | cut -d' ' -f1) @@ -307,6 +299,23 @@ PY secure_directory "$deployed_snapshot" 'deployed snapshot' secure_file "$deployed_snapshot/policy.conf" 'deployed rollback policy' secure_file "$deployed_snapshot/state.json" 'deployed rollback state' + # Deploying consumes the approval; the retained rollback point must be + # fully usable first, or the change proceeds with no way back. + declare -A deployed_policy=() + parse_file "$deployed_snapshot/policy.conf" deployed_policy 'deployed rollback policy' "$config_keys" + for key in ADAPTER_PATH ADAPTER_SHA256 CREDENTIAL_PROVIDER CREDENTIAL_REF CREDENTIAL_SCOPE ENVIRONMENT TARGET_ID; do [[ -v "deployed_policy[$key]" ]] || die "deployed rollback policy is missing $key"; done + inside "${deployed_policy[ADAPTER_PATH]}" "$deployer_etc/adapters" || die 'deployed rollback adapter is outside the protected adapter directory' + [[ ! -L "${deployed_policy[ADAPTER_PATH]}" && -f "${deployed_policy[ADAPTER_PATH]}" && $(stat -c '%u:%a' "${deployed_policy[ADAPTER_PATH]}") == "$expected_uid:700" ]] || die 'deployed rollback adapter is missing or unsafe' + [[ $(sha256sum "${deployed_policy[ADAPTER_PATH]}" | cut -d' ' -f1) == "${deployed_policy[ADAPTER_SHA256]}" ]] || die 'deployed rollback adapter digest does not match its snapshot policy' + [[ ${deployed_policy[CREDENTIAL_SCOPE]} == "${deployed_policy[ENVIRONMENT]}" ]] || die 'deployed rollback credential scope does not match its environment' + if [[ ${deployed_policy[CREDENTIAL_PROVIDER]} == file ]]; then + inside "${deployed_policy[CREDENTIAL_REF]}" "$credential_dir" || die 'deployed rollback credential is outside the protected credential directory' + secure_file "${deployed_policy[CREDENTIAL_REF]}" 'deployed rollback credential' + elif [[ ${deployed_policy[CREDENTIAL_PROVIDER]} == external ]]; then + [[ ${deployed_policy[CREDENTIAL_REF]} =~ ^external:[a-z0-9][a-z0-9-]{0,31}:[A-Za-z0-9._/-]{1,128}$ ]] || die 'deployed rollback policy has an invalid external secret-manager adapter reference' + else + die 'deployed rollback policy has an unsupported credential provider' + fi snapshot=$(mktemp -d "$deployed_root/.snapshot.XXXXXX") chmod 0700 "$snapshot" install -m 0600 "$config" "$snapshot/policy.conf" diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 885f823..e2d6c32 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -106,6 +106,13 @@ PY else printf no; return fi + local rollback_scope rollback_environment rollback_release + rollback_scope=$(awk '$0 ~ /^CREDENTIAL_SCOPE=/ {sub(/^CREDENTIAL_SCOPE=/, ""); print}' "$previous_policy") + rollback_environment=$(awk '$0 ~ /^ENVIRONMENT=/ {sub(/^ENVIRONMENT=/, ""); print}' "$previous_policy") + [[ $rollback_scope == "$rollback_environment" ]] || { printf no; return; } + rollback_release=$(awk '$0 ~ /^CORE_REF=/ {sub(/^CORE_REF=/, ""); print}' "$previous_policy") + [[ $rollback_release =~ ^[0-9a-f]{40}$ ]] || { printf no; return; } + release_complete "$releases/$rollback_release" 2>/dev/null || { printf no; return; } printf yes } die() { error_reported=1; printf 'ERROR: %s\n' "$*" >&2; report FAILED no inspect-and-retry "$(rollback_available)" >&2; exit 2; } @@ -387,7 +394,7 @@ require_maintenance_host() { reject_mixed_role() { local unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" runner_unit - for unit in ci-fleet-health.service ci-fleet-reconcile.service ci-fleet-cleanup.service actions.runner.service; do + for unit in ci-fleet-health.service ci-fleet-reconcile.service ci-fleet-cleanup.service ci-fleet-drift.service actions.runner.service; do [[ ! -e "$systemd_root/$unit" ]] || block 'ordinary CI controller or runner state is present' done shopt -s nullglob @@ -965,7 +972,7 @@ publish_deployed_snapshot() { # validated rollback pair; only identical bytes short-circuit. if [[ -L "$deployed_current" ]]; then incumbent=$(readlink -f "$deployed_current") - if [[ -f "$incumbent/policy.conf" && -f "$incumbent/state.json" ]] && cmp -s "$incumbent/policy.conf" "$policy" && cmp -s "$incumbent/state.json" "$state"; then return; fi + if [[ $incumbent == "$deployed_root"/.snapshot.* && -f "$incumbent/policy.conf" && -f "$incumbent/state.json" ]] && cmp -s "$incumbent/policy.conf" "$policy" && cmp -s "$incumbent/state.json" "$state"; then return; fi fi retired=$(readlink "$deployed_current") [[ "$retired" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'current deployed snapshot pointer is unsafe' @@ -1071,7 +1078,7 @@ perform_check() { perform_converge() { local had_state=0 old_environment old_target candidate_changed=1 old_release [[ -f "$state_file" ]] && had_state=1 - if ((!had_state)) && { [[ -L "$current" ]] || compgen -G "$systemd_root/ci-fleet-deployer*" >/dev/null; }; then + if ((!had_state)) && { [[ -L "$current" ]] || compgen -G "$systemd_root/ci-fleet-deployer*" >/dev/null || [[ -e "$active_policy" || -L "$active_policy" || -e "$deployed_current" || -L "$deployed_current" ]]; }; then block 'installed deployer state is absent or drifted; restore install state before convergence' fi if ((had_state)); then From acd287fca1acc067c6eb9103d3fba9b9f58bbf80 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 00:01:41 -0500 Subject: [PATCH 63/90] test: round-fifteen regressions; state-snapshot helper for recovery fixtures --- scripts/test-install-deployer.sh | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 18ac5eb..7a51469 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -227,6 +227,15 @@ EOF if [[ "$environment" == production ]]; then printf 'PRODUCTION_AUTHORIZATION_EVIDENCE_PATH=%s\n' "$production_gate" >>"$root/etc/ci-fleet-deployer/deployer.conf"; fi chmod 0600 "$root/etc/ci-fleet-deployer/deployer.conf" } +# snapshot_state : record the full current managed state into a +# recovery-transaction fixture so restoration returns to a valid installation. +snapshot_state() { + local tx=$1 name + for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$tx/state/$name"; printf '%s\n' "$name" >>"$tx/state-present"; } + done + printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$tx/current-target" +} write_evidence write_config config=$root/etc/ci-fleet-deployer/deployer.conf @@ -450,6 +459,10 @@ expect_success "$installer" --check --config "$config" >/dev/null # Transaction recovery must restore runtime-only timer enablement exactly. runtime_tx=$root/var/lib/ci-fleet-deployer/.transaction.runtime-enabled mkdir -m 0700 "$runtime_tx" "$runtime_tx/units" "$runtime_tx/state" +for name in install-state.json active-policy.conf last-known-good.json last-known-good-policy.conf; do + [[ ! -e "$root/var/lib/ci-fleet-deployer/$name" ]] || { cp "$root/var/lib/ci-fleet-deployer/$name" "$runtime_tx/state/$name"; printf '%s\n' "$name" >>"$runtime_tx/state-present"; } +done +printf '%s\n' "$(readlink "$root/opt/ci-fleet-deployer/current")" >"$runtime_tx/current-target" : >"$root/var/lib/ci-fleet-deployer/unit-ci-fleet-deployer-health.timer.runtime" printf 'ci-fleet-deployer-health.timer\n' >"$runtime_tx/timers-enabled-runtime" expect_success "$installer" --repair --config "$config" >/dev/null @@ -488,6 +501,7 @@ expect_success "$installer" --check --config "$config" >/dev/null # Transaction recovery must tolerate timers whose unit files were never installed. absent_timer_tx=$root/var/lib/ci-fleet-deployer/.transaction.absent-timer mkdir -m 0700 "$absent_timer_tx" "$absent_timer_tx/units" "$absent_timer_tx/state" +snapshot_state "$absent_timer_tx" mv "$root/etc/systemd/system/ci-fleet-deployer-cleanup.timer" "$tmp/cleanup.timer.saved" expect_success "$installer" --repair --config "$config" >/dev/null [[ ! -e "$absent_timer_tx" ]] || fail 'absent-timer transaction was not recovered' @@ -1028,6 +1042,16 @@ mv "$tmp/install-state.saved" "$root/var/lib/ci-fleet-deployer/install-state.jso chmod 0600 "$root/var/lib/ci-fleet-deployer/install-state.json" expect_success "$installer" --repair --config "$config" >/dev/null +# Lost install state with only a retained deployed snapshot is still an installation. +mv "$root/var/lib/ci-fleet-deployer/install-state.json" "$tmp/install-state.saved" +rm -f "$root/opt/ci-fleet-deployer/current" +for stale_unit in "$root"/etc/systemd/system/ci-fleet-deployer*; do rm -f "$stale_unit"; done +expect_failure 'restore install state before convergence' "$installer" --repair --config "$config" >/dev/null +mv "$tmp/install-state.saved" "$root/var/lib/ci-fleet-deployer/install-state.json" +chmod 0600 "$root/var/lib/ci-fleet-deployer/install-state.json" +expect_success "$installer" --repair --config "$config" >/dev/null +expect_success "$installer" --check --config "$config" >/dev/null + # Rollback must work from the retained pair without a usable candidate config. mv "$config" "$tmp/config.saved" rollback=$(expect_success "$installer" --rollback --config "$config") @@ -1076,6 +1100,24 @@ p=Path(sys.argv[1]); p.write_text(p.read_text().replace('ADAPTER_SHA256=', 'ADAP PY available_check=$(expect_success "$installer" --check --config "$config") grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'digest-mismatched retained adapter still reported rollback_available=yes' +install -m 0600 "$tmp/lkg-policy.saved" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" 2>/dev/null || true +# A retained pair whose release is missing must report rollback_available=no. +missing_core=cccccccccccccccccccccccccccccccccccccccc +cp "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$tmp/lkg-policy-release.saved" +cp "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$tmp/lkg-state-release.saved" +python3 - "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$missing_core" <<'PY' +from pathlib import Path +import json, re, sys +policy, state, missing = sys.argv[1], sys.argv[2], sys.argv[3] +p = Path(policy); p.write_text(re.sub(r'(?m)^CORE_REF=[0-9a-f]{40}$', 'CORE_REF=' + missing, p.read_text())) +s = json.loads(Path(state).read_text()); s['core_ref'] = missing +Path(state).write_text(json.dumps(s, indent=2, sort_keys=True) + '\n') +PY +chmod 0600 "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$root/var/lib/ci-fleet-deployer/last-known-good.json" +available_check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=no' <<<"$available_check" || fail 'release-missing retained pair still reported rollback_available=yes' +install -m 0600 "$tmp/lkg-policy-release.saved" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" +install -m 0600 "$tmp/lkg-state-release.saved" "$root/var/lib/ci-fleet-deployer/last-known-good.json" image='registry.example.invalid/example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' write_evidence write_config @@ -1294,6 +1336,22 @@ request=$root/var/lib/ci-fleet-deployer/request.conf cp "$approval" "$request" chmod 0600 "$request" export CI_FLEET_DEPLOYER_CONFIG=$config CI_FLEET_DEPLOYER_REQUEST=$request +# Production deployment remains separately gated: the runtime must reject it. +cp "$config" "$tmp/config.staging" +write_production_gate +write_evidence production example-production +write_config production example-production +python3 - "$request" "$approval" <<'PY' +from pathlib import Path +import sys +for name in sys.argv[1:]: + p=Path(name); p.write_text(p.read_text().replace('ENVIRONMENT=staging', 'ENVIRONMENT=production').replace('TARGET_ID=example-staging', 'TARGET_ID=example-production')) +PY +expect_failure 'production deployment is not authorized by the current accepted scope' "$runtime" deploy >/dev/null +[[ $(grep -Fxc deploy "$FAKE_ADAPTER_LOG" || true) == 0 ]] || fail 'gated production path reached the deployment adapter' +install -m 0600 "$tmp/config.staging" "$config" +write_evidence staging example-staging +cp "$approval" "$request"; chmod 0600 "$request" expect_failure 'usage: deployer-runtime.sh health|cleanup|deploy|drain' "$runtime" rollback >/dev/null mv "$adapter" "$adapter.saved" expect_success "$runtime" drain >/dev/null From d4f00d63dd95950a3cd1419b16f48b7f7b7b94cd Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 00:40:29 -0500 Subject: [PATCH 64/90] fix: reject ordinary-CI timers, timer-aware uninstall no-op, document production disabled - reject_mixed_role now rejects surviving ordinary-CI timer units and their enablement links in both the installer and runtime. - Uninstall treats a still-loaded or still-enabled deployer timer as managed state instead of reporting a no-op. - DEPLOYER-HOST.md states production deployment is currently disabled and the authorization record shape does not lift the gate. --- docs/DEPLOYER-HOST.md | 2 ++ scripts/deployer-runtime.sh | 4 ++-- scripts/install-deployer.sh | 10 ++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index 2640072..0208cc0 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -155,6 +155,8 @@ GATE_ID=production-gate-20260808-1 AUTHORIZED_AT=2026-08-08T20:00:00Z ``` +Production deployment is currently disabled: `docs/DESIGN-DECISIONS.md` keeps production deployment paths separately gated, and the runtime rejects every production request until an accepted decision enables them. The evidence above documents the intended record shape only; it does not lift the gate. + Create the bounded configuration. Values cannot contain shell expressions; the installer parses `KEY=VALUE` without sourcing it. Unknown, duplicate, empty, malformed, or missing keys fail closed. ```bash diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index d7271d2..dad7c23 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -79,8 +79,8 @@ secure_directory() { } reject_mixed_role() { local unit runner_unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" - for unit in ci-fleet-health.service ci-fleet-reconcile.service ci-fleet-cleanup.service ci-fleet-drift.service actions.runner.service; do - [[ ! -e "$systemd_root/$unit" ]] || die 'ordinary CI controller or runner state is present' + for unit in ci-fleet-health.service ci-fleet-health.timer ci-fleet-reconcile.service ci-fleet-reconcile.timer ci-fleet-cleanup.service ci-fleet-cleanup.timer ci-fleet-drift.service ci-fleet-drift.timer actions.runner.service; do + [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" ]] || block 'ordinary CI controller or runner state is present' done shopt -s nullglob for runner_unit in "$systemd_root"/actions.runner.*.service "$systemd_root"/multi-user.target.wants/actions.runner.*.service; do diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index e2d6c32..a1d1f7e 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -394,8 +394,8 @@ require_maintenance_host() { reject_mixed_role() { local unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" runner_unit - for unit in ci-fleet-health.service ci-fleet-reconcile.service ci-fleet-cleanup.service ci-fleet-drift.service actions.runner.service; do - [[ ! -e "$systemd_root/$unit" ]] || block 'ordinary CI controller or runner state is present' + for unit in ci-fleet-health.service ci-fleet-health.timer ci-fleet-reconcile.service ci-fleet-reconcile.timer ci-fleet-cleanup.service ci-fleet-cleanup.timer ci-fleet-drift.service ci-fleet-drift.timer actions.runner.service; do + [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" ]] || block 'ordinary CI controller or runner state is present' done shopt -s nullglob for runner_unit in "$systemd_root"/actions.runner.*.service "$systemd_root"/multi-user.target.wants/actions.runner.*.service; do @@ -1240,6 +1240,12 @@ perform_uninstall() { local changed=no unit managed_present=no if [[ -e "$state_root" || -L "$state_root" || -e "$lock_root" || -L "$lock_root" || -e "$current" || -L "$current" ]]; then managed_present=yes; fi for unit in "${unit_names[@]}"; do [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/$unit" ]] || managed_present=yes; done + if [[ $managed_present == no ]]; then + for unit in "${timer_names[@]}"; do + systemctl is-enabled "$unit" >/dev/null 2>&1 || systemctl is-active "$unit" >/dev/null 2>&1 || continue + managed_present=yes + done + fi if [[ "$managed_present" == no ]]; then report NO_CHANGE no retained-state "$(rollback_available)"; return; fi if active_deployment; then block 'active deployment prevents this operation'; fi acquire_lock From 146dd2981e4558058d21f5d0e20bfdb90125a182 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 00:51:57 -0500 Subject: [PATCH 65/90] test: ordinary-CI drift timer rejection fixture --- scripts/test-install-deployer.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 7a51469..5ee45eb 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -620,6 +620,9 @@ rm "$unit_path"; mv "$unit_path.real" "$unit_path" printf 'mixed-role\n' >"$root/etc/systemd/system/ci-fleet-health.service" expect_failure 'ordinary CI controller or runner state is present' "$installer" --check --config "$config" >/dev/null rm "$root/etc/systemd/system/ci-fleet-health.service" +printf 'mixed-role\n' >"$root/etc/systemd/system/ci-fleet-drift.timer" +expect_failure 'ordinary CI controller or runner state is present' "$installer" --check --config "$config" >/dev/null +rm "$root/etc/systemd/system/ci-fleet-drift.timer" printf 'runner\n' >"$root/etc/systemd/system/actions.runner.example-org-example-repo.example-runner.service" expect_failure 'ordinary GitHub Actions runner service is present' "$installer" --check --config "$config" >/dev/null rm "$root/etc/systemd/system/actions.runner.example-org-example-repo.example-runner.service" From 48a538d95b516384076e0cbd142c4fc05e84e79b Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 01:17:18 -0500 Subject: [PATCH 66/90] fix: fail-closed mixed-role helper in runtime, policy snapshot removed before re-exec - reject_mixed_role in the runtime used the installer's undefined block helper; it now uses die so detection always fails closed. - The runtime removes its policy snapshot before exec re-execution, since exec skips the EXIT trap and would leak unmanaged policy copies. --- scripts/deployer-runtime.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index dad7c23..96b74ce 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -80,7 +80,7 @@ secure_directory() { reject_mixed_role() { local unit runner_unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" for unit in ci-fleet-health.service ci-fleet-health.timer ci-fleet-reconcile.service ci-fleet-reconcile.timer ci-fleet-cleanup.service ci-fleet-cleanup.timer ci-fleet-drift.service ci-fleet-drift.timer actions.runner.service; do - [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" ]] || block 'ordinary CI controller or runner state is present' + [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" ]] || die 'ordinary CI controller or runner state is present' done shopt -s nullglob for runner_unit in "$systemd_root"/actions.runner.*.service "$systemd_root"/multi-user.target.wants/actions.runner.*.service; do @@ -183,6 +183,11 @@ if [[ $own_realpath == */releases/* ]]; then selected=$install_prefix/releases/${cfg[CORE_REF]}/scripts/deployer-runtime.sh [[ -x $selected && ! -L $selected ]] || die 'active policy revision runtime is unavailable' [[ -z ${CI_FLEET_DEPLOYER_REEXEC:-} ]] || die 'runtime re-exec did not select the active revision' + # exec replaces this process without running the EXIT trap; the policy + # snapshot is passed by content to the new process via the environment, so + # remove the unmanaged copy before re-exec. + rm -f "$policy_snapshot" + policy_snapshot= export CI_FLEET_DEPLOYER_REEXEC=1 exec "$selected" "$operation" fi From 1dc18a07d64155d341ff6f821b31d7f6d15ab60e Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 02:21:31 -0500 Subject: [PATCH 67/90] fix: timer enablement scope, Docker pull-in for scheduled services, repair over damaged release - reject_mixed_role inspects timers.target.wants for timer enablement links in both copies. - The scheduled cleanup and health services now pull in Docker with Wants=docker.service, matching the deploy unit. - --repair no longer requires the damaged active release to be complete; the validated checkout replaces it transactionally without executing it. Upgrade and install still require a complete active release. --- deploy/deployer/ci-fleet-deployer-cleanup.service | 1 + deploy/deployer/ci-fleet-deployer-health.service | 1 + scripts/deployer-runtime.sh | 2 +- scripts/install-deployer.sh | 13 ++++++++++--- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/deploy/deployer/ci-fleet-deployer-cleanup.service b/deploy/deployer/ci-fleet-deployer-cleanup.service index 76ecb11..e1c44f0 100644 --- a/deploy/deployer/ci-fleet-deployer-cleanup.service +++ b/deploy/deployer/ci-fleet-deployer-cleanup.service @@ -1,6 +1,7 @@ [Unit] Description=Run application-owned scoped deployer cleanup After=docker.service +Wants=docker.service [Service] Type=oneshot diff --git a/deploy/deployer/ci-fleet-deployer-health.service b/deploy/deployer/ci-fleet-deployer-health.service index 8d87b06..2cb01b5 100644 --- a/deploy/deployer/ci-fleet-deployer-health.service +++ b/deploy/deployer/ci-fleet-deployer-health.service @@ -1,6 +1,7 @@ [Unit] Description=Check ci-fleet deployer health After=docker.service +Wants=docker.service [Service] Type=oneshot diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 96b74ce..31d5595 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -80,7 +80,7 @@ secure_directory() { reject_mixed_role() { local unit runner_unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" for unit in ci-fleet-health.service ci-fleet-health.timer ci-fleet-reconcile.service ci-fleet-reconcile.timer ci-fleet-cleanup.service ci-fleet-cleanup.timer ci-fleet-drift.service ci-fleet-drift.timer actions.runner.service; do - [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" ]] || die 'ordinary CI controller or runner state is present' + [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" && ! -L "$systemd_root/timers.target.wants/$unit" ]] || die 'ordinary CI controller or runner state is present' done shopt -s nullglob for runner_unit in "$systemd_root"/actions.runner.*.service "$systemd_root"/multi-user.target.wants/actions.runner.*.service; do diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index a1d1f7e..b3bcaf8 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -395,7 +395,7 @@ require_maintenance_host() { reject_mixed_role() { local unit line output expected="deployer|${cfg[DEPLOYER_IDENTITY]}" runner_unit for unit in ci-fleet-health.service ci-fleet-health.timer ci-fleet-reconcile.service ci-fleet-reconcile.timer ci-fleet-cleanup.service ci-fleet-cleanup.timer ci-fleet-drift.service ci-fleet-drift.timer actions.runner.service; do - [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" ]] || block 'ordinary CI controller or runner state is present' + [[ ! -e "$systemd_root/$unit" && ! -L "$systemd_root/multi-user.target.wants/$unit" && ! -L "$systemd_root/timers.target.wants/$unit" ]] || block 'ordinary CI controller or runner state is present' done shopt -s nullglob for runner_unit in "$systemd_root"/actions.runner.*.service "$systemd_root"/multi-user.target.wants/actions.runner.*.service; do @@ -1109,8 +1109,15 @@ PY fi if [[ -L "$current" ]]; then old_release=$(readlink -f "$current") - release_complete "$old_release" || block 'active deployer release is incomplete' - if [[ "$mode" != repair ]]; then policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement'; fi + if [[ "$mode" == repair ]]; then + # Repair must not trust or execute a damaged old release; the validated + # checkout replaces it transactionally below. Upgrade and install still + # require a complete active release. + release_complete "$old_release" || true + else + release_complete "$old_release" || block 'active deployer release is incomplete' + policy_adapter_operation "$active_policy" health 'active policy' || block 'active deployer is unhealthy; recover or roll back before replacement' + fi fi reject_mixed_role credential_reference_safe "${cfg[CREDENTIAL_PROVIDER]}" "${cfg[CREDENTIAL_REF]}" 'candidate policy' From 376eeae1e8f8f60534949e13c3bff86046865c6c Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 02:33:42 -0500 Subject: [PATCH 68/90] test: repair replaces a damaged active release from the validated checkout --- scripts/test-install-deployer.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 5ee45eb..f2b1869 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -392,6 +392,13 @@ chmod 0644 "$root/etc/systemd/system/ci-fleet-deployer.service" expect_success "$installer" --repair --config "$config" >/dev/null [[ $(stat -c %a "$root/etc/systemd/system/ci-fleet-deployer.service") == 644 ]] || fail 'repair did not restore unit mode 0644' +# Repair must replace a damaged active release from the validated checkout. +printf 'truncated\n' >"$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh" +expect_failure 'installed deployer state is absent or drifted' "$installer" --check --config "$config" >/dev/null +expect_success "$installer" --repair --config "$config" >/dev/null +cmp -s "$repo_root/scripts/deployer-runtime.sh" "$root/opt/ci-fleet-deployer/releases/$core_ref/scripts/deployer-runtime.sh" || fail 'repair did not replace the damaged release with reviewed bytes' +expect_success "$installer" --check --config "$config" >/dev/null + preparing=$root/var/lib/ci-fleet-deployer/.transaction-preparing.interrupted mkdir -m 0700 "$preparing" state_before_preparing=$(sha256sum "$root/var/lib/ci-fleet-deployer/install-state.json") From 399cd41f1b1de730ab9c0e0f9631a440fcac70a2 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 03:01:19 -0500 Subject: [PATCH 69/90] fix: marker after boundary durability, active-marker sync, timer probes, rollback inhibitor - perform_rollback makes the /opt and /etc boundaries durable before the authoritative commit marker on the state filesystem. - The runtime fsyncs the active-operation marker and state directory before invoking the deployment adapter. - Transaction recovery probes timer enabled/active state independently of unit-file existence before disabling. - The transactional rollback adapter call runs under systemd-inhibit, matching the deploy runtime's shutdown protection. --- scripts/deployer-runtime.sh | 2 ++ scripts/install-deployer.sh | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 31d5595..9d0a940 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -346,6 +346,8 @@ PY fi mv -Tf "$temporary" "$active" active_temporary= + sync -f "$active" 2>/dev/null || sync "$active" 2>/dev/null || die 'active operation marker is not durable' + sync -f "$state_root" 2>/dev/null || die 'active operation marker publication is not durable' set +e env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index b3bcaf8..9e46269 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -685,7 +685,7 @@ restore_transaction() { (((8#$systemd_restore_mode & 8#022) == 0)) || block 'systemd unit directory is group- or world-writable' [[ -d "$install_root" && ! -L "$install_root" && $(stat -c '%u:%a' "$install_root") == "$expected_uid:755" ]] || block 'managed install boundary has an unsafe owner, mode, or type' for name in "${timer_names[@]}"; do - if [[ -e "$systemd_root/$name" || -L "$systemd_root/$name" ]]; then + if [[ -e "$systemd_root/$name" || -L "$systemd_root/$name" ]] || systemctl is-enabled "$name" >/dev/null 2>&1 || systemctl is-active "$name" >/dev/null 2>&1; then systemctl disable --now "$name" >/dev/null 2>&1 || return fi done @@ -1192,6 +1192,14 @@ PY mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_file.new" "$state_file" reject_mixed_role + # Run the transactional rollback under a shutdown inhibitor, like the deploy + # runtime, so a normal shutdown cannot strand a partially applied adapter + # rollback between core mutation and commit-marker publication. + if [[ -z ${CI_FLEET_DEPLOYER_ROLLBACK_INHIBITED:-} && $testing != 1 ]]; then + export CI_FLEET_DEPLOYER_ROLLBACK_INHIBITED=1 + exec systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ + --why='transactional rollback is active' -- "$0" --rollback --config "$config" + fi # The commit marker is authoritative: if the adapter created it but the # wrapper did not observe a zero exit, recovery finalizes the committed # rollback, so the report must not claim failure with no change. @@ -1202,11 +1210,14 @@ PY # separate filesystems; the rolled-back core must be durable before the # commit marker, or recovery could treat the marker as authoritative while # the pointer, policy, or units reverted. - sync -f "$state_file" "$active_policy" 2>/dev/null || die 'rolled-back host state is not durable' - sync -f "$state_root" 2>/dev/null || die 'rolled-back host state is not durable' sync -f "$install_root" 2>/dev/null || die 'rolled-back install root is not durable' sync -f "$systemd_root" 2>/dev/null || die 'rolled-back systemd boundary is not durable' + # The authoritative commit marker (on the state filesystem) becomes durable + # only after the /opt and /etc boundaries, so recovery can never observe it + # over reverted core state. sync -f "$transaction_dir/application-rollback-committed" 2>/dev/null || sync "$transaction_dir/application-rollback-committed" 2>/dev/null || die 'application rollback commit marker is not durable' + sync -f "$state_file" "$active_policy" 2>/dev/null || die 'rolled-back host state is not durable' + sync -f "$state_root" 2>/dev/null || die 'rolled-back host state is not durable' sync -f "$transaction_dir" 2>/dev/null || sync "$transaction_dir" 2>/dev/null || die 'application rollback commit marker is not durable' finalize_committed_rollback recovered_rollback=0 From 42be82dfb7c76eadf3abdb30db5b5554efa5b251 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 03:52:49 -0500 Subject: [PATCH 70/90] fix: inhibited adapter call, durable marker clearing, honest uninstall, consumption audit - The transactional rollback adapter call runs under systemd-inhibit via run_adapter (no whole-installer re-exec). - deploy_exit synchronizes the state directory after clearing the active-operation marker. - Uninstall reports changed=yes when it stops deployer timers. - The runtime appends and fsyncs a result=consumed audit record before invoking the adapter, so a kill or power loss mid-adapter leaves the consumed approval's identities on disk. - DEPLOYER-HOST.md documents the mandatory Docker role and identity labels for adapter-created resources. --- docs/DEPLOYER-HOST.md | 2 ++ scripts/deployer-runtime.sh | 9 +++++++++ scripts/install-deployer.sh | 23 ++++++++++++----------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index 0208cc0..863b075 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -65,6 +65,8 @@ adapter rollback `validate` must be non-mutating and must prove that the candidate policy is usable before core changes the active release. `health` returns zero only when the deployer and application-owned contract are healthy. `cleanup` may remove only resources carrying the application's exact deployer ownership identity; it must never run global prune or touch unrelated resources. `deploy` reads the active policy and request paths from the documented environment variables and owns application-specific staging, rollout, health, and rollback. `rollback` restores application state compatible with the recorded last-known-good core policy. +Every Docker resource the adapter creates (containers, custom networks, volumes) must carry the labels `io.randomdevelopment.ci-fleet.role=deployer` and `io.randomdevelopment.ci-fleet.identity=` from the active policy. Host-role isolation rejects any unlabelled or differently labelled workload as unrelated, so unlabelled adapter resources break subsequent health, cleanup, deployment, and installer operations. + Operations have no interactive input. Zero means success; nonzero means failure. Direct installer validation and health calls are limited to two minutes; rollback is limited to 45 minutes. The adapter must avoid child processes that outlive those bounds, redact logs, and never print credential contents, authorization headers, cookies, private endpoints, or secret-manager responses. Core validates immutable identifiers and approval evidence; it cannot validate application-specific correctness. Rollback must be atomic from the adapter's perspective: nonzero restores the pre-call application state; zero means rollback health is already verified. Rollback is exposed only through the transactional installer, never as a direct runtime operation. For rollback only, core exports `CI_FLEET_DEPLOYER_ROLLBACK_COMMIT`; the adapter atomically creates that root-owned mode-`0600` file as its final successful step. Core stages and selects the last-known-good core before invoking the adapter, restores the newer core on an uncommitted failure, and consumes committed last-known-good state only after core/application alignment. A committed rollback interrupted after the adapter returns is finalized from the retained transaction on the next serialized installer operation. diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 9d0a940..137a926 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -56,6 +56,7 @@ deploy_exit() { sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || status=1 fi rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true + sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || status=1 if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then target=$(readlink "$deployed_current" 2>/dev/null || true) if [[ $target != "${snapshot##*/}" ]]; then rm -rf -- "$snapshot"; fi @@ -348,6 +349,14 @@ PY active_temporary= sync -f "$active" 2>/dev/null || sync "$active" 2>/dev/null || die 'active operation marker is not durable' sync -f "$state_root" 2>/dev/null || die 'active operation marker publication is not durable' + # Record the durably consumed approval before the adapter runs, so even a + # SIGKILL or power loss mid-adapter leaves the attempt's identities. + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=consumed phase=pre-adapter status=none\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ + "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ + >&8 || die 'deployment consumption audit record failed' + sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || die 'deployment consumption audit record is not durable' set +e env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 9e46269..9e9a9ea 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -1021,6 +1021,14 @@ adapter_deadline() { run_adapter() { local policy=$1 adapter_path=$2 operation_name=$3 marker=${4:-} seconds seconds=$(adapter_deadline "$operation_name") + if [[ -n "$marker" && $testing != 1 ]]; then + # The transactional adapter call runs under a shutdown inhibitor, like the + # deploy runtime, so a normal shutdown cannot strand a partially applied + # rollback between core mutation and commit-marker publication. + timeout --signal=TERM --kill-after=10s "${seconds}s" systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ + --why='transactional rollback is active' -- env CI_FLEET_DEPLOYER_CONFIG="$policy" CI_FLEET_DEPLOYER_ROLLBACK_COMMIT="$marker" "$adapter_path" "$operation_name" + return + fi if [[ -n "$marker" ]]; then timeout --signal=TERM --kill-after=10s "${seconds}s" env CI_FLEET_DEPLOYER_CONFIG="$policy" CI_FLEET_DEPLOYER_ROLLBACK_COMMIT="$marker" "$adapter_path" "$operation_name" else @@ -1192,17 +1200,9 @@ PY mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_file.new" "$state_file" reject_mixed_role - # Run the transactional rollback under a shutdown inhibitor, like the deploy - # runtime, so a normal shutdown cannot strand a partially applied adapter - # rollback between core mutation and commit-marker publication. - if [[ -z ${CI_FLEET_DEPLOYER_ROLLBACK_INHIBITED:-} && $testing != 1 ]]; then - export CI_FLEET_DEPLOYER_ROLLBACK_INHIBITED=1 - exec systemd-inhibit --what=shutdown:sleep --mode=block --who=ci-fleet-deployer \ - --why='transactional rollback is active' -- "$0" --rollback --config "$config" - fi - # The commit marker is authoritative: if the adapter created it but the - # wrapper did not observe a zero exit, recovery finalizes the committed - # rollback, so the report must not claim failure with no change. + # Run the adapter call under a shutdown inhibitor, like the deploy runtime, + # so a normal shutdown cannot strand a partially applied adapter rollback + # between core mutation and commit-marker publication. if ! policy_adapter_operation "$active_policy" rollback 'last-known-good policy' "$transaction_dir/application-rollback-committed" && [[ ! -f "$transaction_dir/application-rollback-committed" || -L "$transaction_dir/application-rollback-committed" ]]; then die 'application adapter rollback failed' fi @@ -1283,6 +1283,7 @@ perform_uninstall() { for unit in "${timer_names[@]}"; do if [[ -e "$systemd_root/$unit" || -L "$systemd_root/$unit" ]] || systemctl is-enabled "$unit" >/dev/null 2>&1 || systemctl is-active "$unit" >/dev/null 2>&1; then systemctl disable --now "$unit" >/dev/null 2>&1 || block 'deployer timers did not stop during uninstall' + changed=yes fi done if [[ -L "$current" ]]; then From 5275e21967bf6563db696c4ce22392825fa82c00 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 04:34:37 -0500 Subject: [PATCH 71/90] fix: last-known-good validation before deploy, production rollback gate, credential-marker rejection, replay protection - The runtime validates a present last-known-good pair and its release before consuming approval. - The installer rejects production rollback until the production gate is accepted, matching the runtime deploy gate. - Both role checks reject retained controller host.env and secrets. - deploy_exit restores an adapter-deleted consumption marker instead of clearing audit_pending, so a failed adapter cannot enable replay. --- scripts/deployer-runtime.sh | 23 +++++++++++++++++++++-- scripts/install-deployer.sh | 5 ++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 137a926..c3dae3e 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -39,7 +39,13 @@ deploy_exit() { local status=$? local recorded_status=${adapter_status:-$status} local target - if [[ ${audit_pending:-0} == 1 && -n ${consumed_marker:-} && ! -e "$consumed_marker" ]]; then audit_pending=0; fi + # An adapter that deletes its own consumption marker must not defeat replay + # protection; restore the durable marker rather than clearing audit_pending. + if [[ ${audit_pending:-0} == 1 && -n ${consumed_marker:-} && ! -e "$consumed_marker" ]]; then + install -m 0600 /dev/null "$consumed_marker" 2>/dev/null || true + sync -f "$consumed_marker" 2>/dev/null || sync "$consumed_marker" 2>/dev/null || status=1 + sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || status=1 + fi if [[ ${audit_pending:-0} == 1 ]]; then printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=failed phase=%s status=%s\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ @@ -89,7 +95,7 @@ reject_mixed_role() { [[ -n "$runner_unit" ]] && die 'ordinary GitHub Actions runner service is present' done shopt -u nullglob - for path in "$(root_path /etc/ci-fleet/ci-fleet.env)" "$(root_path /opt/ci-fleet/current)" "$(root_path /var/lib/ci-fleet/install-state.json)"; do + for path in "$(root_path /etc/ci-fleet/ci-fleet.env)" "$(root_path /etc/ci-fleet/host.env)" "$(root_path /etc/ci-fleet/secrets)" "$(root_path /opt/ci-fleet/current)" "$(root_path /var/lib/ci-fleet/install-state.json)"; do [[ ! -e "$path" && ! -L "$path" ]] || die 'ordinary CI controller or runner state is present' done output=$(docker ps -a --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || die 'Docker workload inventory failed' @@ -298,6 +304,19 @@ PY [[ ${cfg[CORE_REF]} =~ ^[0-9a-f]{40}$ ]] || die 'configuration is missing a valid core revision' if [[ -e "$deployed_root" || -L "$deployed_root" ]]; then secure_directory "$deployed_root" 'deployed snapshot directory'; else install -d -m 0700 "$deployed_root"; fi [[ -e "$deployed_current" || -L "$deployed_current" ]] || die 'deployed rollback snapshot is missing' + # The rollback path consumes the retained last-known-good pair, not the + # deployed snapshot; once that pair exists it must be complete and valid + # before approval consumption. + previous_state=$state_root/last-known-good.json + previous_policy=$state_root/last-known-good-policy.conf + if [[ -e "$previous_state" || -L "$previous_state" || -e "$previous_policy" || -L "$previous_policy" ]]; then + secure_file "$previous_state" 'last-known-good state' + secure_file "$previous_policy" 'last-known-good policy' + declare -A lkg_policy=() + parse_file "$previous_policy" lkg_policy 'last-known-good policy' "$config_keys" + [[ ${lkg_policy[CORE_REF]:-} =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good policy has an invalid core revision' + [[ -x "$(root_path /opt/ci-fleet-deployer)/releases/${lkg_policy[CORE_REF]}/scripts/deployer-runtime.sh" ]] || die 'last-known-good release is incomplete' + fi [[ -L "$deployed_current" ]] || die 'deployed snapshot pointer is absent or unsafe' [[ $(readlink "$deployed_current") =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'deployed snapshot pointer target is not canonical' deployed_snapshot=$(readlink -f "$deployed_current") diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 9e9a9ea..8d0f088 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -403,7 +403,7 @@ reject_mixed_role() { [[ -n "$runner_unit" ]] && block 'ordinary GitHub Actions runner service is present' done shopt -u nullglob - for path in "$(root_path /etc/ci-fleet/ci-fleet.env)" "$(root_path /opt/ci-fleet/current)" "$(root_path /var/lib/ci-fleet/install-state.json)"; do + for path in "$(root_path /etc/ci-fleet/ci-fleet.env)" "$(root_path /etc/ci-fleet/host.env)" "$(root_path /etc/ci-fleet/secrets)" "$(root_path /opt/ci-fleet/current)" "$(root_path /var/lib/ci-fleet/install-state.json)"; do [[ ! -e "$path" && ! -L "$path" ]] || block 'ordinary CI controller or runner state is present' done output=$(docker ps -a --format '{{.ID}}|{{.Label "io.randomdevelopment.ci-fleet.role"}}|{{.Label "io.randomdevelopment.ci-fleet.identity"}}') || block 'Docker workload inventory failed' @@ -1183,6 +1183,9 @@ PY [[ "$core_ref" =~ ^[0-9a-f]{40}$ && "$source_commit" =~ ^[0-9a-f]{40}$ && "$artifact" =~ @sha256:[0-9a-f]{64}$ ]] || block 'last-known-good state has unsafe immutable identifiers' [[ "$core_ref" == "${rollback_policy[CORE_REF]}" && "$environment" == "${rollback_policy[ENVIRONMENT]}" && "$target" == "${rollback_policy[TARGET_ID]}" && "$source_commit" == "${rollback_policy[SOURCE_COMMIT]}" && "$artifact" == "${rollback_policy[ARTIFACT_IMAGE]}" && "$deployer_identity" == "${rollback_policy[DEPLOYER_IDENTITY]}" ]] || block 'last-known-good state and policy do not match' [[ "$environment" =~ ^[a-z][a-z0-9-]{0,31}$ && "$target" =~ ^[a-z0-9][a-z0-9._-]{0,63}$ ]] || block 'last-known-good identity is malformed' + # Production deployment paths, including application rollback, remain + # separately gated; no accepted decision enables them yet. + [[ $environment != production ]] || block 'production rollback is not authorized by the current accepted scope' cfg[DEPLOYER_IDENTITY]=${rollback_policy[DEPLOYER_IDENTITY]} command -v docker >/dev/null || block 'docker is required for rollback isolation validation' reject_mixed_role From 3938a7d7a83a748dbddcf9f2843bddfe02b17357 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 04:46:05 -0500 Subject: [PATCH 72/90] test: marker-deleting adapter cannot replay; regressions pass --- scripts/test-install-deployer.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index f2b1869..06e9499 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -141,6 +141,7 @@ if [[ "$1" == rollback && -n ${CI_FLEET_DEPLOYER_ROLLBACK_COMMIT:-} ]]; then install -m 0600 /dev/null "$CI_FLEET_DEPLOYER_ROLLBACK_COMMIT" fi if [[ "$1" == rollback && -n ${FAKE_ADAPTER_FAIL_AFTER_MARKER:-} ]]; then exit 42; fi +if [[ -n ${FAKE_ADAPTER_DELETE_CONSUMED_GLOB:-} ]]; then rm -f $FAKE_ADAPTER_DELETE_CONSUMED_GLOB; fi if [[ -n ${FAKE_ADAPTER_MUTATE_AUDIT_PATH:-} ]]; then if [[ ${FAKE_ADAPTER_MUTATE_AUDIT_MODE:-symlink} == unlink ]]; then rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH"; printf 'adapter-replacement\n' >"$FAKE_ADAPTER_MUTATE_AUDIT_PATH" @@ -1602,6 +1603,23 @@ grep -Fq 'approval=audit-unlink-attempt' "$root/var/log/ci-fleet-deployer/audit. [[ $(<"$root/var/log/ci-fleet-deployer/audit.log") != *adapter-replacement* ]] || fail 'adapter replacement content entered the trusted audit log' rm "$root/var/log/ci-fleet-deployer/audit.log" +# An adapter that deletes its consumption marker and fails must not enable replay. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=marker-delete-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:45Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +printf 'deploy\n' >"$tmp/fail-deploy-marker"; export FAKE_ADAPTER_FAIL=$tmp/fail-deploy-marker +export FAKE_ADAPTER_DELETE_CONSUMED_GLOB="$root/var/lib/ci-fleet-deployer/consumed-requests/*" +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_FAIL FAKE_ADAPTER_DELETE_CONSUMED_GLOB; rm "$tmp/fail-deploy-marker" +if ! compgen -G "$root/var/lib/ci-fleet-deployer/consumed-requests/*" >/dev/null; then fail 'adapter-deleted consumption marker was not restored'; fi +cp "$approval" "$request"; chmod 0600 "$request" +expect_failure 'deployment request was already consumed' "$runtime" deploy >/dev/null +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" + # Signal at the deployed-snapshot publication boundary must not delete the published snapshot. write_evidence staging example-staging python3 - "$approval" <<'PY' From b968ff2ba0ee2765326d75fd1b6ff176febba812 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 05:16:40 -0500 Subject: [PATCH 73/90] fix: full retained-pair validation before deploy, audit prefix integrity, request snapshot digest, rollback baseline - The runtime cross-validates the last-known-good state against its policy and the retained adapter digest before consuming approval. - The durable audit prefix is verified after the adapter returns, so in-place truncation cannot erase the consumption record. - The request snapshot digest is verified before publication as the completed request. - Rollback now requires a completed deployment (last-request.conf), so an install-time candidate snapshot is never applied as a rollback baseline. --- scripts/deployer-runtime.sh | 39 +++++++++++++++++++++++++++++++++++++ scripts/install-deployer.sh | 4 ++++ 2 files changed, 43 insertions(+) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index c3dae3e..63a9f40 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -229,6 +229,7 @@ case "$operation" in secure_file "$request" 'deployment request' request_snapshot=$(mktemp "$state_root/.request.XXXXXX") install -m 0600 "$request" "$request_snapshot" + request_snapshot_sha=$(sha256sum "$request_snapshot" | cut -d' ' -f1) trap 'rm -f "${request_snapshot:-}" "${policy_snapshot:-}"' EXIT trap 'exit 2' INT TERM request_keys='SCHEMA_VERSION ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE APPROVAL_IDENTITY POLICY_IDENTITY APPROVAL_ID APPROVED_AT' @@ -316,6 +317,37 @@ PY parse_file "$previous_policy" lkg_policy 'last-known-good policy' "$config_keys" [[ ${lkg_policy[CORE_REF]:-} =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good policy has an invalid core revision' [[ -x "$(root_path /opt/ci-fleet-deployer)/releases/${lkg_policy[CORE_REF]}/scripts/deployer-runtime.sh" ]] || die 'last-known-good release is incomplete' + # Cross-validate the retained pair exactly as the rollback path does: + # state must match policy, and the retained adapter must match its pin. + python3 - "$previous_state" "$previous_policy" <<'PY' >/dev/null 2>&1 || die 'last-known-good state and policy do not cross-validate' +import json, re, sys +try: + state = json.load(open(sys.argv[1], encoding='utf-8')) +except (OSError, ValueError): + raise SystemExit(1) +policy = {} +try: + for line in open(sys.argv[2], encoding='utf-8'): + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + key, sep, value = line.partition('=') + if sep and key not in policy: + policy[key] = value +except OSError: + raise SystemExit(1) +sha = re.compile(r'^[0-9a-f]{40}$') +pairs = (('core_ref','CORE_REF'), ('environment','ENVIRONMENT'), ('target','TARGET_ID'), ('source_commit','SOURCE_COMMIT'), ('artifact','ARTIFACT_IMAGE'), ('deployer_identity','DEPLOYER_IDENTITY')) +ok = all(state.get(k) and state.get(k) == policy.get(p) for k, p in pairs) +ok = ok and bool(sha.match(state['core_ref'])) and bool(sha.match(state['source_commit'])) +ok = ok and bool(re.search(r'@sha256:[0-9a-f]{64}$', state['artifact'])) +ok = ok and policy.get('CREDENTIAL_SCOPE') == policy.get('ENVIRONMENT') +raise SystemExit(0 if ok else 1) +PY + [[ -v 'lkg_policy[ADAPTER_PATH]' && -v 'lkg_policy[ADAPTER_SHA256]' ]] || die 'last-known-good policy is missing its adapter pin' + inside "${lkg_policy[ADAPTER_PATH]}" "$deployer_etc/adapters" || die 'last-known-good adapter is outside the protected adapter directory' + [[ ! -L "${lkg_policy[ADAPTER_PATH]}" && -f "${lkg_policy[ADAPTER_PATH]}" && $(stat -c '%u:%a' "${lkg_policy[ADAPTER_PATH]}") == "$expected_uid:700" ]] || die 'last-known-good adapter is missing or unsafe' + [[ $(sha256sum "${lkg_policy[ADAPTER_PATH]}" | cut -d' ' -f1) == "${lkg_policy[ADAPTER_SHA256]}" ]] || die 'last-known-good adapter digest does not match its retained policy' fi [[ -L "$deployed_current" ]] || die 'deployed snapshot pointer is absent or unsafe' [[ $(readlink "$deployed_current") =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'deployed snapshot pointer target is not canonical' @@ -376,10 +408,13 @@ PY "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ >&8 || die 'deployment consumption audit record failed' sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || die 'deployment consumption audit record is not durable' + audit_prefix_sha=$(sha256sum "$audit_log" | cut -d' ' -f1) set +e env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? set -e + # An in-place truncation keeps the same inode; the durable prefix must survive. + [[ $(sha256sum "$audit_log" | cut -d' ' -f1) == "$audit_prefix_sha" ]] || die 'deployer audit log changed during deployment' if ((adapter_status != 0)); then audit_phase=adapter die 'deployment adapter failed after approval consumption' @@ -396,6 +431,10 @@ PY secure_file "$snapshot/policy.conf" 'prepared deployed policy' secure_file "$snapshot/state.json" 'prepared deployed state' [[ $(sha256sum "$snapshot/policy.conf" | cut -d' ' -f1) == "$snapshot_policy_sha" && $(sha256sum "$snapshot/state.json" | cut -d' ' -f1) == "$snapshot_state_sha" ]] || die 'prepared deployed snapshot changed during deployment' + # The adapter receives the request snapshot path; only verified bytes may + # become the completed-request record. + secure_file "$request_snapshot" 'deployment request snapshot' + [[ $(sha256sum "$request_snapshot" | cut -d' ' -f1) == "$request_snapshot_sha" ]] || die 'deployment request snapshot changed during deployment' pointer=$(mktemp -u "$deployed_root/.current.XXXXXX") snapshot_pointer=$pointer retired_snapshot=$(readlink "$deployed_current" 2>/dev/null || true) diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 8d0f088..bb5cf6d 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -1186,6 +1186,10 @@ PY # Production deployment paths, including application rollback, remain # separately gated; no accepted decision enables them yet. [[ $environment != production ]] || block 'production rollback is not authorized by the current accepted scope' + # A deployed snapshot published at install time describes a candidate that + # was never deployed; rollback is meaningful only after a deployment has + # completed (the runtime records last-request.conf on success). + [[ -f "$state_root/last-request.conf" && ! -L "$state_root/last-request.conf" ]] || block 'no completed deployment is available to roll back to' cfg[DEPLOYER_IDENTITY]=${rollback_policy[DEPLOYER_IDENTITY]} command -v docker >/dev/null || block 'docker is required for rollback isolation validation' reject_mixed_role From 0ebff52d2975186eb217e7d52e21e5491e9b940c Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 05:39:55 -0500 Subject: [PATCH 74/90] test: rollback baseline gate, consumption-marker replay guard, audit-change fixtures --- scripts/test-install-deployer.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 06e9499..01cc1ab 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -774,6 +774,10 @@ PY expect_success "$installer" --check --config "$config" >/dev/null printf 'rollback\n' >"$tmp/fail-rollback" +# A candidate never deployed must not be a rollback target. +expect_failure 'no completed deployment is available to roll back to' "$installer" --rollback --config "$config" >/dev/null +# Rollback requires a completed deployment; simulate one for the rollback battery. +install -m 0600 /dev/null "$root/var/lib/ci-fleet-deployer/last-request.conf" export FAKE_ADAPTER_FAIL=$tmp/fail-rollback expect_failure 'application adapter rollback failed' "$installer" --rollback --config "$config" >/dev/null unset FAKE_ADAPTER_FAIL; rm "$tmp/fail-rollback" @@ -1583,7 +1587,7 @@ p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20 PY cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_MUTATE_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_MUTATE_AUDIT_TARGET=$tmp/unrelated-audit -expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null +expect_failure 'deployer audit log changed during deployment' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_MUTATE_AUDIT_PATH FAKE_ADAPTER_MUTATE_AUDIT_TARGET [[ $(<"$tmp/unrelated-audit") == unrelated-audit ]] || fail 'adapter audit replacement redirected the trusted append' rm "$root/var/log/ci-fleet-deployer/audit.log" @@ -1629,7 +1633,7 @@ p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20 PY cp "$approval" "$request"; chmod 0600 "$request" export FAKE_ADAPTER_MUTATE_AUDIT_PATH=$root/var/log/ci-fleet-deployer/audit.log FAKE_ADAPTER_MUTATE_AUDIT_TARGET=$tmp/unrelated-audit -expect_failure 'deployer audit log must be a regular file, not a symlink' "$runtime" deploy >/dev/null +expect_failure 'deployer audit log changed during deployment' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_MUTATE_AUDIT_PATH FAKE_ADAPTER_MUTATE_AUDIT_TARGET [[ -L "$deployed_current" && -e "$deployed_current" && -f "$deployed_current/policy.conf" && -f "$deployed_current/state.json" ]] || fail 'post-publication failure left deployed/current dangling' cmp -s "$deployed_current/policy.conf" "$config" || fail 'published deployed policy does not match the deployed configuration' From 53cf72babe0e6696860e0bc9879ac46713e598bb Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 06:10:38 -0500 Subject: [PATCH 75/90] fix: directory-level marker restore, full release validation, availability gate, identity grammar, pointer restore - deploy_exit recreates a deleted consumed-requests directory before restoring its marker. - The runtime validates the entire retained release (tree digest, installer, runtime, units) before consuming approval. - rollback_available mirrors the completed-deployment gate. - The installer's approval identity grammar now matches the runtime's request grammar. - deploy_exit restores the validated incumbent deployed pointer when a failed adapter removed or replaced it. --- scripts/deployer-runtime.sh | 20 +++++++++++++++++++- scripts/install-deployer.sh | 4 +++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 63a9f40..7de8c8e 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -42,6 +42,8 @@ deploy_exit() { # An adapter that deletes its own consumption marker must not defeat replay # protection; restore the durable marker rather than clearing audit_pending. if [[ ${audit_pending:-0} == 1 && -n ${consumed_marker:-} && ! -e "$consumed_marker" ]]; then + [[ ! -e "$consumed_root" && ! -L "$consumed_root" ]] || secure_directory "$consumed_root" 'consumed request directory' + [[ -e "$consumed_root" ]] || install -d -m 0700 "$consumed_root" 2>/dev/null || true install -m 0600 /dev/null "$consumed_marker" 2>/dev/null || true sync -f "$consumed_marker" 2>/dev/null || sync "$consumed_marker" 2>/dev/null || status=1 sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || status=1 @@ -63,6 +65,15 @@ deploy_exit() { fi rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || status=1 + # On failure, restore the validated incumbent deployed pointer if the adapter + # removed or replaced it. + if [[ ${audit_pending:-0} == 1 && -n ${incumbent_pointer:-} ]]; then + if [[ ! -L "$deployed_current" || $(readlink "$deployed_current") != "$incumbent_pointer" ]]; then + rm -f -- "$deployed_current" + ln -s "$incumbent_pointer" "$deployed_current" 2>/dev/null || status=1 + sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || status=1 + fi + fi if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then target=$(readlink "$deployed_current" 2>/dev/null || true) if [[ $target != "${snapshot##*/}" ]]; then rm -rf -- "$snapshot"; fi @@ -316,7 +327,11 @@ PY declare -A lkg_policy=() parse_file "$previous_policy" lkg_policy 'last-known-good policy' "$config_keys" [[ ${lkg_policy[CORE_REF]:-} =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good policy has an invalid core revision' - [[ -x "$(root_path /opt/ci-fleet-deployer)/releases/${lkg_policy[CORE_REF]}/scripts/deployer-runtime.sh" ]] || die 'last-known-good release is incomplete' + # Validate the entire retained release as perform_rollback's + # release_complete does: tree digest, installer/runtime, and units. + lkg_release=$(root_path /opt/ci-fleet-deployer)/releases/${lkg_policy[CORE_REF]} + [[ -f "$lkg_release/.ci-fleet-tree-sha256" && -x "$lkg_release/scripts/deployer-runtime.sh" && -x "$lkg_release/scripts/install-deployer.sh" ]] || die 'last-known-good release is incomplete' + [[ $(<"$lkg_release/.ci-fleet-tree-sha256") == "$(cd "$lkg_release" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1)" ]] || die 'last-known-good release tree digest mismatch' # Cross-validate the retained pair exactly as the rollback path does: # state must match policy, and the retained adapter must match its pin. python3 - "$previous_state" "$previous_policy" <<'PY' >/dev/null 2>&1 || die 'last-known-good state and policy do not cross-validate' @@ -409,6 +424,9 @@ PY >&8 || die 'deployment consumption audit record failed' sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || die 'deployment consumption audit record is not durable' audit_prefix_sha=$(sha256sum "$audit_log" | cut -d' ' -f1) + # Capture the validated incumbent pointer independently of adapter-writable + # state so a failed adapter cannot destroy or falsify the rollback point. + incumbent_pointer=$(readlink "$deployed_current") set +e env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index bb5cf6d..1c1dc4c 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -113,6 +113,8 @@ PY rollback_release=$(awk '$0 ~ /^CORE_REF=/ {sub(/^CORE_REF=/, ""); print}' "$previous_policy") [[ $rollback_release =~ ^[0-9a-f]{40}$ ]] || { printf no; return; } release_complete "$releases/$rollback_release" 2>/dev/null || { printf no; return; } + # Rollback requires a completed deployment; mirror the perform_rollback gate. + [[ -f "$state_root/last-request.conf" && ! -L "$state_root/last-request.conf" ]] || { printf no; return; } printf yes } die() { error_reported=1; printf 'ERROR: %s\n' "$*" >&2; report FAILED no inspect-and-retry "$(rollback_available)" >&2; exit 2; } @@ -333,7 +335,7 @@ validate_evidence() { for key in ENVIRONMENT TARGET_ID SOURCE_COMMIT ARTIFACT_IMAGE; do [[ ${approval[$key]} == "${cfg[$key]}" ]] || block "approval evidence does not match exact $key" done - [[ ${approval[APPROVAL_IDENTITY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${approval[POLICY_IDENTITY]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ && ${approval[APPROVAL_ID]} =~ ^[A-Za-z0-9._:@/-]{1,128}$ ]] || block 'approval identity is malformed' + [[ ${approval[APPROVAL_IDENTITY]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ && ${approval[POLICY_IDENTITY]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ && ${approval[APPROVAL_ID]} =~ ^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$ ]] || block 'approval identity is malformed' valid_utc "${approval[APPROVED_AT]}" || block 'approval timestamp must be UTC RFC3339' case ${cfg[APPROVAL_PROVIDER]} in manual-exact-head|external-exact-head) ;; From 381ac8bc00b5a999ce2437e1397098d48086963e Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 06:22:51 -0500 Subject: [PATCH 76/90] test: directory-level consumption restore, regressions for round twenty-two --- scripts/test-install-deployer.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 01cc1ab..52dc7e8 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -141,7 +141,7 @@ if [[ "$1" == rollback && -n ${CI_FLEET_DEPLOYER_ROLLBACK_COMMIT:-} ]]; then install -m 0600 /dev/null "$CI_FLEET_DEPLOYER_ROLLBACK_COMMIT" fi if [[ "$1" == rollback && -n ${FAKE_ADAPTER_FAIL_AFTER_MARKER:-} ]]; then exit 42; fi -if [[ -n ${FAKE_ADAPTER_DELETE_CONSUMED_GLOB:-} ]]; then rm -f $FAKE_ADAPTER_DELETE_CONSUMED_GLOB; fi +if [[ -n ${FAKE_ADAPTER_DELETE_CONSUMED_GLOB:-} ]]; then rm -rf $FAKE_ADAPTER_DELETE_CONSUMED_GLOB; fi if [[ -n ${FAKE_ADAPTER_MUTATE_AUDIT_PATH:-} ]]; then if [[ ${FAKE_ADAPTER_MUTATE_AUDIT_MODE:-symlink} == unlink ]]; then rm -f "$FAKE_ADAPTER_MUTATE_AUDIT_PATH"; printf 'adapter-replacement\n' >"$FAKE_ADAPTER_MUTATE_AUDIT_PATH" @@ -1624,6 +1624,23 @@ cp "$approval" "$request"; chmod 0600 "$request" expect_failure 'deployment request was already consumed' "$runtime" deploy >/dev/null rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" +# An adapter that deletes the whole consumption directory and fails must not enable replay. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=dir-delete-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:50Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +printf 'deploy\n' >"$tmp/fail-deploy-dir"; export FAKE_ADAPTER_FAIL=$tmp/fail-deploy-dir +export FAKE_ADAPTER_DELETE_CONSUMED_GLOB="$root/var/lib/ci-fleet-deployer/consumed-requests" +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_FAIL FAKE_ADAPTER_DELETE_CONSUMED_GLOB; rm "$tmp/fail-deploy-dir" +if ! compgen -G "$root/var/lib/ci-fleet-deployer/consumed-requests/*" >/dev/null; then fail 'adapter-deleted consumption directory was not restored with its marker'; fi +cp "$approval" "$request"; chmod 0600 "$request" +expect_failure 'deployment request was already consumed' "$runtime" deploy >/dev/null +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" + # Signal at the deployed-snapshot publication boundary must not delete the published snapshot. write_evidence staging example-staging python3 - "$approval" <<'PY' From f9ea431468b8b9f148ea82d73d3988cbc5cbac69 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 06:54:42 -0500 Subject: [PATCH 77/90] fix: pointer restore ordering, full release predicate, audit prefix restore, production cleanup gate, lock-preserving re-exec - deploy_exit restores the incumbent deployed pointer before durably clearing the active-operation marker. - The runtime applies rollback's complete release predicate (ownership, modes, non-symlink types, tree digest) before consuming approval. - A durable audit prefix copy is kept across the adapter call and used to restore the log after in-place truncation or replacement. - The runtime cleanup operation rejects production configurations like deploy does. - A re-executed runtime reuses the inherited lock descriptor instead of dropping the flock during handoff. --- scripts/deployer-runtime.sh | 49 ++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 7de8c8e..ed1a8dd 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -54,19 +54,23 @@ deploy_exit() { "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ "${audit_phase:-post-consumption}" "$recorded_status" >&8 || status=1 - # If the adapter replaced the audit log, descriptor 8 names an unlinked - # inode; restore it under the durable name before synchronizing. + # If the adapter replaced the audit log, descriptor 8 names an unlinked or + # truncated inode; restore the durable prefix copy when present, else the + # opened inode, before synchronizing. if [[ ! -e "$audit_log" || $(stat -Lc '%d:%i' /proc/self/fd/8 2>/dev/null) != $(stat -c '%d:%i' "$audit_log" 2>/dev/null) ]]; then rm -f -- "$audit_log" - cat /proc/self/fd/8 >"$audit_log" 2>/dev/null || status=1 + if [[ -n ${audit_prefix_copy:-} && -f $audit_prefix_copy ]]; then + cat "$audit_prefix_copy" >"$audit_log" 2>/dev/null || status=1 + else + cat /proc/self/fd/8 >"$audit_log" 2>/dev/null || status=1 + fi chmod 0600 "$audit_log" 2>/dev/null || true fi sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || status=1 + [[ -z ${audit_prefix_copy:-} ]] || rm -f "$audit_prefix_copy" fi - rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true - sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || status=1 - # On failure, restore the validated incumbent deployed pointer if the adapter - # removed or replaced it. + # On failure, restore the validated incumbent deployed pointer before the + # active-operation guard is durably cleared. if [[ ${audit_pending:-0} == 1 && -n ${incumbent_pointer:-} ]]; then if [[ ! -L "$deployed_current" || $(readlink "$deployed_current") != "$incumbent_pointer" ]]; then rm -f -- "$deployed_current" @@ -74,6 +78,8 @@ deploy_exit() { sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || status=1 fi fi + rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true + sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || status=1 if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then target=$(readlink "$deployed_current" 2>/dev/null || true) if [[ $target != "${snapshot##*/}" ]]; then rm -rf -- "$snapshot"; fi @@ -163,8 +169,14 @@ validate_credential() { secure_directory "$state_root" 'deployer state directory' secure_directory "$lock_dir" 'deployer lock directory' -exec 9<"$lock_dir" -flock -n 9 || die 'another deployer operation is running' +if [[ -z ${CI_FLEET_DEPLOYER_REEXEC:-} ]]; then + exec 9<"$lock_dir" + flock -n 9 || die 'another deployer operation is running' +else + # The re-executed process inherits the locked descriptor 9 from its parent; + # reopening it would drop the lock during the handoff. + : +fi shopt -s nullglob transactions=("$state_root"/.transaction.*) shopt -u nullglob @@ -231,6 +243,9 @@ case "$operation" in not_drained reject_mixed_role validate_credential + # Scheduled cleanup mutates application-owned resources; production paths + # remain separately gated like deploy. + [[ ${cfg[ENVIRONMENT]} != production ]] || die 'production cleanup is not authorized by the current accepted scope' env CI_FLEET_DEPLOYER_CONFIG="$config" "$adapter_path" cleanup ;; deploy) @@ -328,10 +343,14 @@ PY parse_file "$previous_policy" lkg_policy 'last-known-good policy' "$config_keys" [[ ${lkg_policy[CORE_REF]:-} =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good policy has an invalid core revision' # Validate the entire retained release as perform_rollback's - # release_complete does: tree digest, installer/runtime, and units. + # release_complete does: ownership, modes, types, and tree digest. lkg_release=$(root_path /opt/ci-fleet-deployer)/releases/${lkg_policy[CORE_REF]} - [[ -f "$lkg_release/.ci-fleet-tree-sha256" && -x "$lkg_release/scripts/deployer-runtime.sh" && -x "$lkg_release/scripts/install-deployer.sh" ]] || die 'last-known-good release is incomplete' + [[ -d "$lkg_release" && ! -L "$lkg_release" && $(stat -c '%u:%a' "$lkg_release") == "$expected_uid:755" ]] || die 'last-known-good release is incomplete' + for dir in "$lkg_release/scripts" "$lkg_release/deploy" "$lkg_release/deploy/deployer"; do [[ -d "$dir" && ! -L "$dir" && $(stat -c '%u:%a' "$dir") == "$expected_uid:755" ]] || die 'last-known-good release is incomplete'; done + for entry in "$lkg_release/scripts/install-deployer.sh" "$lkg_release/scripts/deployer-runtime.sh"; do [[ ! -L "$entry" && -f "$entry" && $(stat -c '%u:%a' "$entry") == "$expected_uid:755" ]] || die 'last-known-good release is incomplete'; done + [[ -f "$lkg_release/.ci-fleet-tree-sha256" ]] || die 'last-known-good release is incomplete' [[ $(<"$lkg_release/.ci-fleet-tree-sha256") == "$(cd "$lkg_release" && sha256sum scripts/install-deployer.sh scripts/deployer-runtime.sh deploy/deployer/* | sha256sum | cut -d' ' -f1)" ]] || die 'last-known-good release tree digest mismatch' + for entry in "$lkg_release"/deploy/deployer/*; do [[ -f "$entry" && ! -L "$entry" && $(stat -c '%u:%a' "$entry") == "$expected_uid:644" ]] || die 'last-known-good release is incomplete'; done # Cross-validate the retained pair exactly as the rollback path does: # state must match policy, and the retained adapter must match its pin. python3 - "$previous_state" "$previous_policy" <<'PY' >/dev/null 2>&1 || die 'last-known-good state and policy do not cross-validate' @@ -424,6 +443,8 @@ PY >&8 || die 'deployment consumption audit record failed' sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || die 'deployment consumption audit record is not durable' audit_prefix_sha=$(sha256sum "$audit_log" | cut -d' ' -f1) + audit_prefix_copy=$(mktemp "$state_root/.audit-prefix.XXXXXX") + install -m 0600 "$audit_log" "$audit_prefix_copy" # Capture the validated incumbent pointer independently of adapter-writable # state so a failed adapter cannot destroy or falsify the rollback point. incumbent_pointer=$(readlink "$deployed_current") @@ -432,7 +453,11 @@ PY adapter_status=$? set -e # An in-place truncation keeps the same inode; the durable prefix must survive. - [[ $(sha256sum "$audit_log" | cut -d' ' -f1) == "$audit_prefix_sha" ]] || die 'deployer audit log changed during deployment' + if [[ $(sha256sum "$audit_log" | cut -d' ' -f1) != "$audit_prefix_sha" ]]; then + install -m 0600 "$audit_prefix_copy" "$audit_log" 2>/dev/null || true + die 'deployer audit log changed during deployment' + fi + rm -f "$audit_prefix_copy" if ((adapter_status != 0)); then audit_phase=adapter die 'deployment adapter failed after approval consumption' From ae6302c68807eb3b30a3399c15fd4924f35788e9 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 07:35:28 -0500 Subject: [PATCH 78/90] fix: retire incumbent snapshot only after the durable success audit The deployed incumbent snapshot is deleted only after the request move and the fsynced success audit record, so a commit-phase failure can no longer leave deployed/current restored to a deleted target. Also drops a stray fixture comment committed earlier. --- scripts/deployer-runtime.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index ed1a8dd..150cea9 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -490,7 +490,6 @@ PY sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'deployed snapshot pointer is not durable' snapshot_pointer=$deployed_current snapshot= - if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi if [[ -f "$request" && ! -L "$request" ]] && cmp -s "$request_snapshot" "$request"; then rm -f "$request"; fi mv -Tf "$request_snapshot" "$last_request" request_snapshot= @@ -502,7 +501,9 @@ PY "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ >&8 sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || die 'deployment success audit is not durable' + # All fallible commit work is done; the incumbent snapshot can be retired. + if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi + sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'retired deployed snapshot is not durable' audit_pending=0 ;; esac -# substituted-live-bytes From f0b62de8ee36a7f70e0374159a4c05e91a77e570 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 08:31:56 -0500 Subject: [PATCH 79/90] fix: post-publication pointer truth, success-path marker preservation, state validation, drain start failure - deploy_exit retains the published pointer after adapter success even when a later commit step fails, since the application has changed. - The success path recreates a missing consumption marker durably before clearing the exit guard. - perform_rollback applies secure_file to last-known-good.json. - The deploy unit drops ConditionPathExists=!drained so a drained host fails the service start via the runtime's not_drained check instead of silently skipping. --- deploy/deployer/ci-fleet-deployer.service | 1 - scripts/deployer-runtime.sh | 15 +++++++++++++-- scripts/install-deployer.sh | 1 + 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/deploy/deployer/ci-fleet-deployer.service b/deploy/deployer/ci-fleet-deployer.service index 786eaad..8a6d131 100644 --- a/deploy/deployer/ci-fleet-deployer.service +++ b/deploy/deployer/ci-fleet-deployer.service @@ -2,7 +2,6 @@ Description=Run one approved ci-fleet deployment After=docker.service network-online.target Wants=docker.service network-online.target -ConditionPathExists=!/var/lib/ci-fleet-deployer/drained [Service] Type=oneshot diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 150cea9..8fd60b3 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -70,8 +70,10 @@ deploy_exit() { [[ -z ${audit_prefix_copy:-} ]] || rm -f "$audit_prefix_copy" fi # On failure, restore the validated incumbent deployed pointer before the - # active-operation guard is durably cleared. - if [[ ${audit_pending:-0} == 1 && -n ${incumbent_pointer:-} ]]; then + # active-operation guard is durably cleared — but only when the new pointer + # was never published. After publication the new pointer is the truth: the + # application has changed and restoring the incumbent would falsify state. + if [[ ${audit_pending:-0} == 1 && -n ${incumbent_pointer:-} && ${snapshot_pointer:-} != "$deployed_current" ]]; then if [[ ! -L "$deployed_current" || $(readlink "$deployed_current") != "$incumbent_pointer" ]]; then rm -f -- "$deployed_current" ln -s "$incumbent_pointer" "$deployed_current" 2>/dev/null || status=1 @@ -501,6 +503,15 @@ PY "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ >&8 sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || die 'deployment success audit is not durable' + # An adapter that deleted its consumption marker must not weaken replay + # protection on success either; ensure the durable marker before the exit + # guard is disabled. + if [[ ! -e "$consumed_marker" ]]; then + [[ -e "$consumed_root" ]] || install -d -m 0700 "$consumed_root" + install -m 0600 /dev/null "$consumed_marker" || die 'deployment request consumption marker failed' + sync -f "$consumed_marker" 2>/dev/null || sync "$consumed_marker" 2>/dev/null || die 'deployment request consumption marker is not durable' + sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || die 'deployment request consumption is not durable' + fi # All fallible commit work is done; the incumbent snapshot can be retired. if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'retired deployed snapshot is not durable' diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 1c1dc4c..66b9080 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -1171,6 +1171,7 @@ perform_rollback() { active_deployment && block 'active deployment prevents rollback' [[ -f "$previous_state" && -f "$previous_policy" ]] || block 'no last-known-good release is available' secure_directory "$state_root" 700 0 + secure_file "$previous_state" 'last-known-good state' secure_file "$previous_policy" 'last-known-good policy' parse_file "$previous_policy" rollback_policy 'last-known-good policy' "$config_keys" for key in CORE_REF ENVIRONMENT TARGET_ID DEPLOYER_IDENTITY SOURCE_COMMIT ARTIFACT_IMAGE; do [[ -v "rollback_policy[$key]" ]] || block "last-known-good policy is missing $key"; done From d119c299b9866fdc329898d14598e9d153c02589 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 09:14:49 -0500 Subject: [PATCH 80/90] fix: incumbent-based retirement, controller-side deployer guard, bounded stop - The runtime retires the validated pre-adapter incumbent snapshot, never an adapter-writable pointer read after the adapter ran. - The controller installer rejects hosts carrying deployer units or state, preserving the role boundary in both install orders. - The deploy unit's stop timeout is 30s so a hung adapter cannot hold the lock and inhibitor past the 45-minute deployment bound. --- deploy/deployer/ci-fleet-deployer.service | 4 +++- scripts/deployer-runtime.sh | 4 +++- scripts/install-worker-controller.sh | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/deploy/deployer/ci-fleet-deployer.service b/deploy/deployer/ci-fleet-deployer.service index 8a6d131..ebefc9e 100644 --- a/deploy/deployer/ci-fleet-deployer.service +++ b/deploy/deployer/ci-fleet-deployer.service @@ -11,7 +11,9 @@ ExecStart=/opt/ci-fleet-deployer/current/scripts/deployer-runtime.sh deploy Environment=CI_FLEET_DEPLOYER_CONFIG=/var/lib/ci-fleet-deployer/active-policy.conf Environment=CI_FLEET_DEPLOYER_REQUEST=/var/lib/ci-fleet-deployer/request.conf TimeoutStartSec=45min -TimeoutStopSec=45min +# A hung adapter must not hold the lock or inhibitor beyond the documented +# 45-minute deployment bound; stop quickly after TERM and let KILL follow. +TimeoutStopSec=30s UMask=0077 NoNewPrivileges=yes PrivateTmp=yes diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 8fd60b3..4bbfa79 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -482,7 +482,9 @@ PY [[ $(sha256sum "$request_snapshot" | cut -d' ' -f1) == "$request_snapshot_sha" ]] || die 'deployment request snapshot changed during deployment' pointer=$(mktemp -u "$deployed_root/.current.XXXXXX") snapshot_pointer=$pointer - retired_snapshot=$(readlink "$deployed_current" 2>/dev/null || true) + # Retire the validated pre-adapter incumbent, never an adapter-writable + # pointer target read after the adapter ran. + retired_snapshot=$incumbent_pointer [[ -z "$retired_snapshot" || "$retired_snapshot" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || die 'current deployed snapshot pointer is unsafe' ln -s "${snapshot##*/}" "$pointer" [[ -z ${CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF:-} || $testing != 1 ]] || kill -"$CI_FLEET_DEPLOYER_TEST_SIGNAL_SELF" $$ diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index ad15656..7356421 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -1024,6 +1024,10 @@ perform_check() { perform_converge() { local count existing_status desired_controller_id=$controller_id + # A deployer host is a separate role; never start a controller on it. + if compgen -G "$(root_path /etc/systemd/system)/ci-fleet-deployer*" >/dev/null || [[ -e "$(root_path /var/lib/ci-fleet-deployer)" || -e "$(root_path /etc/ci-fleet-deployer)" ]]; then + die 'deployer host state is present; controller and deployer roles are separate hosts' + fi if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then die '--upgrade requires an existing managed installation; use --install or --adopt' fi From 886a95d4b2c5096547a5323e3d694a6753d4953c Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 09:34:29 -0500 Subject: [PATCH 81/90] test: controller installer rejects a deployer host --- scripts/test-install-worker-controller.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 445a77d..ab5fd31 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -323,6 +323,13 @@ expect_failure 'host configuration must be owned by root' "$installer" --install unset FAKE_WRONG_HOST_CONFIG_OWNER expect_failure 'managed installs require the default' "$installer" --check "${base_args[@]}" --ref "$ref_one" --host-config "$tmp/custom-host.env" +# A controller must never install onto a deployer host. +mkdir -p "$root/etc/systemd/system" "$root/var/lib/ci-fleet-deployer" +printf '[Unit]\n' >"$root/etc/systemd/system/ci-fleet-deployer.service" +expect_failure 'deployer host state is present' "$installer" --install "${base_args[@]}" --ref "$ref_one" +rm "$root/etc/systemd/system/ci-fleet-deployer.service" +rmdir "$root/var/lib/ci-fleet-deployer" + first=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not converge' [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' From 16d5851caccf9519670a829a3f611e609cb743b2 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 10:09:10 -0500 Subject: [PATCH 82/90] fix: controller rollback guard, incumbent content restore, audit ordering, shortcut metadata, local Docker - The controller rollback path also rejects deployer hosts. - deploy_exit restores the incumbent snapshot's content, not only its pointer, when a failed adapter deleted the target directory. - The terminal failure audit record is appended after restoring the audit log under its durable name. - The identical-bytes publication shortcut now requires the incumbent snapshot directory and files to have safe owner, mode, and type. - Installer and runtime force DOCKER_HOST to the local socket and unset DOCKER_CONTEXT before any Docker call. --- scripts/deployer-runtime.sh | 35 +++++++++++++++++++++------- scripts/install-deployer.sh | 7 +++++- scripts/install-worker-controller.sh | 11 ++++++--- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 4bbfa79..0f3647e 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -49,14 +49,9 @@ deploy_exit() { sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || status=1 fi if [[ ${audit_pending:-0} == 1 ]]; then - printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=failed phase=%s status=%s\n' \ - "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ - "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ - "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ - "${audit_phase:-post-consumption}" "$recorded_status" >&8 || status=1 - # If the adapter replaced the audit log, descriptor 8 names an unlinked or - # truncated inode; restore the durable prefix copy when present, else the - # opened inode, before synchronizing. + # If the adapter replaced the audit log, restore the durable prefix copy + # (or the opened inode) under its name before appending the terminal + # failure record, so the record is never written only to an unlinked inode. if [[ ! -e "$audit_log" || $(stat -Lc '%d:%i' /proc/self/fd/8 2>/dev/null) != $(stat -c '%d:%i' "$audit_log" 2>/dev/null) ]]; then rm -f -- "$audit_log" if [[ -n ${audit_prefix_copy:-} && -f $audit_prefix_copy ]]; then @@ -65,6 +60,17 @@ deploy_exit() { cat /proc/self/fd/8 >"$audit_log" 2>/dev/null || status=1 fi chmod 0600 "$audit_log" 2>/dev/null || true + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=failed phase=%s status=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ + "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ + "${audit_phase:-post-consumption}" "$recorded_status" >>"$audit_log" || status=1 + else + printf 'time=%s environment=%s target=%s source=%s artifact=%s approval=%s approver=%s policy=%s checkpoint=%s authorized_by=%s gate=%s result=failed phase=%s status=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${req[ENVIRONMENT]}" "${req[TARGET_ID]}" \ + "${req[SOURCE_COMMIT]}" "${req[ARTIFACT_IMAGE]#*@}" "${req[APPROVAL_ID]}" "${req[APPROVAL_IDENTITY]}" "${req[POLICY_IDENTITY]}" \ + "${checkpoint[CHECKPOINT_ID]:-none}" "${production[AUTHORIZED_BY]:-none}" "${production[GATE_ID]:-none}" \ + "${audit_phase:-post-consumption}" "$recorded_status" >&8 || status=1 fi sync -f "$audit_log" 2>/dev/null || sync "$audit_log" 2>/dev/null || status=1 [[ -z ${audit_prefix_copy:-} ]] || rm -f "$audit_prefix_copy" @@ -74,12 +80,18 @@ deploy_exit() { # was never published. After publication the new pointer is the truth: the # application has changed and restoring the incumbent would falsify state. if [[ ${audit_pending:-0} == 1 && -n ${incumbent_pointer:-} && ${snapshot_pointer:-} != "$deployed_current" ]]; then + # Restore the incumbent snapshot content when the adapter deleted it. + if [[ -n ${incumbent_backup:-} && ! -e "$deployed_root/$incumbent_pointer/policy.conf" ]]; then + mkdir -m 0700 "$deployed_root/$incumbent_pointer" 2>/dev/null || true + install -m 0600 "$incumbent_backup/policy.conf" "$incumbent_backup/state.json" "$deployed_root/$incumbent_pointer/" 2>/dev/null || status=1 + fi if [[ ! -L "$deployed_current" || $(readlink "$deployed_current") != "$incumbent_pointer" ]]; then rm -f -- "$deployed_current" ln -s "$incumbent_pointer" "$deployed_current" 2>/dev/null || status=1 sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || status=1 fi fi + [[ -z ${incumbent_backup:-} ]] || rm -rf -- "$incumbent_backup" rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || status=1 if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then @@ -91,6 +103,11 @@ deploy_exit() { } expected_uid=0 [[ "$testing" != 1 ]] || expected_uid=$(id -u) +# Docker checks must target the host's local daemon, not an inherited remote +# DOCKER_HOST or selected context. +DOCKER_HOST=unix://$(root_path /run/docker.sock) +export DOCKER_HOST +unset DOCKER_CONTEXT declare -A production=() secure_file() { local path=$1 description=$2 mode=${3:-600} @@ -450,6 +467,8 @@ PY # Capture the validated incumbent pointer independently of adapter-writable # state so a failed adapter cannot destroy or falsify the rollback point. incumbent_pointer=$(readlink "$deployed_current") + incumbent_backup=$(mktemp -d "$state_root/.incumbent.XXXXXX") + install -m 0600 "$deployed_current/policy.conf" "$deployed_current/state.json" "$incumbent_backup/" set +e env CI_FLEET_DEPLOYER_CONFIG="$config" CI_FLEET_DEPLOYER_REQUEST="$request_snapshot" "$adapter_path" deploy adapter_status=$? diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 66b9080..84fab99 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -359,6 +359,11 @@ validate_evidence() { require_host() { local command os_id os_version available required systemd_state disk_path + # Docker checks must target the host's local daemon, not an operator shell's + # remote DOCKER_HOST or selected context. + DOCKER_HOST=unix://$(root_path /run/docker.sock) + export DOCKER_HOST + unset DOCKER_CONTEXT for command in bash awk cut sort stat sha256sum readlink realpath install cmp mv cp rm mkdir mktemp chmod ln flock kill timeout env git python3 docker systemctl systemd-analyze systemd-inhibit timedatectl curl df date; do command -v "$command" >/dev/null || block "$command is required" done @@ -974,7 +979,7 @@ publish_deployed_snapshot() { # validated rollback pair; only identical bytes short-circuit. if [[ -L "$deployed_current" ]]; then incumbent=$(readlink -f "$deployed_current") - if [[ $incumbent == "$deployed_root"/.snapshot.* && -f "$incumbent/policy.conf" && -f "$incumbent/state.json" ]] && cmp -s "$incumbent/policy.conf" "$policy" && cmp -s "$incumbent/state.json" "$state"; then return; fi + if [[ $incumbent == "$deployed_root"/.snapshot.* && ! -L "$incumbent" && -d "$incumbent" && $(stat -c '%u:%a' "$incumbent") == "$expected_uid:700" && -f "$incumbent/policy.conf" && ! -L "$incumbent/policy.conf" && -f "$incumbent/state.json" && ! -L "$incumbent/state.json" ]] && cmp -s "$incumbent/policy.conf" "$policy" && cmp -s "$incumbent/state.json" "$state"; then return; fi fi retired=$(readlink "$deployed_current") [[ "$retired" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'current deployed snapshot pointer is unsafe' diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 7356421..6218fca 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -1022,12 +1022,16 @@ perform_check() { note "CHECK_OK controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" } -perform_converge() { - local count existing_status desired_controller_id=$controller_id - # A deployer host is a separate role; never start a controller on it. +reject_deployer_host() { + # A deployer host is a separate role; never run controller mutations on it. if compgen -G "$(root_path /etc/systemd/system)/ci-fleet-deployer*" >/dev/null || [[ -e "$(root_path /var/lib/ci-fleet-deployer)" || -e "$(root_path /etc/ci-fleet-deployer)" ]]; then die 'deployer host state is present; controller and deployer roles are separate hosts' fi +} + +perform_converge() { + local count existing_status desired_controller_id=$controller_id + reject_deployer_host if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then die '--upgrade requires an existing managed installation; use --install or --adopt' fi @@ -1067,6 +1071,7 @@ latest_checkpoint() { } perform_rollback() { + reject_deployer_host checkpoint_dir=$(latest_checkpoint) [[ -n "$checkpoint_dir" ]] || die 'no controller checkpoint is available' load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" From 5c8302c8bf9e9f69ee399211f3a882dbb37fb5cb Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 11:07:16 -0500 Subject: [PATCH 83/90] fix: shared role-admission lock, maintenance Docker sanitization, per-file incumbent restore, shortcut metadata, pre-adapter durability - Both installers now flock a shared /run/ci-fleet-role-admission.lock for their whole mutating run, closing the concurrent-install race. - require_maintenance_host applies the same local-Docker sanitization as require_host. - deploy_exit validates and restores each incumbent snapshot file independently. - The publication shortcut requires root 0600 policy and state files. - perform_rollback syncs all rolled-back core boundaries before invoking the adapter that publishes the commit marker. --- scripts/deployer-runtime.sh | 10 +++++++--- scripts/install-deployer.sh | 19 ++++++++++++++++++- scripts/install-worker-controller.sh | 6 ++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 0f3647e..e4ddb20 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -81,9 +81,13 @@ deploy_exit() { # application has changed and restoring the incumbent would falsify state. if [[ ${audit_pending:-0} == 1 && -n ${incumbent_pointer:-} && ${snapshot_pointer:-} != "$deployed_current" ]]; then # Restore the incumbent snapshot content when the adapter deleted it. - if [[ -n ${incumbent_backup:-} && ! -e "$deployed_root/$incumbent_pointer/policy.conf" ]]; then - mkdir -m 0700 "$deployed_root/$incumbent_pointer" 2>/dev/null || true - install -m 0600 "$incumbent_backup/policy.conf" "$incumbent_backup/state.json" "$deployed_root/$incumbent_pointer/" 2>/dev/null || status=1 + if [[ -n ${incumbent_backup:-} ]]; then + for incumbent_file in policy.conf state.json; do + if [[ ! -f "$deployed_root/$incumbent_pointer/$incumbent_file" || -L "$deployed_root/$incumbent_pointer/$incumbent_file" ]] || ! cmp -s "$incumbent_backup/$incumbent_file" "$deployed_root/$incumbent_pointer/$incumbent_file"; then + mkdir -m 0700 "$deployed_root/$incumbent_pointer" 2>/dev/null || true + install -m 0600 "$incumbent_backup/$incumbent_file" "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 + fi + done fi if [[ ! -L "$deployed_current" || $(readlink "$deployed_current") != "$incumbent_pointer" ]]; then rm -f -- "$deployed_current" diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 84fab99..0b44078 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -392,6 +392,11 @@ require_host() { } require_maintenance_host() { + # Docker checks must target the host's local daemon, not an operator shell's + # remote DOCKER_HOST or selected context. + DOCKER_HOST=unix://$(root_path /run/docker.sock) + export DOCKER_HOST + unset DOCKER_CONTEXT local command for command in bash awk cut stat sha256sum readlink realpath install cp rm mkdir mktemp chmod ln mv flock kill timeout env python3 systemctl systemd-analyze date; do command -v "$command" >/dev/null || block "$command is required for maintenance" @@ -549,6 +554,11 @@ acquire_lock() { if [[ -e "$state_root" || -L "$state_root" ]]; then secure_directory "$state_root" 700 0 || block 'managed state boundary is unsafe'; fi exec 9<"$lock_root" flock -n 9 || block 'another deployer installer operation is running' + # Role admission is serialized across the controller and deployer installers: + # both flock this shared path for their whole mutating run. + install -d -m 0755 "$(root_path /run)" + exec 8>"$(root_path /run/ci-fleet-role-admission.lock)" + flock -n 8 || block 'another ci-fleet role installation is already running' [[ ! -L "$active_operation" ]] || block 'active operation marker is an unsafe symlink' if [[ -e "$active_operation" && ! -f "$active_operation" ]]; then block 'active operation marker has an unsafe type'; fi if [[ -e "$active_operation" ]] && ! active_deployment; then @@ -979,7 +989,7 @@ publish_deployed_snapshot() { # validated rollback pair; only identical bytes short-circuit. if [[ -L "$deployed_current" ]]; then incumbent=$(readlink -f "$deployed_current") - if [[ $incumbent == "$deployed_root"/.snapshot.* && ! -L "$incumbent" && -d "$incumbent" && $(stat -c '%u:%a' "$incumbent") == "$expected_uid:700" && -f "$incumbent/policy.conf" && ! -L "$incumbent/policy.conf" && -f "$incumbent/state.json" && ! -L "$incumbent/state.json" ]] && cmp -s "$incumbent/policy.conf" "$policy" && cmp -s "$incumbent/state.json" "$state"; then return; fi + if [[ $incumbent == "$deployed_root"/.snapshot.* && ! -L "$incumbent" && -d "$incumbent" && $(stat -c '%u:%a' "$incumbent") == "$expected_uid:700" && -f "$incumbent/policy.conf" && ! -L "$incumbent/policy.conf" && $(stat -c '%u:%a' "$incumbent/policy.conf") == "$expected_uid:600" && -f "$incumbent/state.json" && ! -L "$incumbent/state.json" && $(stat -c '%u:%a' "$incumbent/state.json") == "$expected_uid:600" ]] && cmp -s "$incumbent/policy.conf" "$policy" && cmp -s "$incumbent/state.json" "$state"; then return; fi fi retired=$(readlink "$deployed_current") [[ "$retired" =~ ^\.snapshot\.[A-Za-z0-9._-]+$ ]] || block 'current deployed snapshot pointer is unsafe' @@ -1215,6 +1225,13 @@ PY mv -Tf "$active_policy.new" "$active_policy" mv -Tf "$state_file.new" "$state_file" reject_mixed_role + # The adapter publishes the authoritative commit marker; the rolled-back + # core (pointer, units, policy, state) must be durable before that call, or + # a power loss could leave the marker durable over reverted core state. + sync -f "$state_file" "$active_policy" 2>/dev/null || die 'rolled-back host state is not durable' + sync -f "$state_root" 2>/dev/null || die 'rolled-back host state is not durable' + sync -f "$install_root" 2>/dev/null || die 'rolled-back install root is not durable' + sync -f "$systemd_root" 2>/dev/null || die 'rolled-back systemd boundary is not durable' # Run the adapter call under a shutdown inhibitor, like the deploy runtime, # so a normal shutdown cannot strand a partially applied adapter rollback # between core mutation and commit-marker publication. diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 6218fca..3efb82d 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -1115,6 +1115,12 @@ else exec 9>"$lock_file" flock -n 9 || die 'another ci-fleet installer or drift check is already running' fi +# Role admission is serialized across the controller and deployer installers: +# both flock this shared path for their whole mutating run. +role_lock=$(root_path /run/ci-fleet-role-admission.lock) +install -d -m 0755 "$(dirname "$role_lock")" +exec 8>"$role_lock" +flock -n 8 || die 'another ci-fleet role installation is already running' case "$mode" in check|install|adopt|upgrade) validate_common_arguments From d46ed5d6e107fe70bf5fcb03e456cb8050f72449 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Thu, 13 Aug 2026 11:55:51 -0500 Subject: [PATCH 84/90] fix: incumbent metadata restore, audit metadata repair, retained-pair preservation - deploy_exit restores incumbent snapshot directory and file modes as well as contents, and repairs audit-log mode drift before appending the failure record. - The runtime keeps a protected copy of the last-known-good pair across the adapter call and restores it before committing success or failure. --- scripts/deployer-runtime.sh | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index e4ddb20..c83557c 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -52,6 +52,7 @@ deploy_exit() { # If the adapter replaced the audit log, restore the durable prefix copy # (or the opened inode) under its name before appending the terminal # failure record, so the record is never written only to an unlinked inode. + if [[ -e "$audit_log" && ! -L "$audit_log" && $(stat -c '%u:%a' "$audit_log" 2>/dev/null) != "$expected_uid:600" ]]; then chmod 0600 "$audit_log" 2>/dev/null || status=1; fi if [[ ! -e "$audit_log" || $(stat -Lc '%d:%i' /proc/self/fd/8 2>/dev/null) != $(stat -c '%d:%i' "$audit_log" 2>/dev/null) ]]; then rm -f -- "$audit_log" if [[ -n ${audit_prefix_copy:-} && -f $audit_prefix_copy ]]; then @@ -82,10 +83,16 @@ deploy_exit() { if [[ ${audit_pending:-0} == 1 && -n ${incumbent_pointer:-} && ${snapshot_pointer:-} != "$deployed_current" ]]; then # Restore the incumbent snapshot content when the adapter deleted it. if [[ -n ${incumbent_backup:-} ]]; then + if [[ -e "$deployed_root/$incumbent_pointer" || -L "$deployed_root/$incumbent_pointer" ]]; then + [[ -d "$deployed_root/$incumbent_pointer" && ! -L "$deployed_root/$incumbent_pointer" ]] || { rm -rf -- "${deployed_root:?}/$incumbent_pointer"; mkdir -m 0700 "$deployed_root/$incumbent_pointer"; } + chmod 0700 "$deployed_root/$incumbent_pointer" 2>/dev/null || status=1 + fi for incumbent_file in policy.conf state.json; do if [[ ! -f "$deployed_root/$incumbent_pointer/$incumbent_file" || -L "$deployed_root/$incumbent_pointer/$incumbent_file" ]] || ! cmp -s "$incumbent_backup/$incumbent_file" "$deployed_root/$incumbent_pointer/$incumbent_file"; then mkdir -m 0700 "$deployed_root/$incumbent_pointer" 2>/dev/null || true install -m 0600 "$incumbent_backup/$incumbent_file" "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 + else + chmod 0600 "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 fi done fi @@ -96,6 +103,14 @@ deploy_exit() { fi fi [[ -z ${incumbent_backup:-} ]] || rm -rf -- "$incumbent_backup" + if [[ -n ${lkg_backup:-} ]]; then + for lkg_file in last-known-good.json last-known-good-policy.conf; do + if [[ ! -f "$state_root/$lkg_file" || -L "$state_root/$lkg_file" ]] || ! cmp -s "$lkg_backup/$lkg_file" "$state_root/$lkg_file"; then + install -m 0600 "$lkg_backup/$lkg_file" "$state_root/$lkg_file" 2>/dev/null || status=1 + fi + done + rm -rf -- "$lkg_backup" + fi rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || status=1 if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then @@ -468,6 +483,12 @@ PY audit_prefix_sha=$(sha256sum "$audit_log" | cut -d' ' -f1) audit_prefix_copy=$(mktemp "$state_root/.audit-prefix.XXXXXX") install -m 0600 "$audit_log" "$audit_prefix_copy" + # Preserve the retained rollback pair across the adapter call; the adapter + # has write access to the state root. + if [[ -f "$previous_state" && -f "$previous_policy" ]]; then + lkg_backup=$(mktemp -d "$state_root/.lkg.XXXXXX") + install -m 0600 "$previous_state" "$previous_policy" "$lkg_backup/" + fi # Capture the validated incumbent pointer independently of adapter-writable # state so a failed adapter cannot destroy or falsify the rollback point. incumbent_pointer=$(readlink "$deployed_current") @@ -537,7 +558,18 @@ PY sync -f "$consumed_marker" 2>/dev/null || sync "$consumed_marker" 2>/dev/null || die 'deployment request consumption marker is not durable' sync -f "$consumed_root" 2>/dev/null || sync "$consumed_root" 2>/dev/null || die 'deployment request consumption is not durable' fi - # All fallible commit work is done; the incumbent snapshot can be retired. + # All fallible commit work is done; restore the retained pair if the + # adapter touched it, then retire the incumbent snapshot. + if [[ -n ${lkg_backup:-} ]]; then + for lkg_file in last-known-good.json last-known-good-policy.conf; do + if [[ ! -f "$state_root/$lkg_file" || -L "$state_root/$lkg_file" ]] || ! cmp -s "$lkg_backup/$lkg_file" "$state_root/$lkg_file"; then + install -m 0600 "$lkg_backup/$lkg_file" "$state_root/$lkg_file" || die 'retained rollback pair restoration failed' + fi + done + sync -f "$state_root" 2>/dev/null || die 'retained rollback pair restoration is not durable' + rm -rf -- "$lkg_backup" + lkg_backup= + fi if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'retired deployed snapshot is not durable' audit_pending=0 From abb8dd7a57af031fe327b763047bfd5900dfd134 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Fri, 14 Aug 2026 12:18:05 -0500 Subject: [PATCH 85/90] fix: restore recovery metadata after adapter drift --- scripts/deployer-runtime.sh | 14 ++++++++-- scripts/test-install-deployer.sh | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index c83557c..2802d64 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -52,7 +52,10 @@ deploy_exit() { # If the adapter replaced the audit log, restore the durable prefix copy # (or the opened inode) under its name before appending the terminal # failure record, so the record is never written only to an unlinked inode. - if [[ -e "$audit_log" && ! -L "$audit_log" && $(stat -c '%u:%a' "$audit_log" 2>/dev/null) != "$expected_uid:600" ]]; then chmod 0600 "$audit_log" 2>/dev/null || status=1; fi + if [[ -e "$audit_log" && ! -L "$audit_log" && $(stat -c '%u:%a' "$audit_log" 2>/dev/null) != "$expected_uid:600" ]]; then + chown "$expected_uid" "$audit_log" 2>/dev/null || status=1 + chmod 0600 "$audit_log" 2>/dev/null || status=1 + fi if [[ ! -e "$audit_log" || $(stat -Lc '%d:%i' /proc/self/fd/8 2>/dev/null) != $(stat -c '%d:%i' "$audit_log" 2>/dev/null) ]]; then rm -f -- "$audit_log" if [[ -n ${audit_prefix_copy:-} && -f $audit_prefix_copy ]]; then @@ -85,15 +88,16 @@ deploy_exit() { if [[ -n ${incumbent_backup:-} ]]; then if [[ -e "$deployed_root/$incumbent_pointer" || -L "$deployed_root/$incumbent_pointer" ]]; then [[ -d "$deployed_root/$incumbent_pointer" && ! -L "$deployed_root/$incumbent_pointer" ]] || { rm -rf -- "${deployed_root:?}/$incumbent_pointer"; mkdir -m 0700 "$deployed_root/$incumbent_pointer"; } + chown "$expected_uid" "$deployed_root/$incumbent_pointer" 2>/dev/null || status=1 chmod 0700 "$deployed_root/$incumbent_pointer" 2>/dev/null || status=1 fi for incumbent_file in policy.conf state.json; do if [[ ! -f "$deployed_root/$incumbent_pointer/$incumbent_file" || -L "$deployed_root/$incumbent_pointer/$incumbent_file" ]] || ! cmp -s "$incumbent_backup/$incumbent_file" "$deployed_root/$incumbent_pointer/$incumbent_file"; then mkdir -m 0700 "$deployed_root/$incumbent_pointer" 2>/dev/null || true install -m 0600 "$incumbent_backup/$incumbent_file" "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 - else - chmod 0600 "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 fi + chown "$expected_uid" "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 + chmod 0600 "$deployed_root/$incumbent_pointer/$incumbent_file" 2>/dev/null || status=1 done fi if [[ ! -L "$deployed_current" || $(readlink "$deployed_current") != "$incumbent_pointer" ]]; then @@ -108,6 +112,8 @@ deploy_exit() { if [[ ! -f "$state_root/$lkg_file" || -L "$state_root/$lkg_file" ]] || ! cmp -s "$lkg_backup/$lkg_file" "$state_root/$lkg_file"; then install -m 0600 "$lkg_backup/$lkg_file" "$state_root/$lkg_file" 2>/dev/null || status=1 fi + chown "$expected_uid" "$state_root/$lkg_file" 2>/dev/null || status=1 + chmod 0600 "$state_root/$lkg_file" 2>/dev/null || status=1 done rm -rf -- "$lkg_backup" fi @@ -565,6 +571,8 @@ PY if [[ ! -f "$state_root/$lkg_file" || -L "$state_root/$lkg_file" ]] || ! cmp -s "$lkg_backup/$lkg_file" "$state_root/$lkg_file"; then install -m 0600 "$lkg_backup/$lkg_file" "$state_root/$lkg_file" || die 'retained rollback pair restoration failed' fi + chown "$expected_uid" "$state_root/$lkg_file" || die 'retained rollback pair ownership restoration failed' + chmod 0600 "$state_root/$lkg_file" || die 'retained rollback pair mode restoration failed' done sync -f "$state_root" 2>/dev/null || die 'retained rollback pair restoration is not durable' rm -rf -- "$lkg_backup" diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 52dc7e8..21f4d5d 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -151,11 +151,22 @@ if [[ -n ${FAKE_ADAPTER_MUTATE_AUDIT_PATH:-} ]]; then fi fi if [[ -n ${FAKE_ADAPTER_CHMOD_DURING:-} ]]; then chmod 0644 "$FAKE_ADAPTER_CHMOD_DURING"; fi +if [[ -n ${FAKE_ADAPTER_CHOWN_DURING:-} && $EUID == 0 ]]; then chown 65534 "$FAKE_ADAPTER_CHOWN_DURING"; fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_INCUMBENT_PATH:-} ]]; then + chmod 0755 "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH" + chmod 0644 "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH"/policy.conf "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH"/state.json + if ((EUID == 0)); then chown 65534 "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH" "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH"/policy.conf "$FAKE_ADAPTER_MUTATE_INCUMBENT_PATH"/state.json; fi +fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_LKG_ROOT:-} ]]; then + chmod 0644 "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good.json "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good-policy.conf + if ((EUID == 0)); then chown 65534 "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good.json "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good-policy.conf; fi +fi if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT:-} ]]; then snapshot=$(find "$FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT" -maxdepth 1 -type d -name '.snapshot.*' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-) [[ -n "$snapshot" ]] && printf 'adapter-mutation\n' >>"$snapshot/policy.conf" fi if [[ "$1" == deploy && -n ${FAKE_ADAPTER_SIGNAL_PPID:-} ]]; then kill -TERM "$PPID"; sleep 5; fi +if [[ -e "${FAKE_ADAPTER_FAIL_AFTER_MUTATION:-/nonexistent}" ]]; then exit 42; fi EOF chmod 0700 "$adapter" export FAKE_ADAPTER_LOG=$tmp/adapter.log @@ -1641,6 +1652,43 @@ cp "$approval" "$request"; chmod 0600 "$request" expect_failure 'deployment request was already consumed' "$runtime" deploy >/dev/null rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" +# A failing adapter cannot leave recovery state or the audit log with unsafe metadata. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=metadata-drift-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:55Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +incumbent_dir=$(readlink -f "$deployed_current") +printf 'deploy\n' >"$tmp/fail-after-metadata" +export FAKE_ADAPTER_MUTATE_INCUMBENT_PATH=$incumbent_dir +export FAKE_ADAPTER_MUTATE_LKG_ROOT=$root/var/lib/ci-fleet-deployer +export FAKE_ADAPTER_CHMOD_DURING=$root/var/log/ci-fleet-deployer/audit.log +export FAKE_ADAPTER_CHOWN_DURING=$root/var/log/ci-fleet-deployer/audit.log +export FAKE_ADAPTER_FAIL_AFTER_MUTATION=$tmp/fail-after-metadata +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_MUTATE_INCUMBENT_PATH FAKE_ADAPTER_MUTATE_LKG_ROOT FAKE_ADAPTER_CHMOD_DURING FAKE_ADAPTER_CHOWN_DURING FAKE_ADAPTER_FAIL_AFTER_MUTATION +expected_uid=$(id -u) +[[ $(stat -c '%u:%a' "$incumbent_dir") == "$expected_uid:700" ]] || fail 'failed adapter left incumbent directory metadata unsafe' +for file in "$incumbent_dir"/policy.conf "$incumbent_dir"/state.json "$root/var/lib/ci-fleet-deployer/last-known-good.json "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf "$root/var/log/ci-fleet-deployer/audit.log"; do + [[ $(stat -c '%u:%a' "$file") == "$expected_uid:600" ]] || fail "failed adapter left recovery metadata unsafe: $file" +done +rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" + +# A successful adapter also cannot leave the retained rollback pair unsafe. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=lkg-metadata-drift-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:05:56Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +FAKE_ADAPTER_MUTATE_LKG_ROOT=$root/var/lib/ci-fleet-deployer expect_success "$runtime" deploy >/dev/null +for file in "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf"; do + [[ $(stat -c '%u:%a' "$file") == "$expected_uid:600" ]] || fail "successful adapter left retained rollback metadata unsafe: $file" +done + # Signal at the deployed-snapshot publication boundary must not delete the published snapshot. write_evidence staging example-staging python3 - "$approval" <<'PY' From ee99bfc926bacf4330464cdd7d3f799a3a9ec097 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Fri, 14 Aug 2026 21:06:06 -0500 Subject: [PATCH 86/90] test: separate recovery metadata fixture paths --- scripts/test-install-deployer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 21f4d5d..7e1c70a 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -1671,7 +1671,7 @@ expect_failure 'deployment adapter failed after approval consumption' "$runtime" unset FAKE_ADAPTER_MUTATE_INCUMBENT_PATH FAKE_ADAPTER_MUTATE_LKG_ROOT FAKE_ADAPTER_CHMOD_DURING FAKE_ADAPTER_CHOWN_DURING FAKE_ADAPTER_FAIL_AFTER_MUTATION expected_uid=$(id -u) [[ $(stat -c '%u:%a' "$incumbent_dir") == "$expected_uid:700" ]] || fail 'failed adapter left incumbent directory metadata unsafe' -for file in "$incumbent_dir"/policy.conf "$incumbent_dir"/state.json "$root/var/lib/ci-fleet-deployer/last-known-good.json "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf "$root/var/log/ci-fleet-deployer/audit.log"; do +for file in "$incumbent_dir"/policy.conf "$incumbent_dir"/state.json "$root/var/lib/ci-fleet-deployer/last-known-good.json" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" "$root/var/log/ci-fleet-deployer/audit.log"; do [[ $(stat -c '%u:%a' "$file") == "$expected_uid:600" ]] || fail "failed adapter left recovery metadata unsafe: $file" done rm -rf "$root/var/lib/ci-fleet-deployer/consumed-requests" From b58e8d2e6685e61db5b231889cd9676ff2b1ec7b Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 15 Aug 2026 19:10:17 -0500 Subject: [PATCH 87/90] fix: preserve deployer rollback across adapter updates --- docs/DEPLOYER-HOST.md | 10 +++--- scripts/deployer-runtime.sh | 54 ++++++++++++++++++-------------- scripts/install-deployer.sh | 2 +- scripts/test-install-deployer.sh | 39 ++++++++++++++++++++++- 4 files changed, 75 insertions(+), 30 deletions(-) diff --git a/docs/DEPLOYER-HOST.md b/docs/DEPLOYER-HOST.md index 863b075..6b2045a 100644 --- a/docs/DEPLOYER-HOST.md +++ b/docs/DEPLOYER-HOST.md @@ -81,8 +81,9 @@ sudo install -d -o root -g root -m 0700 \ /etc/ci-fleet-deployer/adapters \ /etc/ci-fleet-deployer/credentials \ /etc/ci-fleet-deployer/evidence +adapter_sha=$(sha256sum ./application-adapter | cut -d' ' -f1) sudo install -o root -g root -m 0700 ./application-adapter \ - /etc/ci-fleet-deployer/adapters/application-adapter + "/etc/ci-fleet-deployer/adapters/application-adapter.$adapter_sha" sudo install -o root -g root -m 0600 /dev/null \ /etc/ci-fleet-deployer/credentials/application.credential sudoedit /etc/ci-fleet-deployer/credentials/application.credential @@ -163,7 +164,8 @@ Create the bounded configuration. Values cannot contain shell expressions; the i ```bash core_ref=$(git rev-parse HEAD) -adapter_sha=$(sudo sha256sum /etc/ci-fleet-deployer/adapters/application-adapter | cut -d' ' -f1) +adapter_sha=$(sha256sum ./application-adapter | cut -d' ' -f1) +adapter_path=/etc/ci-fleet-deployer/adapters/application-adapter.$adapter_sha sudo install -o root -g root -m 0600 /dev/null \ /etc/ci-fleet-deployer/deployer.conf { @@ -172,7 +174,7 @@ sudo install -o root -g root -m 0600 /dev/null \ printf 'ENVIRONMENT=staging\n' printf 'TARGET_ID=example-staging\n' printf 'DEPLOYER_IDENTITY=staging-deployer-01\n' - printf 'ADAPTER_PATH=/etc/ci-fleet-deployer/adapters/application-adapter\n' + printf 'ADAPTER_PATH=%s\n' "$adapter_path" printf 'ADAPTER_SHA256=%s\n' "$adapter_sha" printf 'CREDENTIAL_PROVIDER=file\n' printf 'CREDENTIAL_REF=/etc/ci-fleet-deployer/credentials/application.credential\n' @@ -216,7 +218,7 @@ sudo ./scripts/install-deployer.sh \ --repair --config /etc/ci-fleet-deployer/deployer.conf ``` -Upgrade after updating the exact core/source/artifact/approval/checkpoint fields; changes the host transactionally: +Upgrade after updating the exact core/source/artifact/approval/checkpoint fields; changes the host transactionally. Adapter bytes are immutable too: install changed bytes at a new digest-suffixed path, update `ADAPTER_PATH` and `ADAPTER_SHA256` together, and retain the old path while any deployed or last-known-good policy names it. Never overwrite the adapter at its existing path; pre-convergence verification intentionally requires those old bytes so rollback remains executable. ```bash sudo ./scripts/install-deployer.sh \ diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index 2802d64..b7b8aa4 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -29,6 +29,14 @@ audit_log=$log_root/audit.log systemd_root=$(root_path /etc/systemd/system) die() { printf 'ERROR: %s\n' "$*" >&2; exit 2; } +restore_encoded_file() { + local destination=$1 encoded=$2 temporary + temporary=$(mktemp "$state_root/.restore.XXXXXX") || return + printf '%s' "$encoded" | base64 -d >"$temporary" || { rm -f "$temporary"; return 1; } + chown "$expected_uid" "$temporary" || { rm -f "$temporary"; return 1; } + chmod 0600 "$temporary" || { rm -f "$temporary"; return 1; } + mv -Tf "$temporary" "$destination" +} if [[ $operation == deploy && -z ${CI_FLEET_DEPLOYER_INHIBITED:-} ]]; then export CI_FLEET_DEPLOYER_INHIBITED=1 [[ $testing != 1 || -z ${CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG:-} ]] || printf '%s\n' deploy >>"$CI_FLEET_DEPLOYER_TEST_INHIBITOR_LOG" @@ -38,7 +46,15 @@ fi deploy_exit() { local status=$? local recorded_status=${adapter_status:-$status} - local target + local target state_root_safe=1 + # The adapter may recursively clear its writable state root. Recreate only + # an absent boundary; an unsafe replacement remains a hard failure. + if [[ ${audit_pending:-0} == 1 && ! -e "$state_root" && ! -L "$state_root" ]]; then + install -d -m 0700 "$state_root" 2>/dev/null || { status=1; state_root_safe=0; } + fi + if [[ -e "$state_root" || -L "$state_root" ]]; then + [[ -d "$state_root" && ! -L "$state_root" && $(stat -c '%u:%a' "$state_root" 2>/dev/null) == "$expected_uid:700" ]] || { status=1; state_root_safe=0; } + fi # An adapter that deletes its own consumption marker must not defeat replay # protection; restore the durable marker rather than clearing audit_pending. if [[ ${audit_pending:-0} == 1 && -n ${consumed_marker:-} && ! -e "$consumed_marker" ]]; then @@ -107,15 +123,9 @@ deploy_exit() { fi fi [[ -z ${incumbent_backup:-} ]] || rm -rf -- "$incumbent_backup" - if [[ -n ${lkg_backup:-} ]]; then - for lkg_file in last-known-good.json last-known-good-policy.conf; do - if [[ ! -f "$state_root/$lkg_file" || -L "$state_root/$lkg_file" ]] || ! cmp -s "$lkg_backup/$lkg_file" "$state_root/$lkg_file"; then - install -m 0600 "$lkg_backup/$lkg_file" "$state_root/$lkg_file" 2>/dev/null || status=1 - fi - chown "$expected_uid" "$state_root/$lkg_file" 2>/dev/null || status=1 - chmod 0600 "$state_root/$lkg_file" 2>/dev/null || status=1 - done - rm -rf -- "$lkg_backup" + if [[ ${lkg_backed_up:-0} == 1 && $state_root_safe == 1 ]]; then + restore_encoded_file "$state_root/last-known-good.json" "$lkg_state_backup" 2>/dev/null || status=1 + restore_encoded_file "$state_root/last-known-good-policy.conf" "$lkg_policy_backup" 2>/dev/null || status=1 fi rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || status=1 @@ -489,11 +499,13 @@ PY audit_prefix_sha=$(sha256sum "$audit_log" | cut -d' ' -f1) audit_prefix_copy=$(mktemp "$state_root/.audit-prefix.XXXXXX") install -m 0600 "$audit_log" "$audit_prefix_copy" - # Preserve the retained rollback pair across the adapter call; the adapter - # has write access to the state root. + # Preserve the retained rollback pair in unexported process memory. A + # filesystem backup under the adapter-writable state root would be lost to + # the same recursive cleanup as the retained files. if [[ -f "$previous_state" && -f "$previous_policy" ]]; then - lkg_backup=$(mktemp -d "$state_root/.lkg.XXXXXX") - install -m 0600 "$previous_state" "$previous_policy" "$lkg_backup/" + lkg_state_backup=$(base64 -w0 "$previous_state") + lkg_policy_backup=$(base64 -w0 "$previous_policy") + lkg_backed_up=1 fi # Capture the validated incumbent pointer independently of adapter-writable # state so a failed adapter cannot destroy or falsify the rollback point. @@ -566,17 +578,11 @@ PY fi # All fallible commit work is done; restore the retained pair if the # adapter touched it, then retire the incumbent snapshot. - if [[ -n ${lkg_backup:-} ]]; then - for lkg_file in last-known-good.json last-known-good-policy.conf; do - if [[ ! -f "$state_root/$lkg_file" || -L "$state_root/$lkg_file" ]] || ! cmp -s "$lkg_backup/$lkg_file" "$state_root/$lkg_file"; then - install -m 0600 "$lkg_backup/$lkg_file" "$state_root/$lkg_file" || die 'retained rollback pair restoration failed' - fi - chown "$expected_uid" "$state_root/$lkg_file" || die 'retained rollback pair ownership restoration failed' - chmod 0600 "$state_root/$lkg_file" || die 'retained rollback pair mode restoration failed' - done + if [[ ${lkg_backed_up:-0} == 1 ]]; then + restore_encoded_file "$state_root/last-known-good.json" "$lkg_state_backup" || die 'retained rollback state restoration failed' + restore_encoded_file "$state_root/last-known-good-policy.conf" "$lkg_policy_backup" || die 'retained rollback policy restoration failed' sync -f "$state_root" 2>/dev/null || die 'retained rollback pair restoration is not durable' - rm -rf -- "$lkg_backup" - lkg_backup= + lkg_backed_up=0 fi if [[ -n "$retired_snapshot" && -d "$deployed_root/$retired_snapshot" && ! -L "$deployed_root/$retired_snapshot" ]]; then rm -rf -- "${deployed_root:?}/$retired_snapshot"; fi sync -f "$deployed_root" 2>/dev/null || sync "$deployed_root" 2>/dev/null || die 'retired deployed snapshot is not durable' diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 0b44078..6027f9f 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -364,7 +364,7 @@ require_host() { DOCKER_HOST=unix://$(root_path /run/docker.sock) export DOCKER_HOST unset DOCKER_CONTEXT - for command in bash awk cut sort stat sha256sum readlink realpath install cmp mv cp rm mkdir mktemp chmod ln flock kill timeout env git python3 docker systemctl systemd-analyze systemd-inhibit timedatectl curl df date; do + for command in bash awk base64 cut sort stat sha256sum readlink realpath install cmp mv cp rm mkdir mktemp chmod ln flock kill timeout env git python3 docker systemctl systemd-analyze systemd-inhibit timedatectl curl df date; do command -v "$command" >/dev/null || block "$command is required" done local os_release diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 7e1c70a..22a42e7 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -161,6 +161,7 @@ if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_LKG_ROOT:-} ]]; then chmod 0644 "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good.json "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good-policy.conf if ((EUID == 0)); then chown 65534 "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good.json "$FAKE_ADAPTER_MUTATE_LKG_ROOT"/last-known-good-policy.conf; fi fi +if [[ "$1" == deploy && -n ${FAKE_ADAPTER_DELETE_STATE_ROOT:-} ]]; then rm -rf -- "$FAKE_ADAPTER_DELETE_STATE_ROOT"; fi if [[ "$1" == deploy && -n ${FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT:-} ]]; then snapshot=$(find "$FAKE_ADAPTER_MUTATE_SNAPSHOT_ROOT" -maxdepth 1 -type d -name '.snapshot.*' -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-) [[ -n "$snapshot" ]] && printf 'adapter-mutation\n' >>"$snapshot/policy.conf" @@ -1739,6 +1740,42 @@ rm "$root/var/lib/ci-fleet-deployer/drained" ln -s "$tmp/missing-drain-target" "$root/var/lib/ci-fleet-deployer/drained" expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" deploy >/dev/null expect_failure 'drain marker must be a regular file, not a symlink' "$runtime" cleanup >/dev/null +rm "$root/var/lib/ci-fleet-deployer/drained" + +# Adapter upgrades use immutable versioned paths so the deployed policy keeps +# real rollback bytes while the candidate is validated and activated. +old_adapter=$adapter +old_adapter_sha=$(sha256sum "$old_adapter" | cut -d' ' -f1) +adapter=$root/etc/ci-fleet-deployer/adapters/application-adapter.v2 +cp "$old_adapter" "$adapter" +# shellcheck disable=SC2016 # Write literal adapter variables into the fixture. +printf '\nprintf "adapter-v2:%%s\\n" "$1" >>"${FAKE_ADAPTER_LOG:?}"\n' >>"$adapter" +chmod 0700 "$adapter" +write_config +expect_success "$installer" --upgrade --config "$config" >/dev/null +grep -Fxq "ADAPTER_PATH=$old_adapter" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" || fail 'adapter upgrade did not retain the old adapter path for rollback' +grep -Fxq "ADAPTER_SHA256=$old_adapter_sha" "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf" || fail 'adapter upgrade did not retain the old adapter digest for rollback' +check=$(expect_success "$installer" --check --config "$config") +grep -Fq 'rollback_available=yes' <<<"$check" || fail 'versioned adapter upgrade is not rollback-capable' + +# Recursive adapter cleanup of the writable state root must not destroy the +# retained rollback pair: its backup lives only in unexported process memory. +write_evidence staging example-staging +python3 - "$approval" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); p.write_text(p.read_text().replace('APPROVAL_ID=approval-20260808-1', 'APPROVAL_ID=state-root-delete-attempt').replace('APPROVED_AT=2026-08-08T20:00:00Z', 'APPROVED_AT=2026-08-08T20:07:00Z')) +PY +cp "$approval" "$request"; chmod 0600 "$request" +lkg_state_before=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") +lkg_policy_before=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf") +printf 'deploy\n' >"$tmp/fail-after-state-delete" +export FAKE_ADAPTER_DELETE_STATE_ROOT=$root/var/lib/ci-fleet-deployer FAKE_ADAPTER_FAIL_AFTER_MUTATION=$tmp/fail-after-state-delete +expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null +unset FAKE_ADAPTER_DELETE_STATE_ROOT FAKE_ADAPTER_FAIL_AFTER_MUTATION +[[ $lkg_state_before == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'recursive state cleanup destroyed retained rollback state' +[[ $lkg_policy_before == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf")" ]] || fail 'recursive state cleanup destroyed retained rollback policy' +compgen -G "$root/var/lib/ci-fleet-deployer/.lkg.*" >/dev/null && fail 'rollback backup remained inside adapter-writable state' # Bytes substituted into the live checkout after review must never reach a staged release. cp "$runtime" "$tmp/runtime.saved" @@ -1760,7 +1797,7 @@ if [[ -n $replacement ]]; then fi cat "$tmp/runtime.saved" >"$runtime" git -C "$repo_root" show "HEAD:scripts/deployer-runtime.sh" | cmp -s - "$runtime" || fail 'live checkout restoration diverged from HEAD' -rm "$root/var/lib/ci-fleet-deployer/drained" +rm -f "$root/var/lib/ci-fleet-deployer/drained" grep -Fq 'DEPLOYER-HOST.md' "$repo_root/docs/README.md" || fail 'operator index does not link the deployer runbook' [[ -x "$repo_root/scripts/test-deployer-units.sh" ]] || fail 'real systemd unit verification is not wired' From 515b071461911ad395734792aeb6e82da92da394 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 15 Aug 2026 19:49:45 -0500 Subject: [PATCH 88/90] fix: retain completed deployment rollback marker --- scripts/deployer-runtime.sh | 7 +++++++ scripts/test-install-deployer.sh | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/scripts/deployer-runtime.sh b/scripts/deployer-runtime.sh index b7b8aa4..ba4c1e0 100755 --- a/scripts/deployer-runtime.sh +++ b/scripts/deployer-runtime.sh @@ -127,6 +127,9 @@ deploy_exit() { restore_encoded_file "$state_root/last-known-good.json" "$lkg_state_backup" 2>/dev/null || status=1 restore_encoded_file "$state_root/last-known-good-policy.conf" "$lkg_policy_backup" 2>/dev/null || status=1 fi + if [[ ${audit_pending:-0} == 1 && ${last_request_backed_up:-0} == 1 && $state_root_safe == 1 ]]; then + restore_encoded_file "$last_request" "$last_request_backup" 2>/dev/null || status=1 + fi rm -f "$active" "${active_temporary:-}" "${request_snapshot:-}" "${policy_snapshot:-}" || true sync -f "$state_root" 2>/dev/null || sync "$state_root" 2>/dev/null || status=1 if [[ -n ${snapshot:-} && -d $snapshot && ! -L $snapshot ]]; then @@ -507,6 +510,10 @@ PY lkg_policy_backup=$(base64 -w0 "$previous_policy") lkg_backed_up=1 fi + if [[ -f "$last_request" && ! -L "$last_request" ]]; then + last_request_backup=$(base64 -w0 "$last_request") + last_request_backed_up=1 + fi # Capture the validated incumbent pointer independently of adapter-writable # state so a failed adapter cannot destroy or falsify the rollback point. incumbent_pointer=$(readlink "$deployed_current") diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 22a42e7..84c3fb6 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -1769,13 +1769,17 @@ PY cp "$approval" "$request"; chmod 0600 "$request" lkg_state_before=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json") lkg_policy_before=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf") +last_request_before=$(sha256sum "$root/var/lib/ci-fleet-deployer/last-request.conf") printf 'deploy\n' >"$tmp/fail-after-state-delete" export FAKE_ADAPTER_DELETE_STATE_ROOT=$root/var/lib/ci-fleet-deployer FAKE_ADAPTER_FAIL_AFTER_MUTATION=$tmp/fail-after-state-delete expect_failure 'deployment adapter failed after approval consumption' "$runtime" deploy >/dev/null unset FAKE_ADAPTER_DELETE_STATE_ROOT FAKE_ADAPTER_FAIL_AFTER_MUTATION [[ $lkg_state_before == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good.json")" ]] || fail 'recursive state cleanup destroyed retained rollback state' [[ $lkg_policy_before == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-known-good-policy.conf")" ]] || fail 'recursive state cleanup destroyed retained rollback policy' +[[ $last_request_before == "$(sha256sum "$root/var/lib/ci-fleet-deployer/last-request.conf")" ]] || fail 'recursive state cleanup destroyed completed-deployment rollback marker' compgen -G "$root/var/lib/ci-fleet-deployer/.lkg.*" >/dev/null && fail 'rollback backup remained inside adapter-writable state' +rollback=$(expect_success "$installer" --rollback --config "$config") +grep -Fq 'result=CHANGED' <<<"$rollback" || fail 'rollback after recursive state cleanup did not complete' # Bytes substituted into the live checkout after review must never reach a staged release. cp "$runtime" "$tmp/runtime.saved" From 668213b7b3180bea368a7c93068d7c06594eaa4b Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 21:28:58 -0500 Subject: [PATCH 89/90] fix: accept canonical Debian os-release symlink --- scripts/install-deployer.sh | 7 ++++++- scripts/test-install-deployer.sh | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/install-deployer.sh b/scripts/install-deployer.sh index 6027f9f..5e2be00 100755 --- a/scripts/install-deployer.sh +++ b/scripts/install-deployer.sh @@ -358,7 +358,7 @@ validate_evidence() { } require_host() { - local command os_id os_version available required systemd_state disk_path + local command os_id os_version available required systemd_state disk_path canonical # Docker checks must target the host's local daemon, not an operator shell's # remote DOCKER_HOST or selected context. DOCKER_HOST=unix://$(root_path /run/docker.sock) @@ -369,6 +369,11 @@ require_host() { done local os_release os_release=$(root_path /etc/os-release) + if [[ -L "$os_release" ]]; then + canonical=$(readlink -f -- "$os_release") + [[ "$canonical" == "$(root_path /usr/lib/os-release)" ]] || block 'supported Linux os-release metadata is missing' + os_release=$canonical + fi [[ -f "$os_release" && ! -L "$os_release" ]] || block 'supported Linux os-release metadata is missing' os_id=$(awk -F= '$1=="ID" {gsub(/"/,"",$2); print $2}' "$os_release") os_version=$(awk -F= '$1=="VERSION_ID" {gsub(/"/,"",$2); print $2}' "$os_release") diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 84c3fb6..9a72003 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -286,6 +286,13 @@ cp "$root/etc/os-release" "$tmp/os-release" printf 'ID=alpine\nVERSION_ID=3.20\n' >"$root/etc/os-release" expect_failure 'unsupported Linux distribution or release' "$installer" --check --config "$config" >/dev/null cp "$tmp/os-release" "$root/etc/os-release" +mkdir -p "$root/usr/lib" +cp "$root/etc/os-release" "$root/usr/lib/os-release" +rm "$root/etc/os-release" +ln -s ../usr/lib/os-release "$root/etc/os-release" +"$installer" --check --config "$config" >/dev/null || fail 'canonical Debian os-release symlink was rejected' +rm "$root/etc/os-release" "$root/usr/lib/os-release" +cp "$tmp/os-release" "$root/etc/os-release" FAKE_TIME_SYNC=no expect_failure 'host time is not synchronized' "$installer" --check --config "$config" >/dev/null FAKE_SYSTEMD_FAIL=1 expect_failure 'systemd is unavailable' "$installer" --check --config "$config" >/dev/null FAKE_DOCKER_INFO_EXIT=1 expect_failure 'Docker Engine is unavailable' "$installer" --check --config "$config" >/dev/null From 2ad48c75c1412a146086b7a18fb3027f423515f1 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 21:35:07 -0500 Subject: [PATCH 90/90] test: exercise os-release symlink during install --- scripts/test-install-deployer.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/test-install-deployer.sh b/scripts/test-install-deployer.sh index 9a72003..6165fa3 100755 --- a/scripts/test-install-deployer.sh +++ b/scripts/test-install-deployer.sh @@ -286,13 +286,6 @@ cp "$root/etc/os-release" "$tmp/os-release" printf 'ID=alpine\nVERSION_ID=3.20\n' >"$root/etc/os-release" expect_failure 'unsupported Linux distribution or release' "$installer" --check --config "$config" >/dev/null cp "$tmp/os-release" "$root/etc/os-release" -mkdir -p "$root/usr/lib" -cp "$root/etc/os-release" "$root/usr/lib/os-release" -rm "$root/etc/os-release" -ln -s ../usr/lib/os-release "$root/etc/os-release" -"$installer" --check --config "$config" >/dev/null || fail 'canonical Debian os-release symlink was rejected' -rm "$root/etc/os-release" "$root/usr/lib/os-release" -cp "$tmp/os-release" "$root/etc/os-release" FAKE_TIME_SYNC=no expect_failure 'host time is not synchronized' "$installer" --check --config "$config" >/dev/null FAKE_SYSTEMD_FAIL=1 expect_failure 'systemd is unavailable' "$installer" --check --config "$config" >/dev/null FAKE_DOCKER_INFO_EXIT=1 expect_failure 'Docker Engine is unavailable' "$installer" --check --config "$config" >/dev/null @@ -391,9 +384,15 @@ fresh_uninstall=$(expect_success "$installer" --uninstall --config "$config") [[ "$after" == "$(find "$root" -printf '%P %y %m\n' | sort | sha256sum)" ]] || fail 'fresh uninstall mutated an unmanaged host' grep -Fq 'result=NO_CHANGE' <<<"$fresh_uninstall" || fail 'fresh uninstall did not report NO_CHANGE' +mkdir -p "$root/usr/lib" +cp "$root/etc/os-release" "$root/usr/lib/os-release" +rm "$root/etc/os-release" +ln -s ../usr/lib/os-release "$root/etc/os-release" export FAKE_ADAPTER_FORBID_CONFIG_PATH=$config first=$(expect_success "$installer" --install --config "$config") unset FAKE_ADAPTER_FORBID_CONFIG_PATH +rm "$root/etc/os-release" "$root/usr/lib/os-release" +cp "$tmp/os-release" "$root/etc/os-release" grep -Fq 'REPORT action=install result=CHANGED environment=staging' <<<"$first" || fail 'fresh install report is incomplete' [[ -L "$root/opt/ci-fleet-deployer/current" ]] || fail 'fresh install lacks atomic current release' [[ $(stat -c %a "$root/var/lib/ci-fleet-deployer/install-state.json") == 600 ]] || fail 'install state mode is not 0600'