From b86cb94df90850f0429663045c986df2495852a1 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 15 Aug 2026 22:41:51 -0500 Subject: [PATCH 01/16] feat: add isolated tester host installer --- docs/OPERATOR-WORKFLOWS.md | 2 +- docs/README.md | 1 + docs/TESTER-HOST.md | 110 ++++++++ host/systemd/ci-fleet-tester-cleanup.service | 20 ++ host/systemd/ci-fleet-tester-cleanup.timer | 12 + host/systemd/ci-fleet-tester-health.service | 20 ++ host/systemd/ci-fleet-tester-health.timer | 12 + scripts/fixtures/fake-tester-docker.sh | 41 +++ scripts/install-tester.sh | 174 +++++++++++++ scripts/test-install-tester.sh | 104 ++++++++ scripts/tester-runtime.sh | 258 +++++++++++++++++++ scripts/validate.sh | 1 + 12 files changed, 754 insertions(+), 1 deletion(-) create mode 100644 docs/TESTER-HOST.md create mode 100644 host/systemd/ci-fleet-tester-cleanup.service create mode 100644 host/systemd/ci-fleet-tester-cleanup.timer create mode 100644 host/systemd/ci-fleet-tester-health.service create mode 100644 host/systemd/ci-fleet-tester-health.timer create mode 100755 scripts/fixtures/fake-tester-docker.sh create mode 100755 scripts/install-tester.sh create mode 100755 scripts/test-install-tester.sh create mode 100755 scripts/tester-runtime.sh diff --git a/docs/OPERATOR-WORKFLOWS.md b/docs/OPERATOR-WORKFLOWS.md index f37f144..4477df7 100644 --- a/docs/OPERATOR-WORKFLOWS.md +++ b/docs/OPERATOR-WORKFLOWS.md @@ -19,7 +19,7 @@ Use this page to choose an outcome, confirm the responsible role, and either fol | Authorize and onboard an application project | Current, experimental | Application repository, private configuration, and GitHub runner-group policy | Yes; no host mutation | Follow [adding a project](ADDING-A-PROJECT.md), then the [migration procedure](MIGRATING-EXISTING-CI.md). | | Create controller GitHub credentials and runner-group prerequisites | Current manual procedure | GitHub web UI and the target host | Yes | Follow [GitHub App setup](GITHUB-APP-SETUP.md) and the [runner-group procedure](LIVE-PILOT.md#2-create-the-organization-runner-group). Automated target-host bootstrap remains [planned](https://github.com/RandomDevelopment/ci-fleet/issues/27). | | Run an isolated first-job proof | Prototype only | GitHub Actions and one isolated worker host | Yes, transiently | Follow the [live pilot](LIVE-PILOT.md). Stop if the matching job queue is not proven empty. | -| Install a persistent test/staging environment host | Planned | Separate test host | N/A | No supported command exists. Stop at [issue #23](https://github.com/RandomDevelopment/ci-fleet/issues/23); do not adapt the worker installer. | +| Install a persistent test/staging environment host | In review | Separate test host | Yes; isolated-host proof remains | Follow [test-environment host](TESTER-HOST.md) and draft issue [#23](https://github.com/RandomDevelopment/ci-fleet/issues/23). Do not install until its prepared-host acceptance evidence is complete. | | Install a deployment host | In review | Separate deployment host | N/A on the default branch | [Issue #22](https://github.com/RandomDevelopment/ci-fleet/issues/22) and [PR #69](https://github.com/RandomDevelopment/ci-fleet/pull/69) track the installer and its required real-host evidence. Do not install it from the default branch yet. | | Deploy to production | Not production-ready | Separate production boundary | N/A | Stop. The project status and production evidence do not authorize this workflow. | | Publish or update from the standalone configuration template | Planned release path | Management workstation and a private configuration repository | Yes | Use the template vendored in the exact reviewed engine commit today. A standalone immutable release/compatibility signal remains [open](https://github.com/RandomDevelopment/ci-fleet-config-template/issues/12). | diff --git a/docs/README.md b/docs/README.md index 90b0499..2d54991 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,6 +15,7 @@ New operator? Use [Operator workflows](OPERATOR-WORKFLOWS.md) to select a suppor | Decide whether it fits my infrastructure | [Architecture](ARCHITECTURE.md) and the root [README](../README.md) | | Try the experimental implementation safely | [Live pilot runbook](LIVE-PILOT.md) | | Add another Docker host, VM, computer, or VPS | [Adding a host](ADDING-A-HOST.md) | +| Install an isolated deployed-application test host | [Test-environment host](TESTER-HOST.md) | | Manage controller configuration from Git | [Git-authored controller desired state](DESIRED-STATE.md) | | Create the controller GitHub App identity | [GitHub App setup](GITHUB-APP-SETUP.md) | | Add a private project to the shared runner pool | [Adding a project](ADDING-A-PROJECT.md) | diff --git a/docs/TESTER-HOST.md b/docs/TESTER-HOST.md new file mode 100644 index 0000000..1ef2664 --- /dev/null +++ b/docs/TESTER-HOST.md @@ -0,0 +1,110 @@ +# Test-environment host + +Status: repository implementation complete; prepared isolated-host acceptance remains required. + +This role runs persistent or expiring deployed application test environments. It is deliberately separate from ephemeral CI workers and production deployers. It cannot register ordinary CI runners or promote production releases. + +## Boundary + +A tester host accepts only: + +- a reviewed `ci-fleet` source commit for the tester service; +- root-owned host configuration and environment declarations; +- application images addressed by an immutable `sha256` digest; +- environment secrets stored below that environment's fixed host-local secret directory. + +It rejects mutable images, public port binds, host bind mounts, external/unscoped Docker resources, privileged containers, added capabilities, host namespaces, global container names, and credentials outside the environment secret boundary. Every service must be read-only, drop all capabilities, and set `no-new-privileges`. Test identity, networks, storage, routes, domains, data, and credentials must have no production authority. Host/network isolation is an external acceptance gate, not something a repository script can prove. + +## Prepare host-local configuration + +The installer supports only `/etc/ci-fleet-tester/tester.env`. Create it and all protected directories as root; never commit them: + +```bash +sudo install -d -m 0700 \ + /etc/ci-fleet-tester \ + /etc/ci-fleet-tester/environments \ + /etc/ci-fleet-tester/definitions \ + /etc/ci-fleet-tester/secrets +sudo install -m 0600 /dev/null /etc/ci-fleet-tester/tester.env +``` + +Example non-secret settings: + +```text +CI_FLEET_TESTER_DEFAULT_TTL_SECONDS=86400 +CI_FLEET_TESTER_MAX_ENVIRONMENTS=20 +CI_FLEET_TESTER_DISK_WARN_PERCENT=80 +CI_FLEET_TESTER_NETWORK_PROBE_HOST=tester-probe.invalid +CI_FLEET_TESTER_HTTPS_PROBE_URL=https://tester-probe.invalid/health +CI_FLEET_TESTER_ISOLATION_ACK=test-only-no-production-authority +``` + +Set both probe values to a test-only host whose DNS resolution and HTTPS HEAD response exercise the intended local proxy path without carrying credentials. The acknowledgement is required but is not proof: an authorized operator must still verify that the prepared host has no production identity or network authority. + +For environment `example-preview`, create root-owned mode-`0600` `/etc/ci-fleet-tester/environments/example-preview.env`: + +```text +CI_FLEET_TESTER_PROJECT=example-project +CI_FLEET_TESTER_OWNER=example-owner +CI_FLEET_TESTER_COMPOSE_FILE=/etc/ci-fleet-tester/definitions/example-preview.yaml +CI_FLEET_TESTER_EXPIRES_AT=REVIEWED_FUTURE_UNIX_TIME +CI_FLEET_TESTER_ROUTE_SERVICE=web +CI_FLEET_TESTER_ROUTE_PORT=18080 +``` + +The Compose file is root-owned mode `0644` and may contain no credential value. Each image must use `registry/path@sha256:REVIEWED_64_HEX_DIGEST`. Exactly one route is published, on loopback only, at the declared port. Compose-generated network and volume names must remain below `ci-fleet-test-_...`; explicit external names are rejected. + +If credentials are required, create `/etc/ci-fleet-tester/secrets/example-preview` as root-owned mode `0700`, put only test-scope regular files there as root-owned mode `0600`, and reference them through Compose `secrets.file`. Symlinks, external secrets, production credentials, environment-variable secret transport, and files outside that exact directory are unsupported. + +## Fresh install or repair + +Use a clean reviewed checkout at the exact commit: + +```bash +ref=$(git rev-parse 'HEAD^{commit}') +sudo ./scripts/install-tester.sh --install \ + --config /etc/ci-fleet-tester/tester.env \ + --ref "$ref" +``` + +The command fails before mutation unless it sees Debian 12 or newer, root, the local default Docker context/socket/root, Compose v2, required generic tools, protected paths, and Docker storage below 80%. It stages an immutable source release, validates it, switches the `current` symlink, installs health and expiration timers, and verifies both configuration and active environments. Repeating the same command is idempotent. + +## Environment lifecycle + +The runtime command is the one interface for create/update, inspect, reset, and removal: + +```bash +sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --converge --environment example-preview +sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --inspect --environment example-preview +sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --reset --environment example-preview +sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --remove --environment example-preview +``` + +`--converge` validates the full resolved Compose model before `up --wait`. `--reset` removes only that exact Compose project and its volumes, then recreates it from the approved definition and digest. `--remove` uses the same scoped `compose down --volumes`; no global Docker prune is used. State reports only environment/project/owner, loopback route, expiry, source revision, timestamps, and health—not Compose environment values or secret content. + +`ci-fleet-tester-health.timer` checks every five minutes. `ci-fleet-tester-cleanup.timer` checks expiration every fifteen minutes. Expired environments are removed through the same scoped path. Disposable environment data is intentionally removed on reset/expiry. Reviewed definitions and fixture sources belong outside runtime state and need ordinary configuration backups; credentials and disposable volumes are never backed up by this service. + +## Upgrade, validation, and rollback + +```bash +ref=$(git rev-parse 'HEAD^{commit}') +sudo ./scripts/install-tester.sh --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$ref" +sudo ./scripts/install-tester.sh --check --config /etc/ci-fleet-tester/tester.env +sudo ./scripts/install-tester.sh --rollback --config /etc/ci-fleet-tester/tester.env +``` + +Upgrade validates the candidate before activation and restores the complete previous release if post-switch checks fail. A successful switch records only the previous complete source revision as last known good. Rollback changes the tester service release; it does not rewrite an application's immutable image digest or reset environment data. + +## Removal + +Remove every environment explicitly, verify no state remains, then uninstall: + +```bash +sudo ./scripts/install-tester.sh --uninstall --config /etc/ci-fleet-tester/tester.env +``` + +Uninstall fails while any managed environment exists. It removes only tester units and immutable service releases. Host configuration, definitions, and secrets remain for explicit operator disposition; the script never guesses which credential may be deleted. + +## External acceptance gate + +Before this draft can merge, an authorized operator must use a prepared isolated Debian Docker host with test-only identity/network/storage/domain boundaries and provide evidence for: fresh install, unchanged second install, immutable application converge, health, scoped reset, expiration cleanup, upgrade, failed-upgrade restoration, rollback, and removal. No production credentials, database, network authority, ordinary CI runner, or live production system may participate. diff --git a/host/systemd/ci-fleet-tester-cleanup.service b/host/systemd/ci-fleet-tester-cleanup.service new file mode 100644 index 0000000..ccd6012 --- /dev/null +++ b/host/systemd/ci-fleet-tester-cleanup.service @@ -0,0 +1,20 @@ +[Unit] +Description=Remove expired ci-fleet test environments +After=docker.service +Requires=docker.service +ConditionPathExists=/etc/ci-fleet-tester/tester.env + +[Service] +Type=oneshot +ExecStart=/opt/ci-fleet-tester/current/scripts/tester-runtime.sh --cleanup +User=root +Group=root +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadWritePaths=/var/lib/ci-fleet-tester /run/lock +RestrictAddressFamilies=AF_UNIX +LockPersonality=yes +MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-cleanup.timer b/host/systemd/ci-fleet-tester-cleanup.timer new file mode 100644 index 0000000..d2053d8 --- /dev/null +++ b/host/systemd/ci-fleet-tester-cleanup.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Expire abandoned ci-fleet test environments + +[Timer] +OnBootSec=5m +OnUnitActiveSec=15m +Persistent=true +RandomizedDelaySec=1m +Unit=ci-fleet-tester-cleanup.service + +[Install] +WantedBy=timers.target diff --git a/host/systemd/ci-fleet-tester-health.service b/host/systemd/ci-fleet-tester-health.service new file mode 100644 index 0000000..92585f1 --- /dev/null +++ b/host/systemd/ci-fleet-tester-health.service @@ -0,0 +1,20 @@ +[Unit] +Description=Validate ci-fleet test environments +After=docker.service +Requires=docker.service +ConditionPathExists=/etc/ci-fleet-tester/tester.env + +[Service] +Type=oneshot +ExecStart=/opt/ci-fleet-tester/current/scripts/tester-runtime.sh --health +User=root +Group=root +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=strict +ReadWritePaths=/var/lib/ci-fleet-tester /run/lock +RestrictAddressFamilies=AF_UNIX +LockPersonality=yes +MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-health.timer b/host/systemd/ci-fleet-tester-health.timer new file mode 100644 index 0000000..e9ec1eb --- /dev/null +++ b/host/systemd/ci-fleet-tester-health.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Check ci-fleet test environments every five minutes + +[Timer] +OnBootSec=2m +OnUnitActiveSec=5m +Persistent=true +RandomizedDelaySec=30s +Unit=ci-fleet-tester-health.service + +[Install] +WantedBy=timers.target diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh new file mode 100755 index 0000000..e3d47df --- /dev/null +++ b/scripts/fixtures/fake-tester-docker.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" +if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi +if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi +if [[ $1 == ps ]]; then printf 'fixture-container-id\n'; exit 0; fi +if [[ $1 == inspect ]]; then printf '1024\n'; exit 0; fi +if [[ $1 == volume && $2 == ls ]]; then printf 'fixture-volume\n'; exit 0; fi +if [[ $1 == volume && $2 == inspect ]]; then printf '%s\n' "${FAKE_TESTER_VOLUME_ROOT:?}"; exit 0; fi +if [[ $1 == compose && $2 == version ]]; then printf 'Docker Compose version v2.fixture\n'; exit 0; fi +if [[ $1 != compose ]]; then exit 2; fi +shift +project= +while (($#)); do + case $1 in + -p) project=$2; shift 2 ;; + -f) shift 2 ;; + config|up|down|ps) operation=$1; shift; break ;; + *) shift ;; + esac +done +case ${operation:-} in + config) + digest=$(printf 'a%.0s' {1..64}) + privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}' + case ${FAKE_TESTER_POLICY:-valid} in + mutable) image=registry.example/example/app:latest ;; + privileged) privileged=true ;; + bind) volume='{"type":"bind","source":"/","target":"/host"}' ;; + broad-port) host_ip=0.0.0.0 ;; + external-network) network_name=shared ;; + valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; + esac + volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} + printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["no-new-privileges:true"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]}},"networks":{"default":{"name":"%s"}},"volumes":{"data":{"name":"%s_data"}},"secrets":%s}\n' \ + "$image" "$privileged" "$read_only" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$network_name" "$project" "$secrets" + ;; + up|down) ;; + ps) [[ ${FAKE_TESTER_UNHEALTHY:-0} == 1 ]] || printf 'fixture-container-id\n' ;; + *) exit 2 ;; +esac diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh new file mode 100755 index 0000000..7c6c1f1 --- /dev/null +++ b/scripts/install-tester.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +report() { printf '%s\n' "$*"; } +usage() { printf 'Usage: install-tester.sh {--check|--install|--upgrade|--rollback|--uninstall|--reset} [--ref COMMIT] [--config /etc/ci-fleet-tester/tester.env] [--environment ID]\n'; } + +action=; ref=; config=/etc/ci-fleet-tester/tester.env; environment= +while (($#)); do + case $1 in + --check|--install|--upgrade|--rollback|--uninstall|--reset) [[ -z $action ]] || die 'choose one action'; action=$1; shift ;; + --ref) (($# >= 2)) || die '--ref requires a value'; ref=$2; shift 2 ;; + --config) (($# >= 2)) || die '--config requires a value'; config=$2; shift 2 ;; + --environment) (($# >= 2)) || die '--environment requires a value'; environment=$2; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) usage; die "unknown argument: $1" ;; + esac +done +[[ -n $action ]] || { usage; exit 2; } +[[ $config == /etc/ci-fleet-tester/tester.env ]] || die 'only the fixed tester configuration path is supported' +case $action in + --install|--upgrade) [[ $ref =~ ^[0-9a-f]{40}$ ]] || die '--ref must be an immutable 40-character commit'; [[ -z $environment ]] || die '--environment is not valid for this action' ;; + --reset) [[ $environment =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die '--reset requires --environment'; [[ -z $ref ]] || die '--ref is not valid for reset' ;; + *) [[ -z $ref && -z $environment ]] || die '--ref/--environment is not valid for this action' ;; +esac + +root_prefix=${CI_FLEET_ROOT_PREFIX:-} +[[ -z $root_prefix || ${CI_FLEET_TESTING:-0} == 1 ]] || die 'CI_FLEET_ROOT_PREFIX is test-only' +root_path() { printf '%s%s' "$root_prefix" "$1"; } +expected_uid=0 +[[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) +if [[ ${CI_FLEET_TESTING:-0} != 1 && ${EUID:-$(id -u)} -ne 0 ]]; then die 'run installer as root'; fi +for command in awk bash chmod cmp curl date df dirname docker du find flock getent git grep install ln mktemp mv python3 readlink rm shellcheck stat systemctl tar wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done + +repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")/.." rev-parse --show-toplevel 2>/dev/null) || die 'installer must run from a Git checkout' +opt_dir=$(root_path /opt/ci-fleet-tester) +release_dir=$opt_dir/releases +current_link=$opt_dir/current +state_root=$(root_path /var/lib/ci-fleet-tester) +lkg_file=$state_root/last-known-good +systemd_dir=$(root_path /etc/systemd/system) +config_root=$(root_path /etc/ci-fleet-tester) +environment_dir=$config_root/environments +definition_dir=$config_root/definitions +secret_root=$config_root/secrets +runtime_state=$state_root/environments +docker_root=$(root_path /var/lib/docker) +docker_socket=$(root_path /var/run/docker.sock) +units=(ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer) +timers=(ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer) + +secure_file() { [[ -f $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected file is unsafe: $1"; } +secure_dir() { [[ -d $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected directory is unsafe: $1"; } + +host_preflight() { + local os_release docker_context actual_root used + os_release=$(root_path /etc/os-release) + [[ -f $os_release ]] || die 'supported Debian os-release is missing' + # shellcheck disable=SC1090 + . "$os_release" + [[ ${ID:-} == debian && ${VERSION_ID:-} =~ ^[0-9]+$ && ${VERSION_ID%%.*} -ge 12 ]] || die 'tester hosts require Debian 12 or newer' + docker_context=$(docker context show); [[ $docker_context == default ]] || die 'tester requires the local default Docker context' + [[ -S $docker_socket || ( ${CI_FLEET_TESTING:-0} == 1 && -e $docker_socket ) ]] || die 'local Docker socket is unavailable' + actual_root=$(docker info --format '{{.DockerRootDir}}'); [[ $actual_root == "$docker_root" ]] || die 'Docker root does not match the local managed root' + docker compose version >/dev/null + used=$(df -P "$docker_root" | awk 'NR==2{gsub(/%/,"",$5);print $5}') + [[ $used =~ ^[0-9]+$ && $used -lt 80 ]] || die 'Docker storage is at or above 80%' +} + +ensure_directories() { + local directory + for directory in "$opt_dir" "$release_dir"; do + if [[ -e $directory || -L $directory ]]; then secure_dir "$directory" 755; else install -d -m 0755 "$directory"; fi + done + for directory in "$config_root" "$environment_dir" "$definition_dir" "$secret_root" "$state_root" "$runtime_state"; do + if [[ -e $directory || -L $directory ]]; then secure_dir "$directory" 700; else install -d -m 0700 "$directory"; fi + done + install -d -m 0755 "$systemd_dir" +} + +release_complete() { + local path=$1 expected=$2 unit + [[ -d $path && ! -L $path && -x $path/scripts/tester-runtime.sh && -f $path/.ci-fleet-source-revision ]] || return 1 + [[ $(<"$path/.ci-fleet-source-revision") == "$expected" ]] || return 1 + for unit in "${units[@]}"; do [[ -f $path/host/systemd/$unit ]] || return 1; done +} + +stage_release() { + local commit=$1 target=$release_dir/$1 staging=$release_dir/.staging-$1 + if [[ -e $target ]]; then release_complete "$target" "$commit" || die 'existing tester release is incomplete'; return; fi + rm -rf -- "$staging"; install -d -m 0755 "$staging" + git -C "$repo_root" cat-file -e "$commit^{commit}" 2>/dev/null || die 'requested source commit is unavailable locally' + git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" + printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" + chmod 0755 "$staging/scripts/tester-runtime.sh"; shellcheck "$staging/scripts/tester-runtime.sh"; bash -n "$staging/scripts/tester-runtime.sh" + mv -T "$staging" "$target" +} + +install_units() { + local source=$1 unit + for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit"; done + systemctl daemon-reload + systemctl enable --now "${timers[@]}" >/dev/null +} + +activate_release() { + local commit=$1 target=$release_dir/$1 previous= + release_complete "$target" "$commit" || die 'candidate tester release is incomplete' + [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") + ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" + install_units "$target" + if ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then + if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then + ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" + fi + die 'candidate tester activation failed; previous release restored when available' + fi + [[ ! $previous =~ ^[0-9a-f]{40}$ || $previous == "$commit" ]] || printf '%s\n' "$previous" >"$lkg_file" + report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" +} + +installed_revision() { + [[ -L $current_link ]] || return 1 + local target; target=$(readlink -f "$current_link") + [[ $target == "$release_dir"/* ]] || return 1 + basename "$target" +} + +case $action in + --install|--upgrade) + host_preflight; ensure_directories + secure_file "$(root_path "$config")" 600 + [[ $(git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$ref" ]] || die 'reviewed checkout HEAD does not match --ref' + if ! git -C "$repo_root" diff --quiet || ! git -C "$repo_root" diff --cached --quiet; then die 'reviewed checkout has tracked changes'; fi + current=$(installed_revision || true) + [[ $action != --install || -z $current || $current == "$ref" ]] || die 'tester is already installed at another revision; use --upgrade' + [[ $action != --upgrade || -n $current ]] || die 'tester is not installed; use --install' + stage_release "$ref"; activate_release "$ref" + ;; + --check) + host_preflight; ensure_directories + secure_file "$(root_path "$config")" 600 + current=$(installed_revision) || die 'tester is not installed' + release_complete "$release_dir/$current" "$current" || die 'installed release is incomplete' + for timer in "${timers[@]}"; do + if ! systemctl is-enabled --quiet "$timer" || ! systemctl is-active --quiet "$timer"; then die "timer is inactive: $timer"; fi + done + "$current_link/scripts/tester-runtime.sh" --check + "$current_link/scripts/tester-runtime.sh" --health + report "CHECK_OK source_revision=$current" + ;; + --reset) + current=$(installed_revision) || die 'tester is not installed' + "$current_link/scripts/tester-runtime.sh" --reset --environment "$environment" + ;; + --rollback) + ensure_directories; secure_file "$lkg_file" 600 + target=$(<"$lkg_file"); [[ $target =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good revision is invalid' + current=$(installed_revision || true) + activate_release "$target" + [[ ! $current =~ ^[0-9a-f]{40}$ || $current == "$target" ]] || printf '%s\n' "$current" >"$lkg_file" + report "ROLLBACK_OK source_revision=$target" + ;; + --uninstall) + ensure_directories + if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi + systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true + for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit"; done + systemctl daemon-reload + rm -f -- "$current_link" "$lkg_file"; rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" + report 'UNINSTALL_OK preserved_config=true preserved_definitions=true preserved_secrets=true' + ;; +esac diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh new file mode 100755 index 0000000..029996a --- /dev/null +++ b/scripts/test-install-tester.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +runtime=$repo_root/scripts/tester-runtime.sh +installer=$repo_root/scripts/install-tester.sh +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } +root=$tmp/root +fake_bin=$tmp/bin +mkdir -p "$fake_bin" "$root/etc/ci-fleet-tester/environments" "$root/etc/ci-fleet-tester/definitions" "$root/etc/ci-fleet-tester/secrets" "$root/var/lib/ci-fleet-tester/environments" "$root/var/lib/docker" "$root/var/lib/fake-tester-volume" "$root/var/run" "$root/etc/systemd/system" +chmod 700 "$root/etc/ci-fleet-tester" "$root/etc/ci-fleet-tester/environments" "$root/etc/ci-fleet-tester/definitions" "$root/etc/ci-fleet-tester/secrets" "$root/var/lib/ci-fleet-tester" "$root/var/lib/ci-fleet-tester/environments" +printf 'ID=debian\nVERSION_ID=13\n' >"$root/etc/os-release" +: >"$root/var/run/docker.sock" +printf 'CI_FLEET_TESTER_DEFAULT_TTL_SECONDS=3600\nCI_FLEET_TESTER_MAX_ENVIRONMENTS=3\nCI_FLEET_TESTER_DISK_WARN_PERCENT=80\nCI_FLEET_TESTER_NETWORK_PROBE_HOST=tester-probe.invalid\nCI_FLEET_TESTER_HTTPS_PROBE_URL=https://tester-probe.invalid/health\nCI_FLEET_TESTER_ISOLATION_ACK=test-only-no-production-authority\n' >"$root/etc/ci-fleet-tester/tester.env" +chmod 600 "$root/etc/ci-fleet-tester/tester.env" +cp "$repo_root/scripts/fixtures/fake-tester-docker.sh" "$fake_bin/docker" +chmod 0755 "$fake_bin/docker" +cat >"$fake_bin/df" <<'EOF' +#!/usr/bin/env bash +printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\nfixture 100 20 80 20%% /fixture\n' +EOF +cat >"$fake_bin/systemctl" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" +EOF +printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" +printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/getent" +chmod 0755 "$fake_bin/df" "$fake_bin/systemctl" "$fake_bin/curl" "$fake_bin/getent" +export PATH="$fake_bin:$PATH" CI_FLEET_TESTING=1 CI_FLEET_ROOT_PREFIX=$root +export FAKE_TESTER_DOCKER_ROOT=$root/var/lib/docker FAKE_TESTER_VOLUME_ROOT=$root/var/lib/fake-tester-volume FAKE_TESTER_DOCKER_LOG=$tmp/docker.log FAKE_TESTER_SYSTEMCTL_LOG=$tmp/systemctl.log + +write_environment() { + local id=$1 port=$2 + mkdir -p "$root/etc/ci-fleet-tester/secrets/$id"; chmod 700 "$root/etc/ci-fleet-tester/secrets/$id" + printf 'services: {}\n' >"$root/etc/ci-fleet-tester/definitions/$id.yaml" + chmod 644 "$root/etc/ci-fleet-tester/definitions/$id.yaml" + printf 'CI_FLEET_TESTER_PROJECT=example-project\nCI_FLEET_TESTER_OWNER=example-owner\nCI_FLEET_TESTER_COMPOSE_FILE=%s\nCI_FLEET_TESTER_EXPIRES_AT=%s\nCI_FLEET_TESTER_ROUTE_SERVICE=web\nCI_FLEET_TESTER_ROUTE_PORT=%s\n' \ + "$root/etc/ci-fleet-tester/definitions/$id.yaml" "$(( $(date +%s) + 1800 ))" "$port" >"$root/etc/ci-fleet-tester/environments/$id.env" + chmod 600 "$root/etc/ci-fleet-tester/environments/$id.env" +} + +"$runtime" --check | grep -Fq CHECK_OK || fail 'runtime preflight failed' +if FAKE_TESTER_DOCKER_ROOT=/remote/docker "$runtime" --check >/dev/null 2>&1; then fail 'remote Docker daemon was accepted'; fi +write_environment preview-a 18080 +FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a | grep -Fq CONVERGED || fail 'converge failed' +state=$root/var/lib/ci-fleet-tester/environments/preview-a.state +[[ -f $state && $(stat -c %a "$state") == 600 ]] || fail 'state was not protected' +inspect_output=$(FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --inspect --environment preview-a) +grep -q 'IMAGE_DIGESTS=sha256:[a-f0-9]\{64\}.*STATUS=running DISK_BYTES=[1-9][0-9]*' <<<"$inspect_output" || fail 'inspect did not report health and disk use' +FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null +[[ $(find "$root/var/lib/ci-fleet-tester/environments" -name '*.state' | wc -l) == 1 ]] || fail 'idempotent converge duplicated state' +write_environment preview-b 18080 +if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi +for policy in mutable privileged bind broad-port external-network; do + write_environment "bad-$policy" 18081 + if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi +done +write_environment secret-preview 18082 +secret_file=$root/etc/ci-fleet-tester/secrets/secret-preview/credential +printf 'example-test-scope-value\n' >"$secret_file"; chmod 600 "$secret_file" +FAKE_TESTER_ROUTE_PORT=18082 FAKE_TESTER_POLICY=valid-secret FAKE_TESTER_SECRET_FILE=$secret_file "$runtime" --converge --environment secret-preview >/dev/null +FAKE_TESTER_ROUTE_PORT=18082 "$runtime" --remove --environment secret-preview >/dev/null +outside_secret=$root/etc/ci-fleet-tester/secrets/outside +printf 'example-test-scope-value\n' >"$outside_secret"; chmod 600 "$outside_secret" +write_environment outside-secret 18083 +if FAKE_TESTER_ROUTE_PORT=18083 FAKE_TESTER_POLICY=outside-secret FAKE_TESTER_SECRET_FILE=$outside_secret "$runtime" --converge --environment outside-secret >/dev/null 2>&1; then fail 'out-of-boundary secret was accepted'; fi +FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --reset --environment preview-a >/dev/null +grep -q 'down --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not remove only the scoped Compose project' +if grep -Eq 'system prune|volume prune|network prune' "$tmp/docker.log"; then fail 'global Docker prune was used'; fi +sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$state" +"$runtime" --cleanup >/dev/null +[[ ! -e $state ]] || fail 'expired environment survived cleanup' + +# Commit-backed installer tests run after the implementation commit exists. +ref=$(git -C "$repo_root" rev-parse HEAD) +"$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' +"$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null +check_output=$("$installer" --check --config /etc/ci-fleet-tester/tester.env) +grep -Fq CHECK_OK <<<"$check_output" || fail 'installed check failed' +for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer; do [[ -f $root/etc/systemd/system/$unit ]] || fail "unit missing: $unit"; done + +# A syntactically valid candidate that fails its post-switch check restores the incumbent. +upgrade_repo=$tmp/upgrade-repo +git clone --quiet --shared "$repo_root" "$upgrade_repo" +printf '#!/usr/bin/env bash\nexit 1\n' >"$upgrade_repo/scripts/tester-runtime.sh"; chmod 0755 "$upgrade_repo/scripts/tester-runtime.sh" +git -C "$upgrade_repo" add scripts/tester-runtime.sh +git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: fail tester activation' +bad_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +if "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'failed candidate activation succeeded'; fi +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'failed upgrade did not restore incumbent release' + +# Rollback switches only to a complete recorded release and keeps environments intact. +old=0000000000000000000000000000000000000000 +cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/releases/$old" +printf '%s\n' "$old" >"$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" +printf '%s\n' "$old" >"$root/var/lib/ci-fleet-tester/last-known-good"; chmod 600 "$root/var/lib/ci-fleet-tester/last-known-good" +"$installer" --rollback --config /etc/ci-fleet-tester/tester.env | grep -Fq ROLLBACK_OK || fail 'rollback failed' +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$old" ]] || fail 'rollback selected the wrong release' + +"$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' +[[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' +printf 'TESTER_INSTALLER_TESTS_OK\n' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh new file mode 100755 index 0000000..540b424 --- /dev/null +++ b/scripts/tester-runtime.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +root_prefix=${CI_FLEET_ROOT_PREFIX:-} +[[ -z $root_prefix || ${CI_FLEET_TESTING:-0} == 1 ]] || { printf 'ERROR: CI_FLEET_ROOT_PREFIX is test-only\n' >&2; exit 1; } +root_path() { printf '%s%s' "$root_prefix" "$1"; } +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source_revision=unknown +[[ ! -f $script_dir/../.ci-fleet-source-revision ]] || source_revision=$(<"$script_dir/../.ci-fleet-source-revision") +config_file=$(root_path /etc/ci-fleet-tester/tester.env) +environment_dir=$(root_path /etc/ci-fleet-tester/environments) +definition_dir=$(root_path /etc/ci-fleet-tester/definitions) +secret_root=$(root_path /etc/ci-fleet-tester/secrets) +state_dir=$(root_path /var/lib/ci-fleet-tester/environments) +lock_file=$(root_path /run/lock/ci-fleet-tester.lock) +expected_uid=0 +[[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) + +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +report() { printf '%s\n' "$*"; } +usage() { printf 'Usage: tester-runtime.sh {--check|--converge|--reset|--remove|--inspect|--cleanup|--health} [--environment ID]\n'; } + +action=; environment= +while (($#)); do + case $1 in + --check|--cleanup|--health) [[ -z $action ]] || die 'choose one action'; action=$1; shift ;; + --converge|--reset|--remove|--inspect) [[ -z $action ]] || die 'choose one action'; action=$1; shift ;; + --environment) (($# >= 2)) || die '--environment requires a value'; environment=$2; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) usage; die "unknown argument: $1" ;; + esac +done +[[ -n $action ]] || { usage; exit 2; } +case $action in --converge|--reset|--remove|--inspect) [[ $environment =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die 'environment ID is invalid' ;; *) [[ -z $environment ]] || die '--environment is not valid for this action' ;; esac +for command in awk basename chmod curl date df dirname docker du find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done + +secure_directory() { + local path=$1 mode=$2 + [[ -d $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == "$mode" ]] || die "protected directory is unsafe: $path" +} +secure_file() { + local path=$1 mode=$2 + [[ -f $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == "$mode" ]] || die "protected file is unsafe: $path" +} +load_exact_env() { + local file=$1 allowed=$2 line key value + declare -gA ENV_VALUES=() + while IFS= read -r line || [[ -n $line ]]; do + [[ -z $line || $line == \#* ]] && continue + [[ $line =~ ^([A-Z][A-Z0-9_]*)=(.*)$ ]] || die "invalid configuration line in $file" + key=${BASH_REMATCH[1]}; value=${BASH_REMATCH[2]} + [[ " $allowed " == *" $key "* ]] || die "unsupported configuration key: $key" + [[ -z ${ENV_VALUES[$key]+x} ]] || die "duplicate configuration key: $key" + [[ $value != *$'\n'* && $value != *$'\r'* ]] || die "invalid configuration value: $key" + ENV_VALUES[$key]=$value + done <"$file" +} + +load_global() { + secure_directory "$(dirname "$config_file")" 700 + secure_directory "$environment_dir" 700 + secure_directory "$definition_dir" 700 + secure_directory "$secret_root" 700 + secure_directory "$(dirname "$state_dir")" 700 + secure_directory "$state_dir" 700 + secure_file "$config_file" 600 + load_exact_env "$config_file" 'CI_FLEET_TESTER_DEFAULT_TTL_SECONDS CI_FLEET_TESTER_MAX_ENVIRONMENTS CI_FLEET_TESTER_DISK_WARN_PERCENT CI_FLEET_TESTER_NETWORK_PROBE_HOST CI_FLEET_TESTER_HTTPS_PROBE_URL CI_FLEET_TESTER_ISOLATION_ACK' + default_ttl=${ENV_VALUES[CI_FLEET_TESTER_DEFAULT_TTL_SECONDS]:-86400} + max_environments=${ENV_VALUES[CI_FLEET_TESTER_MAX_ENVIRONMENTS]:-20} + disk_warn=${ENV_VALUES[CI_FLEET_TESTER_DISK_WARN_PERCENT]:-80} + probe_host=${ENV_VALUES[CI_FLEET_TESTER_NETWORK_PROBE_HOST]:-} + probe_url=${ENV_VALUES[CI_FLEET_TESTER_HTTPS_PROBE_URL]:-} + [[ $default_ttl =~ ^[0-9]+$ && $default_ttl -ge 300 && $default_ttl -le 604800 ]] || die 'default TTL must be 300-604800 seconds' + [[ $max_environments =~ ^[0-9]+$ && $max_environments -ge 1 && $max_environments -le 100 ]] || die 'max environments must be 1-100' + [[ $disk_warn =~ ^[0-9]+$ && $disk_warn -ge 50 && $disk_warn -le 95 ]] || die 'disk warning threshold must be 50-95' + [[ $probe_host =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$ && $probe_url =~ ^https://[A-Za-z0-9][A-Za-z0-9.-]{0,252}(:[0-9]+)?/[^[:space:]]*$ ]] || die 'DNS and HTTPS probe targets are required' + [[ ${ENV_VALUES[CI_FLEET_TESTER_ISOLATION_ACK]:-} == test-only-no-production-authority ]] || die 'explicit test-only isolation acknowledgement is required' +} + +project_name() { printf 'ci-fleet-test-%s' "$1"; } +state_path() { printf '%s/%s.state' "$state_dir" "$1"; } +spec_path() { printf '%s/%s.env' "$environment_dir" "$1"; } + +load_spec() { + local id=$1 canonical + spec=$(spec_path "$id") + secure_file "$spec" 600 + load_exact_env "$spec" 'CI_FLEET_TESTER_PROJECT CI_FLEET_TESTER_OWNER CI_FLEET_TESTER_COMPOSE_FILE CI_FLEET_TESTER_EXPIRES_AT CI_FLEET_TESTER_ROUTE_SERVICE CI_FLEET_TESTER_ROUTE_PORT' + project=${ENV_VALUES[CI_FLEET_TESTER_PROJECT]:-} + owner=${ENV_VALUES[CI_FLEET_TESTER_OWNER]:-} + compose_file=${ENV_VALUES[CI_FLEET_TESTER_COMPOSE_FILE]:-} + expires_at=${ENV_VALUES[CI_FLEET_TESTER_EXPIRES_AT]:-} + route_service=${ENV_VALUES[CI_FLEET_TESTER_ROUTE_SERVICE]:-} + route_port=${ENV_VALUES[CI_FLEET_TESTER_ROUTE_PORT]:-} + [[ $project =~ ^[a-z0-9][a-z0-9-]{0,62}$ && $owner =~ ^[A-Za-z0-9][A-Za-z0-9_.@-]{0,127}$ ]] || die 'project or owner is invalid' + [[ $route_service =~ ^[a-z0-9][a-z0-9-]{0,62}$ && $route_port =~ ^[0-9]+$ && $route_port -ge 1024 && $route_port -le 65535 ]] || die 'route service/port is invalid' + if [[ -z $expires_at ]]; then expires_at=$(($(date +%s) + default_ttl)); fi + [[ $expires_at =~ ^[0-9]+$ && $expires_at -gt $(date +%s) && $expires_at -le $(($(date +%s) + 604800)) ]] || die 'expiration must be in the future and at most seven days away' + canonical=$(readlink -f -- "$compose_file") || die 'compose file is unavailable' + [[ $canonical == "$definition_dir"/* && $canonical == "$compose_file" ]] || die 'compose file must be a canonical file below the protected definitions directory' + secure_file "$compose_file" 644 + compose_project=$(project_name "$id") + secret_dir=$secret_root/$id + secure_directory "$secret_dir" 700 +} + +validate_compose() { + local rendered=$1 + docker compose -p "$compose_project" -f "$compose_file" config --format json >"$rendered" || return 1 + chmod 600 "$rendered" + python3 - "$rendered" "$route_service" "$route_port" "$compose_project" "$secret_dir" "$expected_uid" <<'PY' || return 1 +import json,os,re,stat,sys +value=json.load(open(sys.argv[1])); route_service=sys.argv[2]; route_port=int(sys.argv[3]); project=sys.argv[4]; secret_dir=sys.argv[5]; expected_uid=int(sys.argv[6]) +services=value.get('services') +if not isinstance(services,dict) or route_service not in services: raise SystemExit('route service is missing') +image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64}$') +ports=[] +for name,service in services.items(): + if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') + if service.get('privileged') or service.get('network_mode')=='host' or service.get('pid')=='host' or service.get('ipc')=='host': raise SystemExit(f'{name}: host/privileged access is forbidden') + if service.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname'): raise SystemExit(f'{name}: device/capability/global identity is forbidden') + if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') + security=' '.join(service.get('security_opt',[])) + if 'no-new-privileges' not in security: raise SystemExit(f'{name}: no-new-privileges is required') + for mount in service.get('volumes',[]): + if isinstance(mount,str) or mount.get('type') not in ('volume','tmpfs'): raise SystemExit(f'{name}: host bind mounts are forbidden') + for port in service.get('ports',[]): + if not isinstance(port,dict) or str(port.get('host_ip','')) != '127.0.0.1': raise SystemExit(f'{name}: published ports must bind loopback') + ports.append((name,int(port.get('published',0)),int(port.get('target',0)))) +if ports != [(route_service,route_port,ports[0][2] if ports else 0)] or not ports or ports[0][2] < 1: raise SystemExit('exactly one declared loopback route is required') +for section in ('networks','volumes'): + for name,item in value.get(section,{}).items(): + resolved=item.get('name',f'{project}_{name}') + if item.get('external') or not resolved.startswith(f'{project}_'): raise SystemExit(f'{section}.{name}: external/unscoped names are forbidden') +for name,item in value.get('secrets',{}).items(): + path=item.get('file') + if item.get('external') or not isinstance(path,str) or os.path.realpath(path).rsplit('/',1)[0] != secret_dir: raise SystemExit(f'secrets.{name}: secret must be a host-local file in the environment secret directory') + metadata=os.lstat(path) + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != expected_uid or stat.S_IMODE(metadata.st_mode) != 0o600: raise SystemExit(f'secrets.{name}: secret must be owner-controlled mode 0600') +PY + image_digests=$(python3 - "$rendered" <<'PY' +import json,sys +value=json.load(open(sys.argv[1])) +print(','.join(sorted({service['image'].rsplit('@',1)[1] for service in value['services'].values()}))) +PY +) +} + +check_port_unique() { + local file key value other_port + for file in "$state_dir"/*.state; do + [[ -e $file ]] || continue + [[ $file == "$(state_path "$environment")" ]] && continue + other_port=$(awk -F= '$1=="ROUTE_PORT"{print $2}' "$file") + [[ $other_port != "$route_port" ]] || die "loopback route port is already owned by another environment: $route_port" + done +} + +write_state() { + local target tmp + target=$(state_path "$environment"); tmp=$target.new + printf 'ENVIRONMENT=%s\nPROJECT=%s\nOWNER=%s\nCOMPOSE_FILE=%s\nROUTE_SERVICE=%s\nROUTE_PORT=%s\nEXPIRES_AT=%s\nSOURCE_REVISION=%s\nIMAGE_DIGESTS=%s\nUPDATED_AT=%s\n' \ + "$environment" "$project" "$owner" "$compose_file" "$route_service" "$route_port" "$expires_at" "$source_revision" "$image_digests" "$(date +%s)" >"$tmp" + chmod 600 "$tmp"; mv -fT "$tmp" "$target" +} + +converge() { + local rendered count + load_spec "$environment" + check_port_unique + count=$(find "$state_dir" -maxdepth 1 -type f -name '*.state' | wc -l) + [[ -f $(state_path "$environment") || $count -lt $max_environments ]] || die 'maximum environment count reached' + rendered=$(mktemp) + if ! validate_compose "$rendered"; then rm -f "$rendered"; die 'compose policy validation failed'; fi + rm -f "$rendered" + docker compose -p "$compose_project" -f "$compose_file" up -d --remove-orphans --wait + write_state + report "CONVERGED environment=$environment project=$project owner=$owner route=loopback:$route_port expires_at=$expires_at" +} + +remove_environment() { + local target compose id=$1 + target=$(state_path "$id") + if [[ -f $target ]]; then + secure_file "$target" 600 + compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") + [[ $(readlink -f -- "$compose") == "$definition_dir"/* ]] || die 'stored compose path escaped definitions directory' + docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans + rm -f -- "$target" + fi + report "REMOVED environment=$id" +} + +inspect_environment() { + local target=$1 id compose status resource value bytes=0 mount + id=$(basename "$target" .state); secure_file "$target" 600 + while IFS='=' read -r key value; do + case $key in ENVIRONMENT|PROJECT|OWNER|ROUTE_PORT|EXPIRES_AT|SOURCE_REVISION|IMAGE_DIGESTS|UPDATED_AT) printf '%s=%s ' "$key" "$value" ;; esac + done <"$target" + compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") + if docker compose -p "$(project_name "$id")" -f "$compose" ps --status running -q | grep -q .; then status=running; else status=unhealthy; fi + while IFS= read -r resource; do + [[ -n $resource ]] || continue + value=$(docker inspect --size --format '{{.SizeRw}}' "$resource"); [[ $value =~ ^[0-9]+$ ]] || die 'container disk size is invalid'; bytes=$((bytes + value)) + done < <(docker ps -aq --filter "label=com.docker.compose.project=$(project_name "$id")") + while IFS= read -r resource; do + [[ -n $resource ]] || continue + mount=$(docker volume inspect --format '{{.Mountpoint}}' "$resource"); value=$(du -sb "$mount" | awk '{print $1}'); [[ $value =~ ^[0-9]+$ ]] || die 'volume disk size is invalid'; bytes=$((bytes + value)) + done < <(docker volume ls -q --filter "label=com.docker.compose.project=$(project_name "$id")") + printf 'STATUS=%s DISK_BYTES=%s\n' "$status" "$bytes" +} + +load_global +docker_socket=$(root_path /var/run/docker.sock) +if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then + [[ -f $docker_socket && ! -L $docker_socket ]] || die 'local Docker socket is unavailable' +else + [[ -S $docker_socket && ! -L $docker_socket && $(stat -c %u "$docker_socket") == 0 ]] || die 'local root-owned Docker socket is unavailable' +fi +unset DOCKER_CONTEXT +export DOCKER_HOST="unix://$docker_socket" +[[ $(docker info --format '{{.DockerRootDir}}') == "$(root_path /var/lib/docker)" ]] || die 'Docker daemon root is not the expected local path' +install -d -m 0755 "$(dirname "$lock_file")" +exec 9>"$lock_file" +flock -x 9 +case $action in + --check) + docker info --format '{{.DockerRootDir}}' >/dev/null + docker compose version >/dev/null + getent ahosts "$probe_host" >/dev/null || die 'test-host DNS probe failed' + curl --fail --silent --show-error --head --max-time 10 --output /dev/null "$probe_url" || die 'test-host HTTPS/proxy probe failed' + used=$(df -P "$(root_path /var/lib/docker)" | awk 'NR==2{gsub(/%/,"",$5);print $5}') + [[ $used =~ ^[0-9]+$ && $used -lt $disk_warn ]] || die 'Docker storage exceeds configured warning threshold' + report "CHECK_OK max_environments=$max_environments disk_used_percent=$used" + ;; + --converge) converge ;; + --reset) remove_environment "$environment"; converge ;; + --remove) remove_environment "$environment" ;; + --inspect) [[ -f $(state_path "$environment") ]] || die 'environment is not installed'; inspect_environment "$(state_path "$environment")" ;; + --cleanup) + now=$(date +%s) + for target in "$state_dir"/*.state; do + [[ -e $target ]] || continue + secure_file "$target" 600 + expires=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$target") + [[ $expires =~ ^[0-9]+$ ]] || die "invalid expiration in $target" + if ((expires <= now)); then remove_environment "$(basename "$target" .state)"; fi + done + report 'CLEANUP_OK' + ;; + --health) + failed=0 + for target in "$state_dir"/*.state; do [[ -e $target ]] || continue; inspect_environment "$target" | grep -q 'STATUS=running' || failed=1; done + ((failed == 0)) || die 'one or more test environments are unhealthy' + report 'HEALTH_OK' + ;; +esac diff --git a/scripts/validate.sh b/scripts/validate.sh index 9660aaa..60afad0 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -27,6 +27,7 @@ python3 scripts/desired_state.py validate-engine-capabilities --manifest engine- python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group fast >/dev/null python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null scripts/test-capacity-preflight.sh +scripts/test-install-tester.sh scripts/test-install-worker-controller.sh scripts/test-install-status-receiver.sh From e2bf688a3a4fd777ccbb2a22d49bb7b5d257ad4f Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 00:14:57 -0500 Subject: [PATCH 02/16] fix: make tester lifecycle recoverable and scoped --- scripts/fixtures/fake-tester-docker.sh | 20 ++++++--- scripts/install-tester.sh | 43 +++++++++++++------ scripts/test-install-tester.sh | 44 ++++++++++++++++++- scripts/tester-runtime.sh | 58 ++++++++++++++++++-------- 4 files changed, 129 insertions(+), 36 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index e3d47df..2b06e3e 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -4,7 +4,10 @@ printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi if [[ $1 == ps ]]; then printf 'fixture-container-id\n'; exit 0; fi -if [[ $1 == inspect ]]; then printf '1024\n'; exit 0; fi +if [[ $1 == inspect ]]; then + if [[ " $* " == *' --size '* ]]; then printf '1024\n'; else printf '%s\n' "${FAKE_TESTER_CONTAINER_STATE:-running healthy}"; fi + exit 0 +fi if [[ $1 == volume && $2 == ls ]]; then printf 'fixture-volume\n'; exit 0; fi if [[ $1 == volume && $2 == inspect ]]; then printf '%s\n' "${FAKE_TESTER_VOLUME_ROOT:?}"; exit 0; fi if [[ $1 == compose && $2 == version ]]; then printf 'Docker Compose version v2.fixture\n'; exit 0; fi @@ -22,20 +25,27 @@ done case ${operation:-} in config) digest=$(printf 'a%.0s' {1..64}) - privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}' + privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=; top_extra=; volume_extra=; security='no-new-privileges:true' case ${FAKE_TESTER_POLICY:-valid} in mutable) image=registry.example/example/app:latest ;; privileged) privileged=true ;; bind) volume='{"type":"bind","source":"/","target":"/host"}' ;; broad-port) host_ip=0.0.0.0 ;; external-network) network_name=shared ;; + environment) service_extra=',"environment":{"TOKEN":"example"}' ;; + configs) service_extra=',"configs":[{"source":"credential"}]'; top_extra=',"configs":{"credential":{"file":"/tmp/example"}}' ;; + use-api-socket) service_extra=',"use_api_socket":true' ;; + namespace-share) service_extra=',"network_mode":"service:other"' ;; + false-nnp) security='no-new-privileges:false' ;; + custom-volume) volume_extra=',"driver":"local"' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} - printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["no-new-privileges:true"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]}},"networks":{"default":{"name":"%s"}},"volumes":{"data":{"name":"%s_data"}},"secrets":%s}\n' \ - "$image" "$privileged" "$read_only" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$network_name" "$project" "$secrets" + printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["%s"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]%s}},"networks":{"default":{"name":"%s"}},"volumes":{"data":{"name":"%s_data"%s}},"secrets":%s%s}\n' \ + "$image" "$privileged" "$read_only" "$security" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$service_extra" "$network_name" "$project" "$volume_extra" "$secrets" "$top_extra" ;; - up|down) ;; + up) [[ ${FAKE_TESTER_UP_FAIL:-0} != 1 ]] ;; + down) [[ ${FAKE_TESTER_DOWN_FAIL:-0} != 1 ]] ;; ps) [[ ${FAKE_TESTER_UNHEALTHY:-0} == 1 ]] || printf 'fixture-container-id\n' ;; *) exit 2 ;; esac diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 7c6c1f1..8f5569b 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -31,7 +31,7 @@ root_path() { printf '%s%s' "$root_prefix" "$1"; } expected_uid=0 [[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) if [[ ${CI_FLEET_TESTING:-0} != 1 && ${EUID:-$(id -u)} -ne 0 ]]; then die 'run installer as root'; fi -for command in awk bash chmod cmp curl date df dirname docker du find flock getent git grep install ln mktemp mv python3 readlink rm shellcheck stat systemctl tar wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done +for command in awk bash chmod cmp curl date df dirname docker du find flock getent git grep install ln mktemp mv python3 readlink rm sha256sum shellcheck stat systemctl tar wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")/.." rev-parse --show-toplevel 2>/dev/null) || die 'installer must run from a Git checkout' opt_dir=$(root_path /opt/ci-fleet-tester) @@ -60,6 +60,9 @@ host_preflight() { # shellcheck disable=SC1090 . "$os_release" [[ ${ID:-} == debian && ${VERSION_ID:-} =~ ^[0-9]+$ && ${VERSION_ID%%.*} -ge 12 ]] || die 'tester hosts require Debian 12 or newer' + [[ -z ${DOCKER_HOST:-} && -z ${DOCKER_CONTEXT:-} ]] || die 'Docker environment selectors are forbidden' + unset DOCKER_CONTEXT + export DOCKER_HOST="unix://$docker_socket" docker_context=$(docker context show); [[ $docker_context == default ]] || die 'tester requires the local default Docker context' [[ -S $docker_socket || ( ${CI_FLEET_TESTING:-0} == 1 && -e $docker_socket ) ]] || die 'local Docker socket is unavailable' actual_root=$(docker info --format '{{.DockerRootDir}}'); [[ $actual_root == "$docker_root" ]] || die 'Docker root does not match the local managed root' @@ -81,9 +84,10 @@ ensure_directories() { release_complete() { local path=$1 expected=$2 unit - [[ -d $path && ! -L $path && -x $path/scripts/tester-runtime.sh && -f $path/.ci-fleet-source-revision ]] || return 1 + [[ -d $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == 555 && -x $path/scripts/tester-runtime.sh && -f $path/.ci-fleet-source-revision ]] || return 1 [[ $(<"$path/.ci-fleet-source-revision") == "$expected" ]] || return 1 for unit in "${units[@]}"; do [[ -f $path/host/systemd/$unit ]] || return 1; done + (cd "$path" && sha256sum --status -c .ci-fleet-release.sha256) || return 1 } stage_release() { @@ -94,14 +98,29 @@ stage_release() { git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" chmod 0755 "$staging/scripts/tester-runtime.sh"; shellcheck "$staging/scripts/tester-runtime.sh"; bash -n "$staging/scripts/tester-runtime.sh" + (cd "$staging" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) + chmod 0444 "$staging/.ci-fleet-source-revision" "$staging/.ci-fleet-release.sha256" "$staging"/host/systemd/* + chmod 0555 "$staging" "$staging/scripts" "$staging/host" "$staging/host/systemd" "$staging/scripts/tester-runtime.sh" mv -T "$staging" "$target" } install_units() { local source=$1 unit - for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit"; done - systemctl daemon-reload - systemctl enable --now "${timers[@]}" >/dev/null + for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit" || return 1; done + systemctl daemon-reload || return 1 + systemctl enable --now "${timers[@]}" >/dev/null || return 1 +} + +remove_units() { + systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true + local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit"; done + systemctl daemon-reload || true +} + +write_lkg() { + printf '%s\n' "$1" >"$lkg_file.new" + chmod 0600 "$lkg_file.new" + mv -fT "$lkg_file.new" "$lkg_file" } activate_release() { @@ -109,14 +128,16 @@ activate_release() { release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - install_units "$target" - if ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then + if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" + else + rm -f -- "$current_link" + remove_units fi die 'candidate tester activation failed; previous release restored when available' fi - [[ ! $previous =~ ^[0-9a-f]{40}$ || $previous == "$commit" ]] || printf '%s\n' "$previous" >"$lkg_file" + [[ ! $previous =~ ^[0-9a-f]{40}$ || $previous == "$commit" ]] || write_lkg "$previous" report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" } @@ -159,15 +180,13 @@ case $action in target=$(<"$lkg_file"); [[ $target =~ ^[0-9a-f]{40}$ ]] || die 'last-known-good revision is invalid' current=$(installed_revision || true) activate_release "$target" - [[ ! $current =~ ^[0-9a-f]{40}$ || $current == "$target" ]] || printf '%s\n' "$current" >"$lkg_file" + [[ ! $current =~ ^[0-9a-f]{40}$ || $current == "$target" ]] || write_lkg "$current" report "ROLLBACK_OK source_revision=$target" ;; --uninstall) ensure_directories if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi - systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true - for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit"; done - systemctl daemon-reload + remove_units rm -f -- "$current_link" "$lkg_file"; rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" report 'UNINSTALL_OK preserved_config=true preserved_definitions=true preserved_secrets=true' ;; diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 029996a..4aaa05d 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -23,6 +23,7 @@ EOF cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" +[[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/getent" @@ -50,12 +51,20 @@ inspect_output=$(FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --inspect --environment grep -q 'IMAGE_DIGESTS=sha256:[a-f0-9]\{64\}.*STATUS=running DISK_BYTES=[1-9][0-9]*' <<<"$inspect_output" || fail 'inspect did not report health and disk use' FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null [[ $(find "$root/var/lib/ci-fleet-tester/environments" -name '*.state' | wc -l) == 1 ]] || fail 'idempotent converge duplicated state' +original_expiry=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") +grep -v '^CI_FLEET_TESTER_EXPIRES_AT=' "$root/etc/ci-fleet-tester/environments/preview-a.env" >"$tmp/spec" +mv "$tmp/spec" "$root/etc/ci-fleet-tester/environments/preview-a.env"; chmod 600 "$root/etc/ci-fleet-tester/environments/preview-a.env" +FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null +[[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=mutable "$runtime" --reset --environment preview-a >/dev/null 2>&1; then fail 'reset accepted invalid replacement'; fi +[[ -f $state ]] || fail 'reset deleted the incumbent before validation' +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='exited unhealthy' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a stopped managed service'; fi write_environment secret-preview 18082 secret_file=$root/etc/ci-fleet-tester/secrets/secret-preview/credential printf 'example-test-scope-value\n' >"$secret_file"; chmod 600 "$secret_file" @@ -65,6 +74,14 @@ outside_secret=$root/etc/ci-fleet-tester/secrets/outside printf 'example-test-scope-value\n' >"$outside_secret"; chmod 600 "$outside_secret" write_environment outside-secret 18083 if FAKE_TESTER_ROUTE_PORT=18083 FAKE_TESTER_POLICY=outside-secret FAKE_TESTER_SECRET_FILE=$outside_secret "$runtime" --converge --environment outside-secret >/dev/null 2>&1; then fail 'out-of-boundary secret was accepted'; fi +write_environment partial-up 18084 +if FAKE_TESTER_ROUTE_PORT=18084 FAKE_TESTER_UP_FAIL=1 "$runtime" --converge --environment partial-up >/dev/null 2>&1; then fail 'partial activation succeeded'; fi +[[ -f $root/var/lib/ci-fleet-tester/environments/partial-up.state && -f $root/var/lib/ci-fleet-tester/environments/partial-up.compose.json ]] || fail 'partial activation was not tracked for cleanup' +FAKE_TESTER_ROUTE_PORT=18084 "$runtime" --remove --environment partial-up >/dev/null +write_environment immutable-remove 18085 +FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --converge --environment immutable-remove >/dev/null +rm "$root/etc/ci-fleet-tester/definitions/immutable-remove.yaml" +FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --remove --environment immutable-remove >/dev/null FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --reset --environment preview-a >/dev/null grep -q 'down --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not remove only the scoped Compose project' if grep -Eq 'system prune|volume prune|network prune' "$tmp/docker.log"; then fail 'global Docker prune was used'; fi @@ -74,12 +91,18 @@ sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$state" # Commit-backed installer tests run after the implementation commit exists. ref=$(git -C "$repo_root" rev-parse HEAD) +if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null check_output=$("$installer" --check --config /etc/ci-fleet-tester/tester.env) grep -Fq CHECK_OK <<<"$check_output" || fail 'installed check failed' for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer; do [[ -f $root/etc/systemd/system/$unit ]] || fail "unit missing: $unit"; done +release=$root/opt/ci-fleet-tester/releases/$ref +chmod u+w "$release/scripts/tester-runtime.sh"; printf '# tamper\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" +if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'tampered installed release passed check'; fi +git -C "$repo_root" show "$ref:scripts/tester-runtime.sh" >"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" +"$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'restored release failed check' # A syntactically valid candidate that fails its post-switch check restores the incumbent. upgrade_repo=$tmp/upgrade-repo @@ -91,13 +114,32 @@ bad_ref=$(git -C "$upgrade_repo" rev-parse HEAD) if "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'failed candidate activation succeeded'; fi [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'failed upgrade did not restore incumbent release' +# Unit activation failures restore the incumbent symlink and units. +git -C "$upgrade_repo" show "$ref:scripts/tester-runtime.sh" >"$upgrade_repo/scripts/tester-runtime.sh"; chmod 0755 "$upgrade_repo/scripts/tester-runtime.sh" +git -C "$upgrade_repo" add scripts/tester-runtime.sh +git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: valid tester candidate' +unit_fail_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +unit_hash_before=$(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") +if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'unit activation failure succeeded'; fi +[[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" && $(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") == "$unit_hash_before" ]] || fail 'unit activation failure did not restore incumbent release and units' + # Rollback switches only to a complete recorded release and keeps environments intact. old=0000000000000000000000000000000000000000 cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/releases/$old" +chmod 0755 "$root/opt/ci-fleet-tester/releases/$old" +chmod 0644 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" printf '%s\n' "$old" >"$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" +(cd "$root/opt/ci-fleet-tester/releases/$old" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) +chmod 0444 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" +chmod 0555 "$root/opt/ci-fleet-tester/releases/$old" printf '%s\n' "$old" >"$root/var/lib/ci-fleet-tester/last-known-good"; chmod 600 "$root/var/lib/ci-fleet-tester/last-known-good" +write_environment rollback-env 18086 +FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --converge --environment rollback-env >/dev/null +rollback_state_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") "$installer" --rollback --config /etc/ci-fleet-tester/tester.env | grep -Fq ROLLBACK_OK || fail 'rollback failed' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$old" ]] || fail 'rollback selected the wrong release' +[[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") == "$rollback_state_hash" ]] || fail 'rollback changed active environment state' +FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --remove --environment rollback-env >/dev/null "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' [[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 540b424..6747fce 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -80,6 +80,7 @@ load_global() { project_name() { printf 'ci-fleet-test-%s' "$1"; } state_path() { printf '%s/%s.state' "$state_dir" "$1"; } +deployed_compose_path() { printf '%s/%s.compose.json' "$state_dir" "$1"; } spec_path() { printf '%s/%s.env' "$environment_dir" "$1"; } load_spec() { @@ -95,6 +96,7 @@ load_spec() { route_port=${ENV_VALUES[CI_FLEET_TESTER_ROUTE_PORT]:-} [[ $project =~ ^[a-z0-9][a-z0-9-]{0,62}$ && $owner =~ ^[A-Za-z0-9][A-Za-z0-9_.@-]{0,127}$ ]] || die 'project or owner is invalid' [[ $route_service =~ ^[a-z0-9][a-z0-9-]{0,62}$ && $route_port =~ ^[0-9]+$ && $route_port -ge 1024 && $route_port -le 65535 ]] || die 'route service/port is invalid' + if [[ -z $expires_at && -f $(state_path "$id") ]]; then expires_at=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$(state_path "$id")"); fi if [[ -z $expires_at ]]; then expires_at=$(($(date +%s) + default_ttl)); fi [[ $expires_at =~ ^[0-9]+$ && $expires_at -gt $(date +%s) && $expires_at -le $(($(date +%s) + 604800)) ]] || die 'expiration must be in the future and at most seven days away' canonical=$(readlink -f -- "$compose_file") || die 'compose file is unavailable' @@ -118,21 +120,23 @@ image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64} ports=[] for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') - if service.get('privileged') or service.get('network_mode')=='host' or service.get('pid')=='host' or service.get('ipc')=='host': raise SystemExit(f'{name}: host/privileged access is forbidden') - if service.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname'): raise SystemExit(f'{name}: device/capability/global identity is forbidden') + if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc'): raise SystemExit(f'{name}: external namespace/privileged access is forbidden') + if service.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket'): raise SystemExit(f'{name}: device/capability/global identity is forbidden') + if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') - security=' '.join(service.get('security_opt',[])) - if 'no-new-privileges' not in security: raise SystemExit(f'{name}: no-new-privileges is required') + if not any(re.fullmatch(r'no-new-privileges[:=]true', option) for option in service.get('security_opt',[])): raise SystemExit(f'{name}: no-new-privileges=true is required') for mount in service.get('volumes',[]): if isinstance(mount,str) or mount.get('type') not in ('volume','tmpfs'): raise SystemExit(f'{name}: host bind mounts are forbidden') for port in service.get('ports',[]): if not isinstance(port,dict) or str(port.get('host_ip','')) != '127.0.0.1': raise SystemExit(f'{name}: published ports must bind loopback') ports.append((name,int(port.get('published',0)),int(port.get('target',0)))) if ports != [(route_service,route_port,ports[0][2] if ports else 0)] or not ports or ports[0][2] < 1: raise SystemExit('exactly one declared loopback route is required') +if value.get('configs'): raise SystemExit('top-level configs are forbidden') for section in ('networks','volumes'): for name,item in value.get(section,{}).items(): resolved=item.get('name',f'{project}_{name}') if item.get('external') or not resolved.startswith(f'{project}_'): raise SystemExit(f'{section}.{name}: external/unscoped names are forbidden') + if section == 'volumes' and (item.get('driver') or item.get('driver_opts')): raise SystemExit(f'{section}.{name}: custom volume drivers are forbidden') for name,item in value.get('secrets',{}).items(): path=item.get('file') if item.get('external') or not isinstance(path,str) or os.path.realpath(path).rsplit('/',1)[0] != secret_dir: raise SystemExit(f'secrets.{name}: secret must be a host-local file in the environment secret directory') @@ -161,45 +165,62 @@ write_state() { local target tmp target=$(state_path "$environment"); tmp=$target.new printf 'ENVIRONMENT=%s\nPROJECT=%s\nOWNER=%s\nCOMPOSE_FILE=%s\nROUTE_SERVICE=%s\nROUTE_PORT=%s\nEXPIRES_AT=%s\nSOURCE_REVISION=%s\nIMAGE_DIGESTS=%s\nUPDATED_AT=%s\n' \ - "$environment" "$project" "$owner" "$compose_file" "$route_service" "$route_port" "$expires_at" "$source_revision" "$image_digests" "$(date +%s)" >"$tmp" + "$environment" "$project" "$owner" "$(deployed_compose_path "$environment")" "$route_service" "$route_port" "$expires_at" "$source_revision" "$image_digests" "$(date +%s)" >"$tmp" chmod 600 "$tmp"; mv -fT "$tmp" "$target" } -converge() { - local rendered count +prepare_converge() { + local count load_spec "$environment" check_port_unique count=$(find "$state_dir" -maxdepth 1 -type f -name '*.state' | wc -l) [[ -f $(state_path "$environment") || $count -lt $max_environments ]] || die 'maximum environment count reached' - rendered=$(mktemp) - if ! validate_compose "$rendered"; then rm -f "$rendered"; die 'compose policy validation failed'; fi - rm -f "$rendered" - docker compose -p "$compose_project" -f "$compose_file" up -d --remove-orphans --wait + prepared_rendered=$(mktemp) + if ! validate_compose "$prepared_rendered"; then rm -f "$prepared_rendered"; die 'compose policy validation failed'; fi +} + +apply_converge() { + install -m 0600 "$prepared_rendered" "$(deployed_compose_path "$environment")" write_state + if ! docker compose -p "$compose_project" -f "$(deployed_compose_path "$environment")" up -d --remove-orphans --wait; then + rm -f "$prepared_rendered" + die 'environment activation failed; tracked state retained for cleanup' + fi + rm -f "$prepared_rendered" report "CONVERGED environment=$environment project=$project owner=$owner route=loopback:$route_port expires_at=$expires_at" } +converge() { prepare_converge; apply_converge; } + remove_environment() { local target compose id=$1 target=$(state_path "$id") if [[ -f $target ]]; then secure_file "$target" 600 compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") - [[ $(readlink -f -- "$compose") == "$definition_dir"/* ]] || die 'stored compose path escaped definitions directory' + [[ $compose == "$(deployed_compose_path "$id")" ]] || die 'stored compose path is unexpected' + secure_file "$compose" 600 docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans - rm -f -- "$target" + rm -f -- "$target" "$(deployed_compose_path "$id")" fi report "REMOVED environment=$id" } inspect_environment() { - local target=$1 id compose status resource value bytes=0 mount + local target=$1 id compose status=running resource value bytes=0 mount expected running_state + local -a containers=() id=$(basename "$target" .state); secure_file "$target" 600 while IFS='=' read -r key value; do case $key in ENVIRONMENT|PROJECT|OWNER|ROUTE_PORT|EXPIRES_AT|SOURCE_REVISION|IMAGE_DIGESTS|UPDATED_AT) printf '%s=%s ' "$key" "$value" ;; esac done <"$target" compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") - if docker compose -p "$(project_name "$id")" -f "$compose" ps --status running -q | grep -q .; then status=running; else status=unhealthy; fi + expected=$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))["services"]))' "$compose") + mapfile -t containers < <(docker compose -p "$(project_name "$id")" -f "$compose" ps -q) + [[ ${#containers[@]} == "$expected" ]] || status=unhealthy + for resource in "${containers[@]}"; do + running_state=$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$resource") + [[ $running_state == 'running healthy' || $running_state == 'running none' ]] || status=unhealthy + done while IFS= read -r resource; do [[ -n $resource ]] || continue value=$(docker inspect --size --format '{{.SizeRw}}' "$resource"); [[ $value =~ ^[0-9]+$ ]] || die 'container disk size is invalid'; bytes=$((bytes + value)) @@ -235,18 +256,19 @@ case $action in report "CHECK_OK max_environments=$max_environments disk_used_percent=$used" ;; --converge) converge ;; - --reset) remove_environment "$environment"; converge ;; + --reset) prepare_converge; remove_environment "$environment"; apply_converge ;; --remove) remove_environment "$environment" ;; --inspect) [[ -f $(state_path "$environment") ]] || die 'environment is not installed'; inspect_environment "$(state_path "$environment")" ;; --cleanup) - now=$(date +%s) + now=$(date +%s); failed=0 for target in "$state_dir"/*.state; do [[ -e $target ]] || continue secure_file "$target" 600 expires=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$target") [[ $expires =~ ^[0-9]+$ ]] || die "invalid expiration in $target" - if ((expires <= now)); then remove_environment "$(basename "$target" .state)"; fi + if ((expires <= now)) && ! remove_environment "$(basename "$target" .state)"; then failed=1; fi done + ((failed == 0)) || die 'one or more expired environments could not be removed' report 'CLEANUP_OK' ;; --health) From ad5f7c7b3f4ffe8fee8898f15b808cb150b6339c Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 00:17:29 -0500 Subject: [PATCH 03/16] test: cover tester cleanup failures --- scripts/test-install-tester.sh | 15 +++++++++++++-- scripts/tester-runtime.sh | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 4aaa05d..34142f3 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -85,9 +85,20 @@ FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --remove --environment immutable-remove FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --reset --environment preview-a >/dev/null grep -q 'down --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not remove only the scoped Compose project' if grep -Eq 'system prune|volume prune|network prune' "$tmp/docker.log"; then fail 'global Docker prune was used'; fi -sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$state" +write_environment expired-a 18087 +FAKE_TESTER_ROUTE_PORT=18087 "$runtime" --converge --environment expired-a >/dev/null +write_environment expired-b 18088 +FAKE_TESTER_ROUTE_PORT=18088 "$runtime" --converge --environment expired-b >/dev/null +for expired_state in "$state" "$root/var/lib/ci-fleet-tester/environments/expired-a.state" "$root/var/lib/ci-fleet-tester/environments/expired-b.state"; do + sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$expired_state" +done +: >"$tmp/docker.log" +if FAKE_TESTER_DOWN_FAIL=1 "$runtime" --cleanup >/dev/null 2>&1; then fail 'cleanup ignored environment removal failures'; fi +for id in preview-a expired-a expired-b; do + grep -Fq "ci-fleet-test-$id" "$tmp/docker.log" || fail "cleanup stopped before attempting $id" +done "$runtime" --cleanup >/dev/null -[[ ! -e $state ]] || fail 'expired environment survived cleanup' +[[ ! -e $state && ! -e $root/var/lib/ci-fleet-tester/environments/expired-a.state && ! -e $root/var/lib/ci-fleet-tester/environments/expired-b.state ]] || fail 'expired environment survived cleanup' # Commit-backed installer tests run after the implementation commit exists. ref=$(git -C "$repo_root" rev-parse HEAD) diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 6747fce..1a1304e 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -200,7 +200,7 @@ remove_environment() { compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") [[ $compose == "$(deployed_compose_path "$id")" ]] || die 'stored compose path is unexpected' secure_file "$compose" 600 - docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans + docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans || return 1 rm -f -- "$target" "$(deployed_compose_path "$id")" fi report "REMOVED environment=$id" From a75e2fac15b5a2caf6fd3506686a19759a706ab6 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 00:18:41 -0500 Subject: [PATCH 04/16] docs: clarify tester credential boundary --- docs/TESTER-HOST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/TESTER-HOST.md b/docs/TESTER-HOST.md index 1ef2664..5062455 100644 --- a/docs/TESTER-HOST.md +++ b/docs/TESTER-HOST.md @@ -13,7 +13,7 @@ A tester host accepts only: - application images addressed by an immutable `sha256` digest; - environment secrets stored below that environment's fixed host-local secret directory. -It rejects mutable images, public port binds, host bind mounts, external/unscoped Docker resources, privileged containers, added capabilities, host namespaces, global container names, and credentials outside the environment secret boundary. Every service must be read-only, drop all capabilities, and set `no-new-privileges`. Test identity, networks, storage, routes, domains, data, and credentials must have no production authority. Host/network isolation is an external acceptance gate, not something a repository script can prove. +It rejects mutable images, public port binds, host bind mounts, external/unscoped Docker resources, custom volume drivers/options, privileged containers, added capabilities, host or shared namespaces, Docker API access, global container names, and credentials outside the environment secret boundary. Compose environment variables, env files, and configs are forbidden credential channels; use only fixed mode-`0600` Compose secrets. Every service must be read-only, drop all capabilities, and set `no-new-privileges=true`. The validated rendered Compose model is copied into protected runtime state before activation, so partial starts remain tracked and later cleanup does not depend on a mutable or deleted source definition. Test identity, networks, storage, routes, domains, data, and credentials must have no production authority. Host/network isolation is an external acceptance gate, not something this repository-only change can prove. ## Prepare host-local configuration From f36fa02c6d2b2a76d48c68b0e01fe3d960847a18 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 00:32:59 -0500 Subject: [PATCH 05/16] test: support read-only releases as non-root --- scripts/install-tester.sh | 4 +++- scripts/test-install-tester.sh | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 8f5569b..9a3aa6e 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -187,7 +187,9 @@ case $action in ensure_directories if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi remove_units - rm -f -- "$current_link" "$lkg_file"; rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" + rm -f -- "$current_link" "$lkg_file" + [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$release_dir" + rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" report 'UNINSTALL_OK preserved_config=true preserved_definitions=true preserved_secrets=true' ;; esac diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 34142f3..f2bcfea 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -4,7 +4,8 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) runtime=$repo_root/scripts/tester-runtime.sh installer=$repo_root/scripts/install-tester.sh tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT +cleanup() { chmod -R u+w "$tmp" 2>/dev/null || true; rm -rf "$tmp"; } +trap cleanup EXIT fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } root=$tmp/root fake_bin=$tmp/bin @@ -112,7 +113,7 @@ for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet release=$root/opt/ci-fleet-tester/releases/$ref chmod u+w "$release/scripts/tester-runtime.sh"; printf '# tamper\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'tampered installed release passed check'; fi -git -C "$repo_root" show "$ref:scripts/tester-runtime.sh" >"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" +chmod u+w "$release/scripts/tester-runtime.sh"; git -C "$repo_root" show "$ref:scripts/tester-runtime.sh" >"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'restored release failed check' # A syntactically valid candidate that fails its post-switch check restores the incumbent. From ee89e26ffa76a75a1ee86922d3bd3ae48d99add4 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 01:00:11 -0500 Subject: [PATCH 06/16] fix: close remaining tester isolation channels --- scripts/fixtures/fake-tester-docker.sh | 12 +++++++++--- scripts/test-install-tester.sh | 4 ++-- scripts/tester-runtime.sh | 7 +++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 2b06e3e..3eb0fd7 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -25,7 +25,7 @@ done case ${operation:-} in config) digest=$(printf 'a%.0s' {1..64}) - privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=; top_extra=; volume_extra=; security='no-new-privileges:true' + privileged=false; read_only=true; host_ip=127.0.0.1; image="registry.example/example/app@sha256:$digest"; network_name="${project}_default"; secrets='{}'; service_extra=; top_extra=; volume_extra=; network_extra=; security='no-new-privileges:true' case ${FAKE_TESTER_POLICY:-valid} in mutable) image=registry.example/example/app:latest ;; privileged) privileged=true ;; @@ -38,11 +38,17 @@ case ${operation:-} in namespace-share) service_extra=',"network_mode":"service:other"' ;; false-nnp) security='no-new-privileges:false' ;; custom-volume) volume_extra=',"driver":"local"' ;; + volumes-from) service_extra=',"volumes_from":["container:other:rw"]' ;; + custom-network) network_extra=',"driver":"macvlan","driver_opts":{"parent":"eth0"}' ;; + replicas) service_extra=',"deploy":{"replicas":2}' ;; + lifecycle-hook) service_extra=',"post_start":[{"command":"true","privileged":true}]' ;; + gpu) service_extra=',"gpus":"all"' ;; + deploy-device) service_extra=',"deploy":{"resources":{"reservations":{"devices":[{"capabilities":["gpu"]}]}}}' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} - printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["%s"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]%s}},"networks":{"default":{"name":"%s"}},"volumes":{"data":{"name":"%s_data"%s}},"secrets":%s%s}\n' \ - "$image" "$privileged" "$read_only" "$security" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$service_extra" "$network_name" "$project" "$volume_extra" "$secrets" "$top_extra" + printf '{"services":{"web":{"image":"%s","privileged":%s,"read_only":%s,"cap_drop":["ALL"],"security_opt":["%s"],"volumes":[%s],"ports":[{"host_ip":"%s","published":%s,"target":8080,"protocol":"tcp"}]%s}},"networks":{"default":{"name":"%s"%s}},"volumes":{"data":{"name":"%s_data"%s}},"secrets":%s%s}\n' \ + "$image" "$privileged" "$read_only" "$security" "$volume" "$host_ip" "${FAKE_TESTER_ROUTE_PORT:-18080}" "$service_extra" "$network_name" "$network_extra" "$project" "$volume_extra" "$secrets" "$top_extra" ;; up) [[ ${FAKE_TESTER_UP_FAIL:-0} != 1 ]] ;; down) [[ ${FAKE_TESTER_DOWN_FAIL:-0} != 1 ]] ;; diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index f2bcfea..8dbd22f 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -59,7 +59,7 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done @@ -139,7 +139,7 @@ if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-teste old=0000000000000000000000000000000000000000 cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/releases/$old" chmod 0755 "$root/opt/ci-fleet-tester/releases/$old" -chmod 0644 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" +chmod 0644 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" printf '%s\n' "$old" >"$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" (cd "$root/opt/ci-fleet-tester/releases/$old" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) chmod 0444 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 1a1304e..360f61e 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -120,8 +120,10 @@ image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64} ports=[] for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') - if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc'): raise SystemExit(f'{name}: external namespace/privileged access is forbidden') - if service.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket'): raise SystemExit(f'{name}: device/capability/global identity is forbidden') + if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/privileged lifecycle access is forbidden') + deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} + if service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: device/capability/external mount/global identity is forbidden') + if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') if not any(re.fullmatch(r'no-new-privileges[:=]true', option) for option in service.get('security_opt',[])): raise SystemExit(f'{name}: no-new-privileges=true is required') @@ -136,6 +138,7 @@ for section in ('networks','volumes'): for name,item in value.get(section,{}).items(): resolved=item.get('name',f'{project}_{name}') if item.get('external') or not resolved.startswith(f'{project}_'): raise SystemExit(f'{section}.{name}: external/unscoped names are forbidden') + if section == 'networks' and (item.get('driver') not in (None,'bridge') or item.get('driver_opts')): raise SystemExit(f'{section}.{name}: custom network drivers are forbidden') if section == 'volumes' and (item.get('driver') or item.get('driver_opts')): raise SystemExit(f'{section}.{name}: custom volume drivers are forbidden') for name,item in value.get('secrets',{}).items(): path=item.get('file') From 196b121df60e23fa2380d955405775c69d444b89 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 01:05:39 -0500 Subject: [PATCH 07/16] fix: make tester lifecycle repairable --- scripts/install-tester.sh | 39 ++++++++++++++++++++++------------ scripts/test-install-tester.sh | 30 ++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 9a3aa6e..7f4cd3f 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -47,11 +47,19 @@ secret_root=$config_root/secrets runtime_state=$state_root/environments docker_root=$(root_path /var/lib/docker) docker_socket=$(root_path /var/run/docker.sock) +runtime_lock=$(root_path /run/lock/ci-fleet-tester.lock) units=(ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer) timers=(ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer) secure_file() { [[ -f $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected file is unsafe: $1"; } secure_dir() { [[ -d $1 && ! -L $1 && $(stat -c %u "$1") == "$expected_uid" && $(stat -c %a "$1") == "$2" ]] || die "protected directory is unsafe: $1"; } +remove_release_tree() { [[ ! -e $1 ]] || { [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$1"; rm -rf -- "$1"; }; } + +reject_git_replacements() { + local common + common=$(git -C "$repo_root" rev-parse --path-format=absolute --git-common-dir) + [[ -z $(git -C "$repo_root" for-each-ref --format='%(refname)' refs/replace) && ! -s $common/info/grafts ]] || die 'Git replacement or graft metadata is forbidden' +} host_preflight() { local os_release docker_context actual_root used @@ -91,17 +99,19 @@ release_complete() { } stage_release() { - local commit=$1 target=$release_dir/$1 staging=$release_dir/.staging-$1 - if [[ -e $target ]]; then release_complete "$target" "$commit" || die 'existing tester release is incomplete'; return; fi - rm -rf -- "$staging"; install -d -m 0755 "$staging" - git -C "$repo_root" cat-file -e "$commit^{commit}" 2>/dev/null || die 'requested source commit is unavailable locally' - git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" + local commit=$1 target=$release_dir/$1 staging=$release_dir/.staging-$1 replaced=$release_dir/.replaced-$1 + if [[ -e $target ]] && release_complete "$target" "$commit"; then return; fi + remove_release_tree "$staging"; remove_release_tree "$replaced"; install -d -m 0755 "$staging" + GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" cat-file -e "$commit^{commit}" 2>/dev/null || die 'requested source commit is unavailable locally' + GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" chmod 0755 "$staging/scripts/tester-runtime.sh"; shellcheck "$staging/scripts/tester-runtime.sh"; bash -n "$staging/scripts/tester-runtime.sh" (cd "$staging" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) chmod 0444 "$staging/.ci-fleet-source-revision" "$staging/.ci-fleet-release.sha256" "$staging"/host/systemd/* chmod 0555 "$staging" "$staging/scripts" "$staging/host" "$staging/host/systemd" "$staging/scripts/tester-runtime.sh" - mv -T "$staging" "$target" + [[ ! -e $target ]] || mv -T "$target" "$replaced" + if ! mv -T "$staging" "$target"; then [[ ! -e $replaced ]] || mv -T "$replaced" "$target"; die 'could not replace tester release'; fi + remove_release_tree "$replaced" } install_units() { @@ -112,9 +122,9 @@ install_units() { } remove_units() { - systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true - local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit"; done - systemctl daemon-reload || true + systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 + local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done + systemctl daemon-reload } write_lkg() { @@ -133,11 +143,11 @@ activate_release() { ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" else rm -f -- "$current_link" - remove_units + remove_units || true fi die 'candidate tester activation failed; previous release restored when available' fi - [[ ! $previous =~ ^[0-9a-f]{40}$ || $previous == "$commit" ]] || write_lkg "$previous" + if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous"; then write_lkg "$previous"; fi report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" } @@ -152,7 +162,8 @@ case $action in --install|--upgrade) host_preflight; ensure_directories secure_file "$(root_path "$config")" 600 - [[ $(git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$ref" ]] || die 'reviewed checkout HEAD does not match --ref' + reject_git_replacements + [[ $(GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" rev-parse 'HEAD^{commit}') == "$ref" ]] || die 'reviewed checkout HEAD does not match --ref' if ! git -C "$repo_root" diff --quiet || ! git -C "$repo_root" diff --cached --quiet; then die 'reviewed checkout has tracked changes'; fi current=$(installed_revision || true) [[ $action != --install || -z $current || $current == "$ref" ]] || die 'tester is already installed at another revision; use --upgrade' @@ -164,6 +175,7 @@ case $action in secure_file "$(root_path "$config")" 600 current=$(installed_revision) || die 'tester is not installed' release_complete "$release_dir/$current" "$current" || die 'installed release is incomplete' + for unit in "${units[@]}"; do cmp -s "$release_dir/$current/host/systemd/$unit" "$systemd_dir/$unit" || die "installed unit differs from active release: $unit"; done for timer in "${timers[@]}"; do if ! systemctl is-enabled --quiet "$timer" || ! systemctl is-active --quiet "$timer"; then die "timer is inactive: $timer"; fi done @@ -185,8 +197,9 @@ case $action in ;; --uninstall) ensure_directories + install -d -m 0755 "$(dirname "$runtime_lock")"; exec 8>"$runtime_lock"; flock -x 8 if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi - remove_units + remove_units || die 'could not stop and disable tester maintenance units' rm -f -- "$current_link" "$lkg_file" [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$release_dir" rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 8dbd22f..f0edcc7 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -113,8 +113,12 @@ for unit in ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet release=$root/opt/ci-fleet-tester/releases/$ref chmod u+w "$release/scripts/tester-runtime.sh"; printf '# tamper\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'tampered installed release passed check'; fi -chmod u+w "$release/scripts/tester-runtime.sh"; git -C "$repo_root" show "$ref:scripts/tester-runtime.sh" >"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" -"$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'restored release failed check' +"$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null || fail 'same-ref install did not repair an incomplete release' +"$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null || fail 'repaired release failed check' +installed_unit=$root/etc/systemd/system/ci-fleet-tester-health.service +printf '# drift\n' >>"$installed_unit" +if "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installed unit drift passed check'; fi +cp "$release/host/systemd/ci-fleet-tester-health.service" "$installed_unit" # A syntactically valid candidate that fails its post-switch check restores the incumbent. upgrade_repo=$tmp/upgrade-repo @@ -125,6 +129,11 @@ git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.exampl bad_ref=$(git -C "$upgrade_repo" rev-parse HEAD) if "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'failed candidate activation succeeded'; fi [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'failed upgrade did not restore incumbent release' +git -C "$upgrade_repo" checkout --quiet "$ref" +git -C "$upgrade_repo" replace "$ref" "$bad_ref" +if "$upgrade_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null 2>&1; then fail 'Git replacement metadata was accepted'; fi +git -C "$upgrade_repo" replace -d "$ref" >/dev/null +git -C "$upgrade_repo" checkout --quiet "$bad_ref" # Unit activation failures restore the incumbent symlink and units. git -C "$upgrade_repo" show "$ref:scripts/tester-runtime.sh" >"$upgrade_repo/scripts/tester-runtime.sh"; chmod 0755 "$upgrade_repo/scripts/tester-runtime.sh" @@ -135,6 +144,15 @@ unit_hash_before=$(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.se if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'unit activation failure succeeded'; fi [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" && $(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") == "$unit_hash_before" ]] || fail 'unit activation failure did not restore incumbent release and units' +# A corrupt incumbent is never recorded as the rollback target. +chmod u+w "$release/scripts/tester-runtime.sh"; printf '# corrupt incumbent\n' >>"$release/scripts/tester-runtime.sh"; chmod 0555 "$release/scripts/tester-runtime.sh" +printf '# candidate\n' >>"$upgrade_repo/docs/TESTER-HOST.md" +git -C "$upgrade_repo" add docs/TESTER-HOST.md +git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: second valid tester candidate' +valid_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +"$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$valid_ref" >/dev/null || fail 'valid upgrade failed' +[[ ! -e $root/var/lib/ci-fleet-tester/last-known-good ]] || fail 'corrupt incumbent was recorded as last-known-good' + # Rollback switches only to a complete recorded release and keeps environments intact. old=0000000000000000000000000000000000000000 cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/releases/$old" @@ -153,6 +171,14 @@ rollback_state_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/roll [[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") == "$rollback_state_hash" ]] || fail 'rollback changed active environment state' FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --remove --environment rollback-env >/dev/null +lock_file=$root/run/lock/ci-fleet-tester.lock +lock_ready=$tmp/lock-ready +flock "$lock_file" -c "touch '$lock_ready'; sleep 1" & lock_pid=$! +while [[ ! -e $lock_ready ]]; do kill -0 "$lock_pid" 2>/dev/null || fail 'could not acquire fixture lifecycle lock'; done +if timeout 0.2 "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored the runtime lifecycle lock'; fi +wait "$lock_pid" +if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored systemd teardown failure'; fi +[[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' [[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' printf 'TESTER_INSTALLER_TESTS_OK\n' From ca39b25234b859c6f9163278645389b6676c530a Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 01:45:14 -0500 Subject: [PATCH 08/16] fix: serialize tester lifecycle updates --- scripts/fixtures/fake-tester-docker.sh | 2 ++ scripts/install-tester.sh | 12 ++++++++++-- scripts/test-install-tester.sh | 18 ++++++++++++++++-- scripts/tester-runtime.sh | 17 +++++++++++------ 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 3eb0fd7..069b000 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -44,6 +44,8 @@ case ${operation:-} in lifecycle-hook) service_extra=',"post_start":[{"command":"true","privileged":true}]' ;; gpu) service_extra=',"gpus":"all"' ;; deploy-device) service_extra=',"deploy":{"resources":{"reservations":{"devices":[{"capabilities":["gpu"]}]}}}' ;; + build) service_extra=',"build":{"context":"/"}' ;; + changed-model) service_extra=',"pull_policy":"always"' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 7f4cd3f..0e6b0fc 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -61,6 +61,13 @@ reject_git_replacements() { [[ -z $(git -C "$repo_root" for-each-ref --format='%(refname)' refs/replace) && ! -s $common/info/grafts ]] || die 'Git replacement or graft metadata is forbidden' } +acquire_lifecycle_lock() { + install -d -m 0755 "$(dirname "$runtime_lock")" + exec 8>"$runtime_lock" + flock -x 8 + export CI_FLEET_TESTER_LOCK_FD=8 +} + host_preflight() { local os_release docker_context actual_root used os_release=$(root_path /etc/os-release) @@ -142,8 +149,8 @@ activate_release() { if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" else + remove_units || die 'candidate activation failed and fresh-install unit teardown also failed; candidate retained for recovery' rm -f -- "$current_link" - remove_units || true fi die 'candidate tester activation failed; previous release restored when available' fi @@ -158,6 +165,8 @@ installed_revision() { basename "$target" } +case $action in --install|--upgrade|--check|--rollback|--uninstall) acquire_lifecycle_lock ;; esac + case $action in --install|--upgrade) host_preflight; ensure_directories @@ -197,7 +206,6 @@ case $action in ;; --uninstall) ensure_directories - install -d -m 0755 "$(dirname "$runtime_lock")"; exec 8>"$runtime_lock"; flock -x 8 if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi remove_units || die 'could not stop and disable tester maintenance units' rm -f -- "$current_link" "$lkg_file" diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index f0edcc7..274d0a8 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -59,10 +59,13 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device build; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done +deployed_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") +if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=changed-model "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge replaced the incumbent Compose model without reset'; fi +[[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") == "$deployed_hash" ]] || fail 'rejected model replacement changed incumbent state' if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=mutable "$runtime" --reset --environment preview-a >/dev/null 2>&1; then fail 'reset accepted invalid replacement'; fi [[ -f $state ]] || fail 'reset deleted the incumbent before validation' if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_CONTAINER_STATE='exited unhealthy' "$runtime" --health >/dev/null 2>&1; then fail 'health accepted a stopped managed service'; fi @@ -75,6 +78,9 @@ outside_secret=$root/etc/ci-fleet-tester/secrets/outside printf 'example-test-scope-value\n' >"$outside_secret"; chmod 600 "$outside_secret" write_environment outside-secret 18083 if FAKE_TESTER_ROUTE_PORT=18083 FAKE_TESTER_POLICY=outside-secret FAKE_TESTER_SECRET_FILE=$outside_secret "$runtime" --converge --environment outside-secret >/dev/null 2>&1; then fail 'out-of-boundary secret was accepted'; fi +write_environment hardlink-secret 18089 +ln "$outside_secret" "$root/etc/ci-fleet-tester/secrets/hardlink-secret/credential" +if FAKE_TESTER_ROUTE_PORT=18089 FAKE_TESTER_POLICY=valid-secret FAKE_TESTER_SECRET_FILE=$root/etc/ci-fleet-tester/secrets/hardlink-secret/credential "$runtime" --converge --environment hardlink-secret >/dev/null 2>&1; then fail 'hard-linked host secret was accepted'; fi write_environment partial-up 18084 if FAKE_TESTER_ROUTE_PORT=18084 FAKE_TESTER_UP_FAIL=1 "$runtime" --converge --environment partial-up >/dev/null 2>&1; then fail 'partial activation succeeded'; fi [[ -f $root/var/lib/ci-fleet-tester/environments/partial-up.state && -f $root/var/lib/ci-fleet-tester/environments/partial-up.compose.json ]] || fail 'partial activation was not tracked for cleanup' @@ -175,10 +181,18 @@ lock_file=$root/run/lock/ci-fleet-tester.lock lock_ready=$tmp/lock-ready flock "$lock_file" -c "touch '$lock_ready'; sleep 1" & lock_pid=$! while [[ ! -e $lock_ready ]]; do kill -0 "$lock_pid" 2>/dev/null || fail 'could not acquire fixture lifecycle lock'; done -if timeout 0.2 "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored the runtime lifecycle lock'; fi +set +e +timeout 0.2 "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null 2>&1 +lock_rc=$? +set -e +[[ $lock_rc == 124 ]] || fail 'installer lifecycle mutation ignored the shared lock' wait "$lock_pid" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored systemd teardown failure'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' [[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' +git -C "$upgrade_repo" checkout --quiet "$bad_ref" +if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$upgrade_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'invalid fresh install succeeded'; fi +[[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed fresh-install teardown removed the recovery link' +"$upgrade_repo/scripts/install-tester.sh" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null printf 'TESTER_INSTALLER_TESTS_OK\n' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 360f61e..0a95b05 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -33,7 +33,7 @@ while (($#)); do done [[ -n $action ]] || { usage; exit 2; } case $action in --converge|--reset|--remove|--inspect) [[ $environment =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die 'environment ID is invalid' ;; *) [[ -z $environment ]] || die '--environment is not valid for this action' ;; esac -for command in awk basename chmod curl date df dirname docker du find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done +for command in awk basename chmod cmp curl date df dirname docker du find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done secure_directory() { local path=$1 mode=$2 @@ -122,7 +122,7 @@ for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/privileged lifecycle access is forbidden') deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} - if service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: device/capability/external mount/global identity is forbidden') + if service.get('build') or service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: build/device/capability/external mount/global identity is forbidden') if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') @@ -144,7 +144,7 @@ for name,item in value.get('secrets',{}).items(): path=item.get('file') if item.get('external') or not isinstance(path,str) or os.path.realpath(path).rsplit('/',1)[0] != secret_dir: raise SystemExit(f'secrets.{name}: secret must be a host-local file in the environment secret directory') metadata=os.lstat(path) - if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_uid != expected_uid or stat.S_IMODE(metadata.st_mode) != 0o600: raise SystemExit(f'secrets.{name}: secret must be owner-controlled mode 0600') + if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode) or metadata.st_nlink != 1 or metadata.st_uid != expected_uid or stat.S_IMODE(metadata.st_mode) != 0o600: raise SystemExit(f'secrets.{name}: secret must be a singly linked owner-controlled mode-0600 file') PY image_digests=$(python3 - "$rendered" <<'PY' import json,sys @@ -180,6 +180,7 @@ prepare_converge() { [[ -f $(state_path "$environment") || $count -lt $max_environments ]] || die 'maximum environment count reached' prepared_rendered=$(mktemp) if ! validate_compose "$prepared_rendered"; then rm -f "$prepared_rendered"; die 'compose policy validation failed'; fi + if [[ $action == --converge && -f $(state_path "$environment") ]] && ! cmp -s "$prepared_rendered" "$(deployed_compose_path "$environment")"; then rm -f "$prepared_rendered"; die 'changing an installed Compose model requires reset'; fi } apply_converge() { @@ -245,9 +246,13 @@ fi unset DOCKER_CONTEXT export DOCKER_HOST="unix://$docker_socket" [[ $(docker info --format '{{.DockerRootDir}}') == "$(root_path /var/lib/docker)" ]] || die 'Docker daemon root is not the expected local path' -install -d -m 0755 "$(dirname "$lock_file")" -exec 9>"$lock_file" -flock -x 9 +if [[ ${CI_FLEET_TESTER_LOCK_FD:-} == 8 && -e /proc/$$/fd/8 && $(readlink -f /proc/$$/fd/8) == "$lock_file" ]] && flock -n 8; then + unset CI_FLEET_TESTER_LOCK_FD +else + install -d -m 0755 "$(dirname "$lock_file")" + exec 9>"$lock_file" + flock -x 9 +fi case $action in --check) docker info --format '{{.DockerRootDir}}' >/dev/null From b4736e5e1a4e9a77ca3d473d643bd97230765d03 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 02:08:26 -0500 Subject: [PATCH 09/16] fix: contain tester recovery failures --- scripts/install-tester.sh | 8 +++++--- scripts/test-install-tester.sh | 15 ++++++++++++++- scripts/tester-runtime.sh | 17 +++++++++++------ 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 0e6b0fc..007c786 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -123,7 +123,8 @@ stage_release() { install_units() { local source=$1 unit - for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit" || return 1; done + for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit.new" || return 1; done + for unit in "${units[@]}"; do mv -fT "$systemd_dir/$unit.new" "$systemd_dir/$unit" || return 1; done systemctl daemon-reload || return 1 systemctl enable --now "${timers[@]}" >/dev/null || return 1 } @@ -147,7 +148,8 @@ activate_release() { ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then - ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link"; install_units "$release_dir/$previous" + ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" + install_units "$release_dir/$previous" || die 'candidate activation failed and incumbent unit restore failed; incumbent link retained for recovery' else remove_units || die 'candidate activation failed and fresh-install unit teardown also failed; candidate retained for recovery' rm -f -- "$current_link" @@ -165,7 +167,7 @@ installed_revision() { basename "$target" } -case $action in --install|--upgrade|--check|--rollback|--uninstall) acquire_lifecycle_lock ;; esac +case $action in --install|--upgrade|--check|--reset|--rollback|--uninstall) acquire_lifecycle_lock ;; esac case $action in --install|--upgrade) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 274d0a8..951f09b 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -50,8 +50,10 @@ state=$root/var/lib/ci-fleet-tester/environments/preview-a.state [[ -f $state && $(stat -c %a "$state") == 600 ]] || fail 'state was not protected' inspect_output=$(FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --inspect --environment preview-a) grep -q 'IMAGE_DIGESTS=sha256:[a-f0-9]\{64\}.*STATUS=running DISK_BYTES=[1-9][0-9]*' <<<"$inspect_output" || fail 'inspect did not report health and disk use' +deployed_inode=$(stat -c %i "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null [[ $(find "$root/var/lib/ci-fleet-tester/environments" -name '*.state' | wc -l) == 1 ]] || fail 'idempotent converge duplicated state' +[[ $(stat -c %i "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") == "$deployed_inode" ]] || fail 'idempotent converge replaced the protected Compose snapshot' original_expiry=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") grep -v '^CI_FLEET_TESTER_EXPIRES_AT=' "$root/etc/ci-fleet-tester/environments/preview-a.env" >"$tmp/spec" mv "$tmp/spec" "$root/etc/ci-fleet-tester/environments/preview-a.env"; chmod 600 "$root/etc/ci-fleet-tester/environments/preview-a.env" @@ -100,6 +102,11 @@ for expired_state in "$state" "$root/var/lib/ci-fleet-tester/environments/expire sed -i 's/^EXPIRES_AT=.*/EXPIRES_AT=1/' "$expired_state" done : >"$tmp/docker.log" +mv "$root/var/lib/ci-fleet-tester/environments/expired-a.compose.json" "$tmp/expired-a.compose.json" +if FAKE_TESTER_DOWN_FAIL=1 "$runtime" --cleanup >/dev/null 2>&1; then fail 'cleanup ignored a damaged protected environment'; fi +for id in preview-a expired-b; do grep -Fq "ci-fleet-test-$id" "$tmp/docker.log" || fail "damaged state stopped cleanup before $id"; done +mv "$tmp/expired-a.compose.json" "$root/var/lib/ci-fleet-tester/environments/expired-a.compose.json" +: >"$tmp/docker.log" if FAKE_TESTER_DOWN_FAIL=1 "$runtime" --cleanup >/dev/null 2>&1; then fail 'cleanup ignored environment removal failures'; fi for id in preview-a expired-a expired-b; do grep -Fq "ci-fleet-test-$id" "$tmp/docker.log" || fail "cleanup stopped before attempting $id" @@ -147,7 +154,8 @@ git -C "$upgrade_repo" add scripts/tester-runtime.sh git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: valid tester candidate' unit_fail_ref=$(git -C "$upgrade_repo" rev-parse HEAD) unit_hash_before=$(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") -if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >/dev/null 2>&1; then fail 'unit activation failure succeeded'; fi +if FAKE_TESTER_SYSTEMCTL_FAIL=daemon-reload "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$unit_fail_ref" >"$tmp/unit-restore.log" 2>&1; then fail 'unit activation failure succeeded'; fi +grep -Fq 'incumbent unit restore failed' "$tmp/unit-restore.log" || fail 'secondary incumbent restore failure was not surfaced' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" && $(sha256sum "$root/etc/systemd/system/ci-fleet-tester-health.service") == "$unit_hash_before" ]] || fail 'unit activation failure did not restore incumbent release and units' # A corrupt incumbent is never recorded as the rollback target. @@ -186,6 +194,11 @@ timeout 0.2 "$installer" --install --config /etc/ci-fleet-tester/tester.env --re lock_rc=$? set -e [[ $lock_rc == 124 ]] || fail 'installer lifecycle mutation ignored the shared lock' +set +e +timeout 0.2 "$installer" --reset --environment absent >/dev/null 2>&1 +reset_lock_rc=$? +set -e +[[ $reset_lock_rc == 124 ]] || fail 'reset resolved the active release before acquiring the lifecycle lock' wait "$lock_pid" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored systemd teardown failure'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 0a95b05..fc5636f 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -180,11 +180,14 @@ prepare_converge() { [[ -f $(state_path "$environment") || $count -lt $max_environments ]] || die 'maximum environment count reached' prepared_rendered=$(mktemp) if ! validate_compose "$prepared_rendered"; then rm -f "$prepared_rendered"; die 'compose policy validation failed'; fi - if [[ $action == --converge && -f $(state_path "$environment") ]] && ! cmp -s "$prepared_rendered" "$(deployed_compose_path "$environment")"; then rm -f "$prepared_rendered"; die 'changing an installed Compose model requires reset'; fi + if [[ $action == --converge && -f $(state_path "$environment") ]]; then + secure_file "$(deployed_compose_path "$environment")" 600 + if ! cmp -s "$prepared_rendered" "$(deployed_compose_path "$environment")"; then rm -f "$prepared_rendered"; die 'changing an installed Compose model requires reset'; fi + fi } apply_converge() { - install -m 0600 "$prepared_rendered" "$(deployed_compose_path "$environment")" + [[ -f $(state_path "$environment") ]] || install -m 0600 "$prepared_rendered" "$(deployed_compose_path "$environment")" write_state if ! docker compose -p "$compose_project" -f "$(deployed_compose_path "$environment")" up -d --remove-orphans --wait; then rm -f "$prepared_rendered" @@ -271,10 +274,12 @@ case $action in now=$(date +%s); failed=0 for target in "$state_dir"/*.state; do [[ -e $target ]] || continue - secure_file "$target" 600 - expires=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$target") - [[ $expires =~ ^[0-9]+$ ]] || die "invalid expiration in $target" - if ((expires <= now)) && ! remove_environment "$(basename "$target" .state)"; then failed=1; fi + if ! ( + secure_file "$target" 600 + expires=$(awk -F= '$1=="EXPIRES_AT"{print $2}' "$target") + [[ $expires =~ ^[0-9]+$ ]] || die "invalid expiration in $target" + ((expires > now)) || remove_environment "$(basename "$target" .state)" + ); then failed=1; fi done ((failed == 0)) || die 'one or more expired environments could not be removed' report 'CLEANUP_OK' From 841406c946339fcca682a8b54bffc49550d485ff Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 02:28:16 -0500 Subject: [PATCH 10/16] fix: quiesce tester timers during activation --- scripts/fixtures/fake-tester-docker.sh | 2 ++ scripts/install-tester.sh | 8 +++++--- scripts/test-install-tester.sh | 11 +++++++++-- scripts/tester-runtime.sh | 4 ++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 069b000..1458c88 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" +[[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'docker %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi if [[ $1 == ps ]]; then printf 'fixture-container-id\n'; exit 0; fi @@ -46,6 +47,7 @@ case ${operation:-} in deploy-device) service_extra=',"deploy":{"resources":{"reservations":{"devices":[{"capabilities":["gpu"]}]}}}' ;; build) service_extra=',"build":{"context":"/"}' ;; changed-model) service_extra=',"pull_policy":"always"' ;; + unconfined) security='no-new-privileges:true","seccomp=unconfined' ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 007c786..9dbf373 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -126,9 +126,10 @@ install_units() { for unit in "${units[@]}"; do install -m 0644 "$source/host/systemd/$unit" "$systemd_dir/$unit.new" || return 1; done for unit in "${units[@]}"; do mv -fT "$systemd_dir/$unit.new" "$systemd_dir/$unit" || return 1; done systemctl daemon-reload || return 1 - systemctl enable --now "${timers[@]}" >/dev/null || return 1 } +enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } + remove_units() { systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done @@ -145,11 +146,12 @@ activate_release() { local commit=$1 target=$release_dir/$1 previous= release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") + if [[ $previous =~ ^[0-9a-f]{40}$ ]]; then systemctl disable --now "${timers[@]}" >/dev/null || die 'could not quiesce tester maintenance timers'; fi ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health; then + if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - install_units "$release_dir/$previous" || die 'candidate activation failed and incumbent unit restore failed; incumbent link retained for recovery' + if ! install_units "$release_dir/$previous" || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; incumbent link retained for recovery'; fi else remove_units || die 'candidate activation failed and fresh-install unit teardown also failed; candidate retained for recovery' rm -f -- "$current_link" diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 951f09b..f9b1376 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -24,6 +24,7 @@ EOF cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" +[[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'systemctl %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" [[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" @@ -31,6 +32,7 @@ printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/getent" chmod 0755 "$fake_bin/df" "$fake_bin/systemctl" "$fake_bin/curl" "$fake_bin/getent" export PATH="$fake_bin:$PATH" CI_FLEET_TESTING=1 CI_FLEET_ROOT_PREFIX=$root export FAKE_TESTER_DOCKER_ROOT=$root/var/lib/docker FAKE_TESTER_VOLUME_ROOT=$root/var/lib/fake-tester-volume FAKE_TESTER_DOCKER_LOG=$tmp/docker.log FAKE_TESTER_SYSTEMCTL_LOG=$tmp/systemctl.log +export FAKE_TESTER_EVENT_LOG=$tmp/events.log write_environment() { local id=$1 port=$2 @@ -61,7 +63,7 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device build; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device build; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done @@ -92,7 +94,7 @@ FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --converge --environment immutable-remov rm "$root/etc/ci-fleet-tester/definitions/immutable-remove.yaml" FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --remove --environment immutable-remove >/dev/null FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --reset --environment preview-a >/dev/null -grep -q 'down --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not remove only the scoped Compose project' +grep -q 'down --timeout 10 --volumes --remove-orphans' "$tmp/docker.log" || fail 'reset did not use bounded scoped Compose teardown' if grep -Eq 'system prune|volume prune|network prune' "$tmp/docker.log"; then fail 'global Docker prune was used'; fi write_environment expired-a 18087 FAKE_TESTER_ROUTE_PORT=18087 "$runtime" --converge --environment expired-a >/dev/null @@ -164,8 +166,13 @@ printf '# candidate\n' >>"$upgrade_repo/docs/TESTER-HOST.md" git -C "$upgrade_repo" add docs/TESTER-HOST.md git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: second valid tester candidate' valid_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +: >"$tmp/events.log" "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$valid_ref" >/dev/null || fail 'valid upgrade failed' [[ ! -e $root/var/lib/ci-fleet-tester/last-known-good ]] || fail 'corrupt incumbent was recorded as last-known-good' +disable_line=$(grep -n '^systemctl disable --now ' "$tmp/events.log" | tail -1 | cut -d: -f1) +check_line=$(grep -n '^docker info ' "$tmp/events.log" | tail -1 | cut -d: -f1) +enable_line=$(grep -n '^systemctl enable --now ' "$tmp/events.log" | tail -1 | cut -d: -f1) +[[ -n $disable_line && -n $check_line && -n $enable_line && $disable_line -lt $check_line && $check_line -lt $enable_line ]] || fail 'timers were not quiesced until candidate validation completed' # Rollback switches only to a complete recorded release and keeps environments intact. old=0000000000000000000000000000000000000000 diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index fc5636f..c68f235 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -126,7 +126,7 @@ for name,service in services.items(): if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') if service.get('environment') or service.get('env_file') or service.get('configs'): raise SystemExit(f'{name}: alternate credential channels are forbidden') if service.get('read_only') is not True or 'ALL' not in service.get('cap_drop',[]): raise SystemExit(f'{name}: read_only and cap_drop ALL are required') - if not any(re.fullmatch(r'no-new-privileges[:=]true', option) for option in service.get('security_opt',[])): raise SystemExit(f'{name}: no-new-privileges=true is required') + if service.get('security_opt') not in (['no-new-privileges:true'],['no-new-privileges=true']): raise SystemExit(f'{name}: no-new-privileges=true must be the only security option') for mount in service.get('volumes',[]): if isinstance(mount,str) or mount.get('type') not in ('volume','tmpfs'): raise SystemExit(f'{name}: host bind mounts are forbidden') for port in service.get('ports',[]): @@ -207,7 +207,7 @@ remove_environment() { compose=$(awk -F= '$1=="COMPOSE_FILE"{print substr($0,index($0,"=")+1)}' "$target") [[ $compose == "$(deployed_compose_path "$id")" ]] || die 'stored compose path is unexpected' secure_file "$compose" 600 - docker compose -p "$(project_name "$id")" -f "$compose" down --volumes --remove-orphans || return 1 + docker compose -p "$(project_name "$id")" -f "$compose" down --timeout 10 --volumes --remove-orphans || return 1 rm -f -- "$target" "$(deployed_compose_path "$id")" fi report "REMOVED environment=$id" From d88df91e1dd4895d808e766b8d0ba6dcf9db1528 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 02:45:29 -0500 Subject: [PATCH 11/16] fix: close tester isolation gaps --- scripts/fixtures/fake-tester-docker.sh | 4 ++++ scripts/install-tester.sh | 5 ++++- scripts/test-install-tester.sh | 10 +++++++++- scripts/tester-runtime.sh | 14 +++++++++----- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index 1458c88..e5afbc7 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -48,6 +48,10 @@ case ${operation:-} in build) service_extra=',"build":{"context":"/"}' ;; changed-model) service_extra=',"pull_policy":"always"' ;; unconfined) security='no-new-privileges:true","seccomp=unconfined' ;; + external-links) service_extra=',"external_links":["other-environment-db:db"]' ;; + userns-host) service_extra=',"userns_mode":"host"' ;; + cgroup-host) service_extra=',"cgroup":"host"' ;; + interpolation) [[ -z ${TOKEN:-} ]] || service_extra=$(printf ',"command":["app","--token=%s"]' "$TOKEN") ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac volume=${volume:-'{"type":"volume","source":"data","target":"/data"}'} diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 9dbf373..6f385e7 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -146,7 +146,10 @@ activate_release() { local commit=$1 target=$release_dir/$1 previous= release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") - if [[ $previous =~ ^[0-9a-f]{40}$ ]]; then systemctl disable --now "${timers[@]}" >/dev/null || die 'could not quiesce tester maintenance timers'; fi + if [[ $previous =~ ^[0-9a-f]{40}$ ]] && ! systemctl disable --now "${timers[@]}" >/dev/null; then + enable_timers || die 'could not quiesce or restore tester maintenance timers' + die 'could not quiesce tester maintenance timers; incumbent timers restored' + fi ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index f9b1376..ec42439 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -63,10 +63,14 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from custom-network replicas lifecycle-hook gpu deploy-device build; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host custom-network replicas lifecycle-hook gpu deploy-device build; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done +write_environment interpolation 18090 +TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null +if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi +FAKE_TESTER_ROUTE_PORT=18090 "$runtime" --remove --environment interpolation >/dev/null deployed_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") if FAKE_TESTER_ROUTE_PORT=18080 FAKE_TESTER_POLICY=changed-model "$runtime" --converge --environment preview-a >/dev/null 2>&1; then fail 'converge replaced the incumbent Compose model without reset'; fi [[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") == "$deployed_hash" ]] || fail 'rejected model replacement changed incumbent state' @@ -88,6 +92,7 @@ if FAKE_TESTER_ROUTE_PORT=18089 FAKE_TESTER_POLICY=valid-secret FAKE_TESTER_SECR write_environment partial-up 18084 if FAKE_TESTER_ROUTE_PORT=18084 FAKE_TESTER_UP_FAIL=1 "$runtime" --converge --environment partial-up >/dev/null 2>&1; then fail 'partial activation succeeded'; fi [[ -f $root/var/lib/ci-fleet-tester/environments/partial-up.state && -f $root/var/lib/ci-fleet-tester/environments/partial-up.compose.json ]] || fail 'partial activation was not tracked for cleanup' +grep -q 'up -d --remove-orphans --wait --wait-timeout 60' "$tmp/docker.log" || fail 'Compose activation wait was not bounded' FAKE_TESTER_ROUTE_PORT=18084 "$runtime" --remove --environment partial-up >/dev/null write_environment immutable-remove 18085 FAKE_TESTER_ROUTE_PORT=18085 "$runtime" --converge --environment immutable-remove >/dev/null @@ -166,6 +171,9 @@ printf '# candidate\n' >>"$upgrade_repo/docs/TESTER-HOST.md" git -C "$upgrade_repo" add docs/TESTER-HOST.md git -C "$upgrade_repo" -c user.name=Example -c user.email=example@invalid.example commit --quiet -m 'fixture: second valid tester candidate' valid_ref=$(git -C "$upgrade_repo" rev-parse HEAD) +: >"$tmp/systemctl.log" +if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$valid_ref" >/dev/null 2>&1; then fail 'upgrade ignored timer quiescence failure'; fi +grep -Fq 'enable --now ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer' "$tmp/systemctl.log" || fail 'quiescence failure did not restore incumbent timers' : >"$tmp/events.log" "$upgrade_repo/scripts/install-tester.sh" --upgrade --config /etc/ci-fleet-tester/tester.env --ref "$valid_ref" >/dev/null || fail 'valid upgrade failed' [[ ! -e $root/var/lib/ci-fleet-tester/last-known-good ]] || fail 'corrupt incumbent was recorded as last-known-good' diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index c68f235..89679b8 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -33,7 +33,7 @@ while (($#)); do done [[ -n $action ]] || { usage; exit 2; } case $action in --converge|--reset|--remove|--inspect) [[ $environment =~ ^[a-z0-9][a-z0-9-]{0,62}$ ]] || die 'environment ID is invalid' ;; *) [[ -z $environment ]] || die '--environment is not valid for this action' ;; esac -for command in awk basename chmod cmp curl date df dirname docker du find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done +for command in awk basename chmod cmp curl date df dirname docker du env find flock getent grep install mktemp mv python3 readlink rm stat wc; do command -v "$command" >/dev/null || die "required command is unavailable: $command"; done secure_directory() { local path=$1 mode=$2 @@ -108,8 +108,12 @@ load_spec() { } validate_compose() { - local rendered=$1 - docker compose -p "$compose_project" -f "$compose_file" config --format json >"$rendered" || return 1 + local rendered=$1 empty_env variable + local -a clean_environment=(env -i "PATH=$PATH" "DOCKER_HOST=$DOCKER_HOST") + if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done; fi + empty_env=$(mktemp); chmod 600 "$empty_env" + if ! "${clean_environment[@]}" docker compose --env-file "$empty_env" -p "$compose_project" -f "$compose_file" config --format json >"$rendered"; then rm -f "$empty_env"; return 1; fi + rm -f "$empty_env" chmod 600 "$rendered" python3 - "$rendered" "$route_service" "$route_port" "$compose_project" "$secret_dir" "$expected_uid" <<'PY' || return 1 import json,os,re,stat,sys @@ -120,7 +124,7 @@ image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64} ports=[] for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') - if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/privileged lifecycle access is forbidden') + if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('userns_mode') or service.get('cgroup') or service.get('external_links') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/link/privileged lifecycle access is forbidden') deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} if service.get('build') or service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: build/device/capability/external mount/global identity is forbidden') if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') @@ -189,7 +193,7 @@ prepare_converge() { apply_converge() { [[ -f $(state_path "$environment") ]] || install -m 0600 "$prepared_rendered" "$(deployed_compose_path "$environment")" write_state - if ! docker compose -p "$compose_project" -f "$(deployed_compose_path "$environment")" up -d --remove-orphans --wait; then + if ! docker compose -p "$compose_project" -f "$(deployed_compose_path "$environment")" up -d --remove-orphans --wait --wait-timeout 60; then rm -f "$prepared_rendered" die 'environment activation failed; tracked state retained for cleanup' fi From bec7c76a708b841eb93e02959ef3a71c2d939ba8 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 03:08:49 -0500 Subject: [PATCH 12/16] fix: route tester lifecycle through stable launcher --- docs/TESTER-HOST.md | 8 +++---- host/systemd/ci-fleet-tester-cleanup.service | 4 ++-- host/systemd/ci-fleet-tester-health.service | 4 ++-- scripts/fixtures/fake-tester-docker.sh | 2 ++ scripts/install-tester.sh | 23 +++++++++++--------- scripts/test-install-tester.sh | 16 ++++++++++++-- scripts/tester-launcher.sh | 21 ++++++++++++++++++ scripts/tester-runtime.sh | 11 ++++++++-- 8 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 scripts/tester-launcher.sh diff --git a/docs/TESTER-HOST.md b/docs/TESTER-HOST.md index 5062455..670caa7 100644 --- a/docs/TESTER-HOST.md +++ b/docs/TESTER-HOST.md @@ -74,10 +74,10 @@ The command fails before mutation unless it sees Debian 12 or newer, root, the l The runtime command is the one interface for create/update, inspect, reset, and removal: ```bash -sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --converge --environment example-preview -sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --inspect --environment example-preview -sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --reset --environment example-preview -sudo /opt/ci-fleet-tester/current/scripts/tester-runtime.sh --remove --environment example-preview +sudo /opt/ci-fleet-tester/tester-runtime --converge --environment example-preview +sudo /opt/ci-fleet-tester/tester-runtime --inspect --environment example-preview +sudo /opt/ci-fleet-tester/tester-runtime --reset --environment example-preview +sudo /opt/ci-fleet-tester/tester-runtime --remove --environment example-preview ``` `--converge` validates the full resolved Compose model before `up --wait`. `--reset` removes only that exact Compose project and its volumes, then recreates it from the approved definition and digest. `--remove` uses the same scoped `compose down --volumes`; no global Docker prune is used. State reports only environment/project/owner, loopback route, expiry, source revision, timestamps, and health—not Compose environment values or secret content. diff --git a/host/systemd/ci-fleet-tester-cleanup.service b/host/systemd/ci-fleet-tester-cleanup.service index ccd6012..b98d87b 100644 --- a/host/systemd/ci-fleet-tester-cleanup.service +++ b/host/systemd/ci-fleet-tester-cleanup.service @@ -6,7 +6,7 @@ ConditionPathExists=/etc/ci-fleet-tester/tester.env [Service] Type=oneshot -ExecStart=/opt/ci-fleet-tester/current/scripts/tester-runtime.sh --cleanup +ExecStart=/opt/ci-fleet-tester/tester-runtime --cleanup User=root Group=root UMask=0077 @@ -14,7 +14,7 @@ NoNewPrivileges=yes PrivateTmp=yes ProtectHome=yes ProtectSystem=strict -ReadWritePaths=/var/lib/ci-fleet-tester /run/lock +ReadWritePaths=/var/lib/ci-fleet-tester /run/lock/ci-fleet-tester RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-health.service b/host/systemd/ci-fleet-tester-health.service index 92585f1..658f712 100644 --- a/host/systemd/ci-fleet-tester-health.service +++ b/host/systemd/ci-fleet-tester-health.service @@ -6,7 +6,7 @@ ConditionPathExists=/etc/ci-fleet-tester/tester.env [Service] Type=oneshot -ExecStart=/opt/ci-fleet-tester/current/scripts/tester-runtime.sh --health +ExecStart=/opt/ci-fleet-tester/tester-runtime --health User=root Group=root UMask=0077 @@ -14,7 +14,7 @@ NoNewPrivileges=yes PrivateTmp=yes ProtectHome=yes ProtectSystem=strict -ReadWritePaths=/var/lib/ci-fleet-tester /run/lock +ReadWritePaths=/var/lib/ci-fleet-tester /run/lock/ci-fleet-tester RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index e5afbc7..f0bf394 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -51,6 +51,8 @@ case ${operation:-} in external-links) service_extra=',"external_links":["other-environment-db:db"]' ;; userns-host) service_extra=',"userns_mode":"host"' ;; cgroup-host) service_extra=',"cgroup":"host"' ;; + uts-host) service_extra=',"uts":"host"' ;; + remote-logging) service_extra=',"logging":{"driver":"syslog","options":{"syslog-address":"tcp://example.invalid:514"}}' ;; interpolation) [[ -z ${TOKEN:-} ]] || service_extra=$(printf ',"command":["app","--token=%s"]' "$TOKEN") ;; valid-secret|outside-secret) secrets=$(printf '{"credential":{"file":"%s"}}' "${FAKE_TESTER_SECRET_FILE:?}") ;; esac diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index 6f385e7..a435f00 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -37,6 +37,7 @@ repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")/.." rev-parse --show-toplevel opt_dir=$(root_path /opt/ci-fleet-tester) release_dir=$opt_dir/releases current_link=$opt_dir/current +stable_launcher=$opt_dir/tester-runtime state_root=$(root_path /var/lib/ci-fleet-tester) lkg_file=$state_root/last-known-good systemd_dir=$(root_path /etc/systemd/system) @@ -47,7 +48,7 @@ secret_root=$config_root/secrets runtime_state=$state_root/environments docker_root=$(root_path /var/lib/docker) docker_socket=$(root_path /var/run/docker.sock) -runtime_lock=$(root_path /run/lock/ci-fleet-tester.lock) +runtime_lock=$(root_path /run/lock/ci-fleet-tester/runtime.lock) units=(ci-fleet-tester-health.service ci-fleet-tester-health.timer ci-fleet-tester-cleanup.service ci-fleet-tester-cleanup.timer) timers=(ci-fleet-tester-health.timer ci-fleet-tester-cleanup.timer) @@ -99,7 +100,7 @@ ensure_directories() { release_complete() { local path=$1 expected=$2 unit - [[ -d $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == 555 && -x $path/scripts/tester-runtime.sh && -f $path/.ci-fleet-source-revision ]] || return 1 + [[ -d $path && ! -L $path && $(stat -c %u "$path") == "$expected_uid" && $(stat -c %a "$path") == 555 && -x $path/scripts/tester-runtime.sh && -x $path/scripts/tester-launcher.sh && -f $path/.ci-fleet-source-revision ]] || return 1 [[ $(<"$path/.ci-fleet-source-revision") == "$expected" ]] || return 1 for unit in "${units[@]}"; do [[ -f $path/host/systemd/$unit ]] || return 1; done (cd "$path" && sha256sum --status -c .ci-fleet-release.sha256) || return 1 @@ -110,12 +111,12 @@ stage_release() { if [[ -e $target ]] && release_complete "$target" "$commit"; then return; fi remove_release_tree "$staging"; remove_release_tree "$replaced"; install -d -m 0755 "$staging" GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" cat-file -e "$commit^{commit}" 2>/dev/null || die 'requested source commit is unavailable locally' - GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" + GIT_NO_REPLACE_OBJECTS=1 git -C "$repo_root" archive "$commit" scripts/tester-runtime.sh scripts/tester-launcher.sh host/systemd/ci-fleet-tester-health.service host/systemd/ci-fleet-tester-health.timer host/systemd/ci-fleet-tester-cleanup.service host/systemd/ci-fleet-tester-cleanup.timer | tar -x -C "$staging" printf '%s\n' "$commit" >"$staging/.ci-fleet-source-revision"; chmod 0644 "$staging/.ci-fleet-source-revision" - chmod 0755 "$staging/scripts/tester-runtime.sh"; shellcheck "$staging/scripts/tester-runtime.sh"; bash -n "$staging/scripts/tester-runtime.sh" - (cd "$staging" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) + chmod 0755 "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh"; shellcheck "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh"; bash -n "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh" + (cd "$staging" && sha256sum scripts/tester-runtime.sh scripts/tester-launcher.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) chmod 0444 "$staging/.ci-fleet-source-revision" "$staging/.ci-fleet-release.sha256" "$staging"/host/systemd/* - chmod 0555 "$staging" "$staging/scripts" "$staging/host" "$staging/host/systemd" "$staging/scripts/tester-runtime.sh" + chmod 0555 "$staging" "$staging/scripts" "$staging/host" "$staging/host/systemd" "$staging/scripts/tester-runtime.sh" "$staging/scripts/tester-launcher.sh" [[ ! -e $target ]] || mv -T "$target" "$replaced" if ! mv -T "$staging" "$target"; then [[ ! -e $replaced ]] || mv -T "$replaced" "$target"; die 'could not replace tester release'; fi remove_release_tree "$replaced" @@ -129,6 +130,7 @@ install_units() { } enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } +install_launcher() { install -m 0555 "$1/scripts/tester-launcher.sh" "$stable_launcher.new" && mv -fT "$stable_launcher.new" "$stable_launcher"; } remove_units() { systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 @@ -151,13 +153,13 @@ activate_release() { die 'could not quiesce tester maintenance timers; incumbent timers restored' fi ln -sfn "$target" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then + if ! install_launcher "$target" || ! install_units "$target" || ! "$target/scripts/tester-runtime.sh" --check || ! "$target/scripts/tester-runtime.sh" --health || ! enable_timers; then if [[ $previous =~ ^[0-9a-f]{40}$ ]] && release_complete "$release_dir/$previous" "$previous"; then ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" - if ! install_units "$release_dir/$previous" || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; incumbent link retained for recovery'; fi + if ! install_launcher "$release_dir/$previous" || ! install_units "$release_dir/$previous" || ! enable_timers; then die 'candidate activation failed and incumbent unit restore failed; launcher and incumbent link retained for recovery'; fi else remove_units || die 'candidate activation failed and fresh-install unit teardown also failed; candidate retained for recovery' - rm -f -- "$current_link" + rm -f -- "$current_link" "$stable_launcher" fi die 'candidate tester activation failed; previous release restored when available' fi @@ -191,6 +193,7 @@ case $action in secure_file "$(root_path "$config")" 600 current=$(installed_revision) || die 'tester is not installed' release_complete "$release_dir/$current" "$current" || die 'installed release is incomplete' + cmp -s "$release_dir/$current/scripts/tester-launcher.sh" "$stable_launcher" || die 'installed launcher differs from active release' for unit in "${units[@]}"; do cmp -s "$release_dir/$current/host/systemd/$unit" "$systemd_dir/$unit" || die "installed unit differs from active release: $unit"; done for timer in "${timers[@]}"; do if ! systemctl is-enabled --quiet "$timer" || ! systemctl is-active --quiet "$timer"; then die "timer is inactive: $timer"; fi @@ -215,7 +218,7 @@ case $action in ensure_directories if find "$runtime_state" -maxdepth 1 -type f -name '*.state' | grep -q .; then die 'remove every test environment before uninstalling the tester service'; fi remove_units || die 'could not stop and disable tester maintenance units' - rm -f -- "$current_link" "$lkg_file" + rm -f -- "$current_link" "$stable_launcher" "$lkg_file" [[ ${CI_FLEET_TESTING:-0} != 1 ]] || chmod -R u+w "$release_dir" rm -rf -- "$release_dir"; install -d -m 0755 "$release_dir" report 'UNINSTALL_OK preserved_config=true preserved_definitions=true preserved_secrets=true' diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index ec42439..9876522 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -10,6 +10,7 @@ fail() { printf 'FAIL %s\n' "$*" >&2; exit 1; } root=$tmp/root fake_bin=$tmp/bin mkdir -p "$fake_bin" "$root/etc/ci-fleet-tester/environments" "$root/etc/ci-fleet-tester/definitions" "$root/etc/ci-fleet-tester/secrets" "$root/var/lib/ci-fleet-tester/environments" "$root/var/lib/docker" "$root/var/lib/fake-tester-volume" "$root/var/run" "$root/etc/systemd/system" +mkdir -p "$root/run/lock"; chmod 1777 "$root/run/lock" chmod 700 "$root/etc/ci-fleet-tester" "$root/etc/ci-fleet-tester/environments" "$root/etc/ci-fleet-tester/definitions" "$root/etc/ci-fleet-tester/secrets" "$root/var/lib/ci-fleet-tester" "$root/var/lib/ci-fleet-tester/environments" printf 'ID=debian\nVERSION_ID=13\n' >"$root/etc/os-release" : >"$root/var/run/docker.sock" @@ -45,6 +46,7 @@ write_environment() { } "$runtime" --check | grep -Fq CHECK_OK || fail 'runtime preflight failed' +[[ $(stat -c %a "$root/run/lock") == 1777 && $(stat -c %a "$root/run/lock/ci-fleet-tester") == 755 ]] || fail 'runtime changed shared lock-directory permissions' if FAKE_TESTER_DOCKER_ROOT=/remote/docker "$runtime" --check >/dev/null 2>&1; then fail 'remote Docker daemon was accepted'; fi write_environment preview-a 18080 FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a | grep -Fq CONVERGED || fail 'converge failed' @@ -63,10 +65,13 @@ FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev [[ $(awk -F= '$1=="EXPIRES_AT"{print $2}' "$state") == "$original_expiry" ]] || fail 'idempotent converge extended expiration' write_environment preview-b 18080 if FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-b >/dev/null 2>&1; then fail 'duplicate route port was accepted'; fi -for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host custom-network replicas lifecycle-hook gpu deploy-device build; do +for policy in mutable privileged bind broad-port external-network environment configs use-api-socket namespace-share false-nnp unconfined custom-volume volumes-from external-links userns-host cgroup-host uts-host remote-logging custom-network replicas lifecycle-hook gpu deploy-device build; do write_environment "bad-$policy" 18081 if FAKE_TESTER_ROUTE_PORT=18081 FAKE_TESTER_POLICY=$policy "$runtime" --converge --environment "bad-$policy" >/dev/null 2>&1; then fail "unsafe compose policy was accepted: $policy"; fi done +write_environment bad-include 18091 +printf 'include:\n - path: /etc/passwd\nservices: {}\n' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" +if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'Compose include was accepted before rendering'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi @@ -126,6 +131,8 @@ ref=$(git -C "$repo_root" rev-parse HEAD) if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' +[[ -x $root/opt/ci-fleet-tester/tester-runtime ]] || fail 'stable tester launcher was not installed' +for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; do grep -Fq 'ExecStart=/opt/ci-fleet-tester/tester-runtime' "$root/etc/systemd/system/$service" || fail "$service bypasses stable launcher"; done "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" >/dev/null check_output=$("$installer" --check --config /etc/ci-fleet-tester/tester.env) grep -Fq CHECK_OK <<<"$check_output" || fail 'installed check failed' @@ -188,7 +195,7 @@ cp -a "$root/opt/ci-fleet-tester/releases/$ref" "$root/opt/ci-fleet-tester/relea chmod 0755 "$root/opt/ci-fleet-tester/releases/$old" chmod 0644 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" printf '%s\n' "$old" >"$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" -(cd "$root/opt/ci-fleet-tester/releases/$old" && sha256sum scripts/tester-runtime.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) +(cd "$root/opt/ci-fleet-tester/releases/$old" && sha256sum scripts/tester-runtime.sh scripts/tester-launcher.sh .ci-fleet-source-revision host/systemd/* >.ci-fleet-release.sha256) chmod 0444 "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-source-revision" "$root/opt/ci-fleet-tester/releases/$old/.ci-fleet-release.sha256" chmod 0555 "$root/opt/ci-fleet-tester/releases/$old" printf '%s\n' "$old" >"$root/var/lib/ci-fleet-tester/last-known-good"; chmod 600 "$root/var/lib/ci-fleet-tester/last-known-good" @@ -214,6 +221,11 @@ timeout 0.2 "$installer" --reset --environment absent >/dev/null 2>&1 reset_lock_rc=$? set -e [[ $reset_lock_rc == 124 ]] || fail 'reset resolved the active release before acquiring the lifecycle lock' +set +e +timeout 0.2 "$root/opt/ci-fleet-tester/tester-runtime" --inspect --environment absent >/dev/null 2>&1 +launcher_lock_rc=$? +set -e +[[ $launcher_lock_rc == 124 ]] || fail 'stable launcher resolved the active release before acquiring the lifecycle lock' wait "$lock_pid" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'uninstall ignored systemd teardown failure'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' diff --git a/scripts/tester-launcher.sh b/scripts/tester-launcher.sh new file mode 100644 index 0000000..6c8fd13 --- /dev/null +++ b/scripts/tester-launcher.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 + +root_prefix=${CI_FLEET_ROOT_PREFIX:-} +[[ -z $root_prefix || ${CI_FLEET_TESTING:-0} == 1 ]] || { printf 'ERROR: CI_FLEET_ROOT_PREFIX is test-only\n' >&2; exit 1; } +root_path() { printf '%s%s' "$root_prefix" "$1"; } +lock_dir=$(root_path /run/lock/ci-fleet-tester) +lock_file=$lock_dir/runtime.lock +release_dir=$(root_path /opt/ci-fleet-tester/releases) +current_link=$(root_path /opt/ci-fleet-tester/current) +expected_uid=0 +[[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) +[[ -d $lock_dir && ! -L $lock_dir && $(stat -c %u "$lock_dir") == "$expected_uid" && $(stat -c %a "$lock_dir") == 755 ]] || { printf 'ERROR: tester lock directory is unsafe\n' >&2; exit 1; } +exec 8>"$lock_file" +flock -x 8 +target=$(readlink -f "$current_link") +revision=$(basename "$target") +[[ $target == "$release_dir/$revision" && $revision =~ ^[0-9a-f]{40}$ && -x $target/scripts/tester-runtime.sh ]] || { printf 'ERROR: active tester release is invalid\n' >&2; exit 1; } +export CI_FLEET_TESTER_LOCK_FD=8 +exec "$target/scripts/tester-runtime.sh" "$@" diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 89679b8..2a5b441 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -13,7 +13,7 @@ environment_dir=$(root_path /etc/ci-fleet-tester/environments) definition_dir=$(root_path /etc/ci-fleet-tester/definitions) secret_root=$(root_path /etc/ci-fleet-tester/secrets) state_dir=$(root_path /var/lib/ci-fleet-tester/environments) -lock_file=$(root_path /run/lock/ci-fleet-tester.lock) +lock_file=$(root_path /run/lock/ci-fleet-tester/runtime.lock) expected_uid=0 [[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) @@ -110,6 +110,13 @@ load_spec() { validate_compose() { local rendered=$1 empty_env variable local -a clean_environment=(env -i "PATH=$PATH" "DOCKER_HOST=$DOCKER_HOST") + if ! python3 - "$compose_file" <<'PY' +import re,sys +text=open(sys.argv[1],encoding='utf-8').read() +key=r'(?:!!str[ \t]+)?(?:include|"include"|\x27include\x27)[ \t]*:' +if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text): raise SystemExit('Compose include is forbidden') +PY + then return 1; fi if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done; fi empty_env=$(mktemp); chmod 600 "$empty_env" if ! "${clean_environment[@]}" docker compose --env-file "$empty_env" -p "$compose_project" -f "$compose_file" config --format json >"$rendered"; then rm -f "$empty_env"; return 1; fi @@ -124,7 +131,7 @@ image=re.compile(r'^[a-z0-9.-]+(?::[0-9]+)?/[A-Za-z0-9_./-]+@sha256:[0-9a-f]{64} ports=[] for name,service in services.items(): if not image.fullmatch(str(service.get('image',''))): raise SystemExit(f'{name}: image must use an immutable sha256 digest') - if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('userns_mode') or service.get('cgroup') or service.get('external_links') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/link/privileged lifecycle access is forbidden') + if service.get('privileged') or service.get('network_mode') or service.get('pid') or service.get('ipc') or service.get('uts') or service.get('userns_mode') or service.get('cgroup') or service.get('external_links') or service.get('logging') or service.get('post_start') or service.get('pre_stop'): raise SystemExit(f'{name}: external namespace/link/logging/privileged lifecycle access is forbidden') deploy=service.get('deploy') or {}; reservations=(deploy.get('resources') or {}).get('reservations') or {} if service.get('build') or service.get('devices') or service.get('gpus') or reservations.get('devices') or service.get('cap_add') or service.get('container_name') or service.get('hostname') or service.get('use_api_socket') or service.get('volumes_from'): raise SystemExit(f'{name}: build/device/capability/external mount/global identity is forbidden') if deploy.get('replicas',1) != 1: raise SystemExit(f'{name}: exactly one replica is required') From 7c0e0e34b66d8e0607cc13297c2d2a3cbb92881d Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 03:20:42 -0500 Subject: [PATCH 13/16] test: follow dedicated tester lifecycle lock --- scripts/test-install-tester.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 9876522..20aabf5 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -207,7 +207,7 @@ rollback_state_hash=$(sha256sum "$root/var/lib/ci-fleet-tester/environments/roll [[ $(sha256sum "$root/var/lib/ci-fleet-tester/environments/rollback-env.state") == "$rollback_state_hash" ]] || fail 'rollback changed active environment state' FAKE_TESTER_ROUTE_PORT=18086 "$runtime" --remove --environment rollback-env >/dev/null -lock_file=$root/run/lock/ci-fleet-tester.lock +lock_file=$root/run/lock/ci-fleet-tester/runtime.lock lock_ready=$tmp/lock-ready flock "$lock_file" -c "touch '$lock_ready'; sleep 1" & lock_pid=$! while [[ ! -e $lock_ready ]]; do kill -0 "$lock_pid" 2>/dev/null || fail 'could not acquire fixture lifecycle lock'; done From 3c56ef7ee32423cfe093fe30f1faaa0c0fa4746b Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 21:13:18 -0500 Subject: [PATCH 14/16] fix: make tester uninstall idempotent --- scripts/install-tester.sh | 8 ++++++-- scripts/test-install-tester.sh | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index a435f00..b6daa82 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -133,8 +133,12 @@ enable_timers() { systemctl enable --now "${timers[@]}" >/dev/null; } install_launcher() { install -m 0555 "$1/scripts/tester-launcher.sh" "$stable_launcher.new" && mv -fT "$stable_launcher.new" "$stable_launcher"; } remove_units() { - systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 - local unit; for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done + local unit present=0 + for unit in "${units[@]}"; do [[ ! -e $systemd_dir/$unit ]] || present=1; done + if ((present)); then systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || return 1 + else systemctl disable --now "${timers[@]}" >/dev/null 2>&1 || true + fi + for unit in "${units[@]}"; do rm -f -- "$systemd_dir/$unit" || return 1; done systemctl daemon-reload } diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index 20aabf5..f4b945c 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -26,6 +26,7 @@ cat >"$fake_bin/systemctl" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_TESTER_SYSTEMCTL_LOG:?}" [[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'systemctl %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" +if [[ ${FAKE_TESTER_SYSTEMCTL_FAIL_IF_UNITS_MISSING:-0} == 1 && $1 == disable && ! -e ${CI_FLEET_ROOT_PREFIX:?}/etc/systemd/system/ci-fleet-tester-health.timer ]]; then exit 5; fi [[ -z ${FAKE_TESTER_SYSTEMCTL_FAIL:-} || " $* " != *" $FAKE_TESTER_SYSTEMCTL_FAIL "* ]] EOF printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_bin/curl" @@ -231,6 +232,7 @@ if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$installer" --uninstall --config [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed uninstall removed the active release' "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'uninstall failed' [[ -f $root/etc/ci-fleet-tester/tester.env && ! -L $root/opt/ci-fleet-tester/current ]] || fail 'uninstall did not preserve config/remove runtime' +FAKE_TESTER_SYSTEMCTL_FAIL_IF_UNITS_MISSING=1 "$installer" --uninstall --config /etc/ci-fleet-tester/tester.env | grep -Fq UNINSTALL_OK || fail 'repeated uninstall failed on absent units' git -C "$upgrade_repo" checkout --quiet "$bad_ref" if FAKE_TESTER_SYSTEMCTL_FAIL='disable --now' "$upgrade_repo/scripts/install-tester.sh" --install --config /etc/ci-fleet-tester/tester.env --ref "$bad_ref" >/dev/null 2>&1; then fail 'invalid fresh install succeeded'; fi [[ -L $root/opt/ci-fleet-tester/current ]] || fail 'failed fresh-install teardown removed the recovery link' From a725c9881ae1a153ca60009c03df6acaa18d3471 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 21:42:10 -0500 Subject: [PATCH 15/16] fix: close final tester lifecycle gaps --- host/systemd/ci-fleet-tester-cleanup.service | 2 +- host/systemd/ci-fleet-tester-health.service | 2 +- scripts/fixtures/fake-tester-docker.sh | 5 +++-- scripts/install-tester.sh | 7 +++++-- scripts/test-install-tester.sh | 10 ++++++++++ scripts/tester-launcher.sh | 1 + scripts/tester-runtime.sh | 12 ++++++++---- 7 files changed, 29 insertions(+), 10 deletions(-) diff --git a/host/systemd/ci-fleet-tester-cleanup.service b/host/systemd/ci-fleet-tester-cleanup.service index b98d87b..83929a9 100644 --- a/host/systemd/ci-fleet-tester-cleanup.service +++ b/host/systemd/ci-fleet-tester-cleanup.service @@ -14,7 +14,7 @@ NoNewPrivileges=yes PrivateTmp=yes ProtectHome=yes ProtectSystem=strict -ReadWritePaths=/var/lib/ci-fleet-tester /run/lock/ci-fleet-tester +ReadWritePaths=/var/lib/ci-fleet-tester -/run/lock/ci-fleet-tester RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/host/systemd/ci-fleet-tester-health.service b/host/systemd/ci-fleet-tester-health.service index 658f712..be56d1a 100644 --- a/host/systemd/ci-fleet-tester-health.service +++ b/host/systemd/ci-fleet-tester-health.service @@ -14,7 +14,7 @@ NoNewPrivileges=yes PrivateTmp=yes ProtectHome=yes ProtectSystem=strict -ReadWritePaths=/var/lib/ci-fleet-tester /run/lock/ci-fleet-tester +ReadWritePaths=/var/lib/ci-fleet-tester -/run/lock/ci-fleet-tester RestrictAddressFamilies=AF_UNIX LockPersonality=yes MemoryDenyWriteExecute=yes diff --git a/scripts/fixtures/fake-tester-docker.sh b/scripts/fixtures/fake-tester-docker.sh index f0bf394..0e9421e 100755 --- a/scripts/fixtures/fake-tester-docker.sh +++ b/scripts/fixtures/fake-tester-docker.sh @@ -4,14 +4,15 @@ printf '%s\n' "$*" >>"${FAKE_TESTER_DOCKER_LOG:?}" [[ -z ${FAKE_TESTER_EVENT_LOG:-} ]] || printf 'docker %s\n' "$*" >>"$FAKE_TESTER_EVENT_LOG" if [[ $1 == context && $2 == show ]]; then printf 'default\n'; exit 0; fi if [[ $1 == info ]]; then printf '%s\n' "${FAKE_TESTER_DOCKER_ROOT:?}"; exit 0; fi -if [[ $1 == ps ]]; then printf 'fixture-container-id\n'; exit 0; fi +if [[ $1 == ps ]]; then [[ ${FAKE_TESTER_PS_FAIL:-0} != 1 ]] || exit 9; printf 'fixture-container-id\n'; exit 0; fi if [[ $1 == inspect ]]; then if [[ " $* " == *' --size '* ]]; then printf '1024\n'; else printf '%s\n' "${FAKE_TESTER_CONTAINER_STATE:-running healthy}"; fi exit 0 fi -if [[ $1 == volume && $2 == ls ]]; then printf 'fixture-volume\n'; exit 0; fi +if [[ $1 == volume && $2 == ls ]]; then [[ ${FAKE_TESTER_VOLUME_LS_FAIL:-0} != 1 ]] || exit 9; printf 'fixture-volume\n'; exit 0; fi if [[ $1 == volume && $2 == inspect ]]; then printf '%s\n' "${FAKE_TESTER_VOLUME_ROOT:?}"; exit 0; fi if [[ $1 == compose && $2 == version ]]; then printf 'Docker Compose version v2.fixture\n'; exit 0; fi +if [[ $1 == compose && $2 == up && ${3:-} == --help ]]; then [[ ${FAKE_TESTER_NO_WAIT_TIMEOUT:-0} != 1 ]] && printf '%s\n' ' --wait-timeout int'; exit 0; fi if [[ $1 != compose ]]; then exit 2; fi shift project= diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index b6daa82..fb9332a 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -63,7 +63,8 @@ reject_git_replacements() { } acquire_lifecycle_lock() { - install -d -m 0755 "$(dirname "$runtime_lock")" + mkdir -m 0755 "$(dirname "$runtime_lock")" 2>/dev/null || true + secure_dir "$(dirname "$runtime_lock")" 755 exec 8>"$runtime_lock" flock -x 8 export CI_FLEET_TESTER_LOCK_FD=8 @@ -83,6 +84,8 @@ host_preflight() { [[ -S $docker_socket || ( ${CI_FLEET_TESTING:-0} == 1 && -e $docker_socket ) ]] || die 'local Docker socket is unavailable' actual_root=$(docker info --format '{{.DockerRootDir}}'); [[ $actual_root == "$docker_root" ]] || die 'Docker root does not match the local managed root' docker compose version >/dev/null + printf 'services: {}\n' | docker compose -f - config --format json >/dev/null || die 'Compose JSON rendering is unavailable' + docker compose up --help | grep -q -- '--wait-timeout' || die 'Compose wait-timeout support is unavailable' used=$(df -P "$docker_root" | awk 'NR==2{gsub(/%/,"",$5);print $5}') [[ $used =~ ^[0-9]+$ && $used -lt 80 ]] || die 'Docker storage is at or above 80%' } @@ -152,6 +155,7 @@ activate_release() { local commit=$1 target=$release_dir/$1 previous= release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") + if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous"; then write_lkg "$previous"; fi if [[ $previous =~ ^[0-9a-f]{40}$ ]] && ! systemctl disable --now "${timers[@]}" >/dev/null; then enable_timers || die 'could not quiesce or restore tester maintenance timers' die 'could not quiesce tester maintenance timers; incumbent timers restored' @@ -167,7 +171,6 @@ activate_release() { fi die 'candidate tester activation failed; previous release restored when available' fi - if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous"; then write_lkg "$previous"; fi report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" } diff --git a/scripts/test-install-tester.sh b/scripts/test-install-tester.sh index f4b945c..6e2208a 100755 --- a/scripts/test-install-tester.sh +++ b/scripts/test-install-tester.sh @@ -48,6 +48,10 @@ write_environment() { "$runtime" --check | grep -Fq CHECK_OK || fail 'runtime preflight failed' [[ $(stat -c %a "$root/run/lock") == 1777 && $(stat -c %a "$root/run/lock/ci-fleet-tester") == 755 ]] || fail 'runtime changed shared lock-directory permissions' +rm -rf "$root/run/lock/ci-fleet-tester"; mkdir "$tmp/lock-target"; chmod 700 "$tmp/lock-target"; ln -s "$tmp/lock-target" "$root/run/lock/ci-fleet-tester" +if "$runtime" --check >/dev/null 2>&1; then fail 'symlinked lifecycle lock directory was accepted'; fi +[[ $(stat -c %a "$tmp/lock-target") == 700 ]] || fail 'symlinked lock target permissions changed' +rm "$root/run/lock/ci-fleet-tester" if FAKE_TESTER_DOCKER_ROOT=/remote/docker "$runtime" --check >/dev/null 2>&1; then fail 'remote Docker daemon was accepted'; fi write_environment preview-a 18080 FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a | grep -Fq CONVERGED || fail 'converge failed' @@ -55,6 +59,8 @@ state=$root/var/lib/ci-fleet-tester/environments/preview-a.state [[ -f $state && $(stat -c %a "$state") == 600 ]] || fail 'state was not protected' inspect_output=$(FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --inspect --environment preview-a) grep -q 'IMAGE_DIGESTS=sha256:[a-f0-9]\{64\}.*STATUS=running DISK_BYTES=[1-9][0-9]*' <<<"$inspect_output" || fail 'inspect did not report health and disk use' +if FAKE_TESTER_PS_FAIL=1 "$runtime" --inspect --environment preview-a >/dev/null 2>&1; then fail 'container inventory failure was hidden'; fi +if FAKE_TESTER_VOLUME_LS_FAIL=1 "$runtime" --inspect --environment preview-a >/dev/null 2>&1; then fail 'volume inventory failure was hidden'; fi deployed_inode=$(stat -c %i "$root/var/lib/ci-fleet-tester/environments/preview-a.compose.json") FAKE_TESTER_ROUTE_PORT=18080 "$runtime" --converge --environment preview-a >/dev/null [[ $(find "$root/var/lib/ci-fleet-tester/environments" -name '*.state' | wc -l) == 1 ]] || fail 'idempotent converge duplicated state' @@ -73,6 +79,8 @@ done write_environment bad-include 18091 printf 'include:\n - path: /etc/passwd\nservices: {}\n' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'Compose include was accepted before rendering'; fi +printf '"incl\\u0075de": [{path: /etc/passwd}]\nservices: {}\n' >"$root/etc/ci-fleet-tester/definitions/bad-include.yaml" +if FAKE_TESTER_ROUTE_PORT=18091 "$runtime" --converge --environment bad-include >/dev/null 2>&1; then fail 'escaped Compose include was accepted'; fi write_environment interpolation 18090 TOKEN=must-not-render FAKE_TESTER_ROUTE_PORT=18090 FAKE_TESTER_POLICY=interpolation "$runtime" --converge --environment interpolation >/dev/null if grep -Fq must-not-render "$root/var/lib/ci-fleet-tester/environments/interpolation.compose.json"; then fail 'caller environment was interpolated into the Compose model'; fi @@ -131,6 +139,8 @@ done ref=$(git -C "$repo_root" rev-parse HEAD) if DOCKER_HOST=tcp://example.invalid:2375 "$installer" --check --config /etc/ci-fleet-tester/tester.env >/dev/null 2>&1; then fail 'installer accepted a remote Docker selector'; fi "$installer" --install --config /etc/ci-fleet-tester/tester.env --ref "$ref" | grep -Fq INSTALL_OK || fail 'fresh install failed' +rm -rf "$root/run/lock/ci-fleet-tester" +"$root/opt/ci-fleet-tester/tester-runtime" --health >/dev/null || fail 'stable launcher did not recreate the volatile lock directory' [[ $(readlink -f "$root/opt/ci-fleet-tester/current") == "$root/opt/ci-fleet-tester/releases/$ref" ]] || fail 'current release link is wrong' [[ -x $root/opt/ci-fleet-tester/tester-runtime ]] || fail 'stable tester launcher was not installed' for service in ci-fleet-tester-health.service ci-fleet-tester-cleanup.service; do grep -Fq 'ExecStart=/opt/ci-fleet-tester/tester-runtime' "$root/etc/systemd/system/$service" || fail "$service bypasses stable launcher"; done diff --git a/scripts/tester-launcher.sh b/scripts/tester-launcher.sh index 6c8fd13..e5c2987 100644 --- a/scripts/tester-launcher.sh +++ b/scripts/tester-launcher.sh @@ -11,6 +11,7 @@ release_dir=$(root_path /opt/ci-fleet-tester/releases) current_link=$(root_path /opt/ci-fleet-tester/current) expected_uid=0 [[ ${CI_FLEET_TESTING:-0} != 1 ]] || expected_uid=$(id -u) +mkdir -m 0755 "$lock_dir" 2>/dev/null || true [[ -d $lock_dir && ! -L $lock_dir && $(stat -c %u "$lock_dir") == "$expected_uid" && $(stat -c %a "$lock_dir") == 755 ]] || { printf 'ERROR: tester lock directory is unsafe\n' >&2; exit 1; } exec 8>"$lock_file" flock -x 8 diff --git a/scripts/tester-runtime.sh b/scripts/tester-runtime.sh index 2a5b441..44da4b8 100755 --- a/scripts/tester-runtime.sh +++ b/scripts/tester-runtime.sh @@ -114,7 +114,8 @@ validate_compose() { import re,sys text=open(sys.argv[1],encoding='utf-8').read() key=r'(?:!!str[ \t]+)?(?:include|"include"|\x27include\x27)[ \t]*:' -if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text): raise SystemExit('Compose include is forbidden') +escaped_key=r'"[^"\n]*\\[^"\n]*"[ \t]*:' +if re.search(r'(?m)^[ \t]*'+key,text) or re.search(r'[,{][ \t]*'+key,text) or re.search(escaped_key,text): raise SystemExit('Compose include or escaped mapping key is forbidden') PY then return 1; fi if [[ ${CI_FLEET_TESTING:-0} == 1 ]]; then for variable in ${!FAKE_@}; do clean_environment+=("$variable=${!variable}"); done; fi @@ -239,14 +240,16 @@ inspect_environment() { running_state=$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$resource") [[ $running_state == 'running healthy' || $running_state == 'running none' ]] || status=unhealthy done + inventory=$(docker ps -aq --filter "label=com.docker.compose.project=$(project_name "$id")") || die 'container inventory failed' while IFS= read -r resource; do [[ -n $resource ]] || continue value=$(docker inspect --size --format '{{.SizeRw}}' "$resource"); [[ $value =~ ^[0-9]+$ ]] || die 'container disk size is invalid'; bytes=$((bytes + value)) - done < <(docker ps -aq --filter "label=com.docker.compose.project=$(project_name "$id")") + done <<<"$inventory" + inventory=$(docker volume ls -q --filter "label=com.docker.compose.project=$(project_name "$id")") || die 'volume inventory failed' while IFS= read -r resource; do [[ -n $resource ]] || continue mount=$(docker volume inspect --format '{{.Mountpoint}}' "$resource"); value=$(du -sb "$mount" | awk '{print $1}'); [[ $value =~ ^[0-9]+$ ]] || die 'volume disk size is invalid'; bytes=$((bytes + value)) - done < <(docker volume ls -q --filter "label=com.docker.compose.project=$(project_name "$id")") + done <<<"$inventory" printf 'STATUS=%s DISK_BYTES=%s\n' "$status" "$bytes" } @@ -263,7 +266,8 @@ export DOCKER_HOST="unix://$docker_socket" if [[ ${CI_FLEET_TESTER_LOCK_FD:-} == 8 && -e /proc/$$/fd/8 && $(readlink -f /proc/$$/fd/8) == "$lock_file" ]] && flock -n 8; then unset CI_FLEET_TESTER_LOCK_FD else - install -d -m 0755 "$(dirname "$lock_file")" + mkdir -m 0755 "$(dirname "$lock_file")" 2>/dev/null || true + secure_directory "$(dirname "$lock_file")" 755 exec 9>"$lock_file" flock -x 9 fi From e7640e23622aee43751844477a410554ba1cb4e9 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 21:44:17 -0500 Subject: [PATCH 16/16] fix: restore tester when rollback record fails --- scripts/install-tester.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/install-tester.sh b/scripts/install-tester.sh index fb9332a..0333b65 100755 --- a/scripts/install-tester.sh +++ b/scripts/install-tester.sh @@ -155,7 +155,6 @@ activate_release() { local commit=$1 target=$release_dir/$1 previous= release_complete "$target" "$commit" || die 'candidate tester release is incomplete' [[ ! -L $current_link ]] || previous=$(basename "$(readlink -f "$current_link")") - if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous"; then write_lkg "$previous"; fi if [[ $previous =~ ^[0-9a-f]{40}$ ]] && ! systemctl disable --now "${timers[@]}" >/dev/null; then enable_timers || die 'could not quiesce or restore tester maintenance timers' die 'could not quiesce tester maintenance timers; incumbent timers restored' @@ -171,6 +170,11 @@ activate_release() { fi die 'candidate tester activation failed; previous release restored when available' fi + if [[ $previous =~ ^[0-9a-f]{40}$ && $previous != "$commit" ]] && release_complete "$release_dir/$previous" "$previous" && ! write_lkg "$previous"; then + ln -sfn "$release_dir/$previous" "$current_link.new"; mv -Tf "$current_link.new" "$current_link" + if ! install_launcher "$release_dir/$previous" || ! install_units "$release_dir/$previous" || ! enable_timers; then die 'last-known-good recording failed and incumbent restore failed; incumbent link retained for recovery'; fi + die 'last-known-good recording failed; previous release restored' + fi report "INSTALL_OK source_revision=$commit previous_revision=${previous:-none} config=$config" }