From 30386ff273b9a82811fc565aadd83ea09f988947 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 15 Aug 2026 20:58:44 -0500 Subject: [PATCH 01/16] feat: verify reviewed managed image digests --- controller/Dockerfile | 12 +++-- docs/MANAGED-IMAGES.md | 52 +++++++++++++++++++ docs/QUICKSTART.md | 4 +- docs/README.md | 2 +- runner/Dockerfile | 32 ++++++++---- scripts/desired_state.py | 3 ++ scripts/install-worker-controller.sh | 28 ++++++++-- scripts/test-install-worker-controller.sh | 24 ++++++--- scripts/test_desired_state.py | 11 ++++ templates/config-repository/README.md | 4 +- .../examples/multi-host/fleet.json | 38 +++++++++++--- templates/config-repository/fleet.json | 27 ++++++++-- templates/config-repository/fleet.schema.json | 18 ++++++- templates/config-repository/scripts/init.py | 12 +++++ .../config-repository/scripts/test_policy.py | 7 ++- .../config-repository/scripts/validate.py | 15 ++++++ 16 files changed, 248 insertions(+), 41 deletions(-) create mode 100644 docs/MANAGED-IMAGES.md diff --git a/controller/Dockerfile b/controller/Dockerfile index e7e943a..335ff8d 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -1,5 +1,6 @@ # syntax=docker/dockerfile:1.18 -FROM golang:1.26.5-bookworm AS build +FROM golang:1.26.5-bookworm@sha256:53eeac89074db483fdf0ab3be1df32bf6e47562263d2d0d6baa7f26acb4957dd AS build +ARG SOURCE_DATE_EPOCH=1786752000 WORKDIR /src COPY go.mod *.go ./ RUN go mod tidy && go mod verify @@ -8,11 +9,16 @@ ARG CI_FLEET_VERSION=dev ARG CI_FLEET_COMMIT=unknown RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=${CI_FLEET_VERSION} -X main.commitSHA=${CI_FLEET_COMMIT}" -o /out/ci-fleet-controller . -FROM debian:13.6-slim +FROM debian:13.6-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dcbbee90edb3bb48c7f8e94f258 +ARG SOURCE_DATE_EPOCH=1786752000 +ARG DEBIAN_SNAPSHOT=20260815T000000Z ARG CI_FLEET_COMMIT=unknown LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" \ io.randomdevelopment.ci-fleet.managed="true" -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \ +RUN rm -f /etc/apt/sources.list.d/debian.sources \ + && printf 'deb http://snapshot.debian.org/archive/debian/%s trixie main\n' "$DEBIAN_SNAPSHOT" >/etc/apt/sources.list \ + && apt-get -o Acquire::Check-Valid-Until=false update \ + && apt-get install -y --no-install-recommends ca-certificates \ && install -d -o 65532 -g 65532 /run/ci-fleet \ && rm -rf /var/lib/apt/lists/* COPY --from=build /out/ci-fleet-controller /usr/local/bin/ci-fleet-controller diff --git a/docs/MANAGED-IMAGES.md b/docs/MANAGED-IMAGES.md new file mode 100644 index 0000000..83842f8 --- /dev/null +++ b/docs/MANAGED-IMAGES.md @@ -0,0 +1,52 @@ +# Reproducible managed images + +Managed controller and runner images have two independent pins: + +1. the engine checkout fixes all build instructions and checked module hashes; +2. the reviewed private desired state maps that full engine commit to exact local Docker image IDs (`sha256:...`). + +Convergence rejects an image whose revision label is correct but whose image ID differs. A missing image is rebuilt only from the pinned engine release and accepted only when its rebuilt ID equals the reviewed value. Rollback applies the same check and deterministic rebuild before it starts the restored controller. + +## Reviewed upstream inputs + +Reviewed on 2026-08-16 from official upstream endpoints: + +| Input | Immutable selection | Evidence | +| --- | --- | --- | +| Debian 13.6 slim, both managed images | `sha256:3a39a0592364683e6bab97937b72cad5a8fa6dcbbee90edb3bb48c7f8e94f258` | Docker Hub registry manifest for `library/debian:13.6-slim` | +| Go 1.26.5 Bookworm build image | `sha256:53eeac89074db483fdf0ab3be1df32bf6e47562263d2d0d6baa7f26acb4957dd` | Docker Hub registry manifest for `library/golang:1.26.5-bookworm` | +| Debian packages | signed snapshot `20260815T000000Z` | `snapshot.debian.org/archive/debian/20260815T000000Z/dists/trixie/InRelease` | +| Build timestamps | `SOURCE_DATE_EPOCH=1786752000` | BuildKit reproducible-build timestamp input declared in every stage | +| Docker CLI 29.7.2 amd64 / arm64 | exact package version and SHA-256 `3d1a00d1549f7539606252b1e2b88ae2a0c855e02e5fa92ad962c8a118d7f6ad` / `6bb694f01789b9869e99e34df3492104c0150d12df93192034db5f88cf0f7c8c` | official Docker Debian `trixie` package index | +| Compose plugin 5.4.0 amd64 / arm64 | exact package version and SHA-256 `75391b38459edf75a975504903673fdb87a5a4016d689d9b198e2db1ae2796e0` / `81261f286d48c1c9e586065bc52065bbbe3b023705ebdb72377ec465c216dcaf` | official Docker Debian `trixie` package index | +| Actions Runner 2.335.1 | architecture-specific archive hashes in `runner/Dockerfile` | official GitHub Actions Runner release | +| Controller modules | versions plus checksums in `controller/go.mod` and `controller/go.sum` | Go checksum-verified module inputs | + +The Dockerfiles fail closed if an upstream artifact no longer matches. Updating any pin requires a reviewed source change and new managed image IDs. + +## Record image IDs for an engine commit + +From a reviewed engine checkout on an authorized isolated build host, build both images twice from clean BuildKit state and compare IDs: + +```bash +CI_FLEET_COMMIT=$(git rev-parse 'HEAD^{commit}') +CI_FLEET_VERSION=$(git rev-parse --short=12 HEAD) +export CI_FLEET_COMMIT CI_FLEET_VERSION +docker compose -f deploy/compose.yaml build --no-cache runner-image controller +docker image inspect --format '{{.Id}}' \ + "ci-fleet-controller:${CI_FLEET_COMMIT:0:12}" \ + "ci-fleet-runner:${CI_FLEET_COMMIT:0:12}" +``` + +After the repeated IDs match, add one private desired-state entry: + +```json +"managed_images": { + "FULL_ENGINE_COMMIT": { + "controller": "sha256:REVIEWED_CONTROLLER_IMAGE_ID", + "runner": "sha256:REVIEWED_RUNNER_IMAGE_ID" + } +} +``` + +The checked-in template uses conspicuous `111...` / `222...` non-production fixture IDs only so validation and mocked tests are runnable; replace them during initialization. The repository-only marathon does not run a Docker daemon or invent production IDs, so live private configuration remains an external evidence gate. diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index a587cc2..a18c3c7 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -98,7 +98,9 @@ jobs through the Docker socket. --runner-group YOUR-CREATED-RUNNER-GROUP \ --capacity-budget 1 \ --max-runners 1 \ - --engine-ref PINNED_ENGINE_COMMIT + --engine-ref PINNED_ENGINE_COMMIT \ + --controller-image-digest sha256:REVIEWED_CONTROLLER_IMAGE_ID \ + --runner-image-digest sha256:REVIEWED_RUNNER_IMAGE_ID # Validate before committing so a rejected file or accidental value # never enters branch history, then commit the initialized result: ./scripts/validate.sh --strict diff --git a/docs/README.md b/docs/README.md index 90b0499..968cd48 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,7 +21,7 @@ New operator? Use [Operator workflows](OPERATOR-WORKFLOWS.md) to select a suppor | Convert an existing GitHub Actions workflow | [Migrating existing CI](MIGRATING-EXISTING-CI.md) | | Make a project compliant | [Project CI standard](PROJECT-STANDARD.md) and [compliance checklist](COMPLIANCE-CHECKLIST.md) | | Split tests across parallel workers | [Project CI standard](PROJECT-STANDARD.md) and the [parallel workflow example](../examples/workflows/parallel-ci.yml.example) | -| Configure automatic updates and cleanup | [Host maintenance](HOST-MAINTENANCE.md) | +| Configure automatic updates and cleanup | [Host maintenance](HOST-MAINTENANCE.md) and [reproducible managed images](MANAGED-IMAGES.md) | | Monitor hosts and detect missed reports | [Fleet health monitoring](HEALTH-MONITORING.md) and [authenticated status reporting](STATUS-REPORTING.md) | | Handle GitHub App, workflow, or deployment secrets | [Secrets model](SECRETS.md) and [security policy](../SECURITY.md) | | Review accepted implementation scope | [Design decisions](DESIGN-DECISIONS.md) | diff --git a/runner/Dockerfile b/runner/Dockerfile index d428597..e34261b 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -1,23 +1,37 @@ # syntax=docker/dockerfile:1.18 -FROM debian:13.6-slim +FROM debian:13.6-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dcbbee90edb3bb48c7f8e94f258 ARG TARGETARCH +ARG SOURCE_DATE_EPOCH=1786752000 +ARG DEBIAN_SNAPSHOT=20260815T000000Z ARG CI_FLEET_COMMIT=unknown ARG RUNNER_VERSION=2.335.1 ARG RUNNER_SHA256_AMD64=4ef2f25285f0ae4477f1fe1e346db76d2f3ebf03824e2ddd1973a2819bf6c8cf ARG RUNNER_SHA256_ARM64=6d1e85bfd1a506a8b17c1f1b9b57dba458ffed90898799aaa9f599520b0d9207 +ARG DOCKER_CLI_VERSION=29.7.2-1~debian.13~trixie +ARG DOCKER_CLI_SHA256_AMD64=3d1a00d1549f7539606252b1e2b88ae2a0c855e02e5fa92ad962c8a118d7f6ad +ARG DOCKER_CLI_SHA256_ARM64=6bb694f01789b9869e99e34df3492104c0150d12df93192034db5f88cf0f7c8c +ARG COMPOSE_VERSION=5.4.0-1~debian.13~trixie +ARG COMPOSE_SHA256_AMD64=75391b38459edf75a975504903673fdb87a5a4016d689d9b198e2db1ae2796e0 +ARG COMPOSE_SHA256_ARM64=81261f286d48c1c9e586065bc52065bbbe3b023705ebdb72377ec465c216dcaf LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" \ io.randomdevelopment.ci-fleet.managed="true" SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates curl git gnupg jq sudo \ - && install -m 0755 -d /etc/apt/keyrings \ - && curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc \ - && chmod a+r /etc/apt/keyrings/docker.asc \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian trixie stable" > /etc/apt/sources.list.d/docker.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends docker-ce-cli docker-compose-plugin \ +RUN rm -f /etc/apt/sources.list.d/debian.sources \ + && printf 'deb http://snapshot.debian.org/archive/debian/%s trixie main\n' "$DEBIAN_SNAPSHOT" >/etc/apt/sources.list \ + && apt-get -o Acquire::Check-Valid-Until=false update \ + && apt-get install -y --no-install-recommends ca-certificates curl git jq sudo \ + && case "$TARGETARCH" in \ + amd64) docker_sha="$DOCKER_CLI_SHA256_AMD64"; compose_sha="$COMPOSE_SHA256_AMD64" ;; \ + arm64) docker_sha="$DOCKER_CLI_SHA256_ARM64"; compose_sha="$COMPOSE_SHA256_ARM64" ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac \ + && curl -fsSLo docker-cli.deb "https://download.docker.com/linux/debian/dists/trixie/pool/stable/${TARGETARCH}/docker-ce-cli_${DOCKER_CLI_VERSION#*:}_${TARGETARCH}.deb" \ + && curl -fsSLo compose.deb "https://download.docker.com/linux/debian/dists/trixie/pool/stable/${TARGETARCH}/docker-compose-plugin_${COMPOSE_VERSION#*:}_${TARGETARCH}.deb" \ + && printf '%s docker-cli.deb\n%s compose.deb\n' "$docker_sha" "$compose_sha" | sha256sum -c - \ + && apt-get install -y --no-install-recommends ./docker-cli.deb ./compose.deb \ + && rm docker-cli.deb compose.deb \ && rm -rf /var/lib/apt/lists/* RUN useradd --create-home --uid 1001 --shell /bin/bash runner \ diff --git a/scripts/desired_state.py b/scripts/desired_state.py index f19af22..cace686 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -170,6 +170,7 @@ def build_rendered_env( configured_max = controller["max_runners"] effective_max = configured_max if state == "active" else 0 short_commit = engine_commit[:12] + images = config["managed_images"][engine_commit] rendered = { "CI_FLEET_CAPACITY_BUDGET": str(pool["capacity_budget"]), "CI_FLEET_COMMIT": engine_commit, @@ -177,6 +178,7 @@ def build_rendered_env( "CI_FLEET_CONFIG_REF": config_ref, "CI_FLEET_CONFIG_REPOSITORY": config_repository, "CI_FLEET_CONTROLLER_IMAGE": f"ci-fleet-controller:{short_commit}", + "CI_FLEET_CONTROLLER_IMAGE_DIGEST": images["controller"], "CI_FLEET_CONTROLLER_STATE": state, "CI_FLEET_DESIRED_STATE_SCHEMA": "3", "CI_FLEET_DOCKER_GID": str(docker_gid), @@ -189,6 +191,7 @@ def build_rendered_env( "CI_FLEET_RUNNER_CPUS": str(controller["runner_resources"]["cpu_cores"]), "CI_FLEET_RUNNER_GROUP": pool["runner_group"], "CI_FLEET_RUNNER_IMAGE": f"ci-fleet-runner:{short_commit}", + "CI_FLEET_RUNNER_IMAGE_DIGEST": images["runner"], "CI_FLEET_RUNNER_MEMORY_MIB": str(controller["runner_resources"]["memory_mib"]), "CI_FLEET_SCALE_SET_NAME": controller["scale_set_name"], "CI_FLEET_VERSION": short_commit, diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index ad15656..93ebbaa 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -527,13 +527,24 @@ release_matches() { } managed_images_match() { - local image provenance + local environment=${1:-$candidate_env} digest expected_engine image provenance local -a expected_images=() - mapfile -t expected_images < <(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" || $1 == "CI_FLEET_RUNNER_IMAGE" {print substr($0, index($0, "=") + 1)}' "$candidate_env") + expected_engine=$(awk -F= '$1 == "CI_FLEET_ENGINE_REF" {print $2}' "$environment") + [[ "$expected_engine" =~ ^[0-9a-f]{40}$ ]] || return 1 + mapfile -t expected_images < <(awk -F= ' + $1 == "CI_FLEET_CONTROLLER_IMAGE" {controller=$2} + $1 == "CI_FLEET_CONTROLLER_IMAGE_DIGEST" {controller_digest=$2} + $1 == "CI_FLEET_RUNNER_IMAGE" {runner=$2} + $1 == "CI_FLEET_RUNNER_IMAGE_DIGEST" {runner_digest=$2} + END {if (controller && controller_digest && runner && runner_digest) {print controller " " controller_digest; print runner " " runner_digest}} + ' "$environment") [[ ${#expected_images[@]} == 2 ]] || return 1 - for image in "${expected_images[@]}"; do + for expected in "${expected_images[@]}"; do + read -r image digest <<<"$expected" + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || return 1 provenance=$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$image" 2>/dev/null) || return 1 - [[ "$provenance" == "$engine_ref" ]] || return 1 + [[ "$provenance" == "$expected_engine" ]] || return 1 + [[ $(docker image inspect --format '{{.Id}}' "$image" 2>/dev/null) == "$digest" ]] || return 1 done } @@ -672,6 +683,7 @@ build_candidate() { run_candidate_preflight compose "$release_dir" "$candidate_env" config --quiet compose "$release_dir" "$candidate_env" build runner-image controller + managed_images_match "$candidate_env" || die 'managed image digest does not match reviewed desired state' } make_checkpoint() { @@ -983,6 +995,14 @@ restore_checkpoint() { if [[ -n "$release_dir" && -f "$rendered_env" ]]; then restored_state=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_STATE" {print $2}' "$rendered_env") || failed=1 [[ "$restored_state" == active || "$restored_state" == drained || "$restored_state" == disabled ]] || failed=1 + if ((failed == 0)) && grep -Eq '^CI_FLEET_(CONTROLLER|RUNNER)_IMAGE_DIGEST=sha256:[0-9a-f]{64}$' "$rendered_env"; then + if ! managed_images_match "$rendered_env"; then + compose "$release_dir" "$rendered_env" build runner-image controller || failed=1 + if ((failed == 0)); then managed_images_match "$rendered_env" || failed=1; fi + fi + else + note 'ROLLBACK_IMAGE_DIGEST_UNAVAILABLE legacy_checkpoint=true' + fi if [[ "$restored_state" == active && "$failed" == 0 ]]; then if [[ $(managed_runner_count) != 0 ]]; then failed=1 diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 445a77d..4b89e5c 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -67,7 +67,7 @@ case "${1:-}" in exit 1 fi [[ -f "$image_state" ]] || exit 1 - if [[ "$*" == *'{{.Id}}'* ]]; then printf 'sha256:%s\n' "$(<"$image_state")"; else cat "$image_state"; fi + if [[ "$*" == *'org.opencontainers.image.revision'* ]]; then printf '%s\n' "${FAKE_ENGINE_REF:?}"; elif [[ "$*" == *'{{.Id}}'* ]]; then printf 'sha256:%s\n' "$(<"$image_state")"; else cat "$image_state"; fi ;; rm) (($# >= 2)) || exit 1 @@ -100,7 +100,10 @@ case "${1:-}" in fi : >"$state" [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_PROVENANCE_FILE" - [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || printf 'sha256:%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" + if [[ -n "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]]; then + controller_digest=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE_DIGEST" {print $2}' "$env_file") + printf '%s\n' "$controller_digest" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" + fi [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || cp "$env_file" "$FAKE_CONTROLLER_ENV_FILE" if [[ -n "${FAKE_RESTART_AFTER_UP:-}" && -f "$FAKE_RESTART_AFTER_UP" ]]; then rm -f "$FAKE_RESTART_AFTER_UP" @@ -127,8 +130,10 @@ case "${1:-}" in [[ -z "$paused_state" ]] || rm -f "$paused_state" ;; build) - [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_RUNNER_IMAGE_STATE" - [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_STATE" + runner_digest=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE_DIGEST" {print $2}' "$env_file") + controller_digest=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE_DIGEST" {print $2}' "$env_file") + [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${runner_digest#sha256:}" >"$FAKE_RUNNER_IMAGE_STATE" + [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${controller_digest#sha256:}" >"$FAKE_CONTROLLER_IMAGE_STATE" ;; config|logs) ;; *) exit 1 ;; @@ -218,6 +223,8 @@ expect_command_failure() { } engine_ref=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') +controller_image_digest=$(printf '1%.0s' {1..64}) +runner_image_digest=$(printf '2%.0s' {1..64}) export FAKE_ENGINE_REF=$engine_ref runner_image="ci-fleet-runner:${engine_ref:0:12}" export FAKE_RUNNER_IMAGE=$runner_image @@ -249,6 +256,7 @@ value = json.load(open(source, encoding="utf-8")) value["organization"]["slug"] = "fixture-org" value["runner_pools"]["trusted-ci"]["allowed_repositories"] = ["fixture-org/example-app"] value["projects"]["example-app"]["repository"] = "fixture-org/example-app" +value["managed_images"] = {engine_ref: {"controller": "sha256:" + "1" * 64, "runner": "sha256:" + "2" * 64}} controller = value["controllers"]["example-ci-01"] controller["engine_ref"] = engine_ref controller["state"] = state @@ -408,10 +416,10 @@ printf '%040d\n' 0 >"$FAKE_CONTROLLER_PROVENANCE_FILE" expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ $(<"$FAKE_CONTROLLER_PROVENANCE_FILE") == "$engine_ref" ]] || fail 'controller convergence did not restore running image provenance' -printf 'sha256:%040d\n' 0 >"$FAKE_CONTROLLER_IMAGE_ID_FILE" +printf 'sha256:%064d\n' 3 >"$FAKE_CONTROLLER_IMAGE_ID_FILE" expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null -[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "sha256:$engine_ref" ]] || fail 'controller convergence did not restore live image identity' +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "sha256:$controller_image_digest" ]] || fail 'controller convergence did not restore live image identity' python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace("CI_FLEET_MAX_RUNNERS=1", "CI_FLEET_MAX_RUNNERS=9"))' "$FAKE_CONTROLLER_ENV_FILE" grep -Fxq 'CI_FLEET_MAX_RUNNERS=9' "$FAKE_CONTROLLER_ENV_FILE" || fail 'live-environment fixture did not mutate' expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" @@ -427,10 +435,10 @@ rm -f "$FAKE_CONTROLLER_IMAGE_STATE" expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ -f "$FAKE_RUNNER_IMAGE_STATE" && -f "$FAKE_CONTROLLER_IMAGE_STATE" ]] || fail 'candidate build did not restore the controller image' -printf '%040d\n' 0 >"$FAKE_RUNNER_IMAGE_STATE" +printf '%064d\n' 3 >"$FAKE_RUNNER_IMAGE_STATE" expect_failure 'DRIFT managed_images' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null -[[ $(<"$FAKE_RUNNER_IMAGE_STATE") == "$engine_ref" && $(<"$FAKE_CONTROLLER_IMAGE_STATE") == "$engine_ref" ]] || fail 'candidate build did not restore managed image provenance' +[[ $(<"$FAKE_RUNNER_IMAGE_STATE") == "$runner_image_digest" && $(<"$FAKE_CONTROLLER_IMAGE_STATE") == "$controller_image_digest" ]] || fail 'candidate build did not restore reviewed managed image digests' if docker image inspect unrelated:image >/dev/null 2>&1; then fail 'unrelated image fixture unexpectedly exists'; fi : >"$FAKE_IMAGE_INSPECT_LOG" expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null diff --git a/scripts/test_desired_state.py b/scripts/test_desired_state.py index c37a441..2d24212 100755 --- a/scripts/test_desired_state.py +++ b/scripts/test_desired_state.py @@ -54,8 +54,19 @@ def test_active_controller_renders_configured_capacity(self) -> None: self.assertEqual(environment["CI_FLEET_CONFIGURED_MAX_RUNNERS"], "1") self.assertEqual(environment["CI_FLEET_LABELS"], "docker-ci") self.assertEqual(environment["CI_FLEET_COMMIT"], environment["CI_FLEET_ENGINE_REF"]) + self.assertEqual(environment["CI_FLEET_CONTROLLER_IMAGE_DIGEST"], "sha256:" + "1" * 64) + self.assertEqual(environment["CI_FLEET_RUNNER_IMAGE_DIGEST"], "sha256:" + "2" * 64) self.assertEqual(metadata["controller_state"], "active") + def test_requires_reviewed_image_digests_for_engine(self) -> None: + value = config() + value["managed_images"].clear() + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "fleet.json" + path.write_text(json.dumps(value), encoding="utf-8") + with self.assertRaisesRegex(DesiredStateError, "reviewed controller and runner digests"): + load_and_validate_config(path) + def test_status_reporting_requires_fixed_host_local_configuration(self) -> None: value = config() value["controllers"]["example-ci-01"]["status_reporting"] = { diff --git a/templates/config-repository/README.md b/templates/config-repository/README.md index e54b315..9105b58 100644 --- a/templates/config-repository/README.md +++ b/templates/config-repository/README.md @@ -36,7 +36,9 @@ flowchart LR --location primary-site \ --capacity-budget 1 \ --max-runners 1 \ - --engine-ref + --engine-ref REVIEWED_CI_FLEET_COMMIT \ + --controller-image-digest sha256:REVIEWED_CONTROLLER_IMAGE_ID \ + --runner-image-digest sha256:REVIEWED_RUNNER_IMAGE_ID ``` 3. Edit `fleet.json` to add the organization's real logical mappings. diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 4d7acec..d3f2845 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -10,13 +10,24 @@ "runner_pools": { "trusted-ci": { "runner_group": "sample-trusted-ci", - "routing_labels": ["docker-ci"], - "allowed_repositories": ["sample-company/api", "sample-company/web"], + "routing_labels": [ + "docker-ci" + ], + "allowed_repositories": [ + "sample-company/api", + "sample-company/web" + ], "public_repositories": false, "capacity_budget": 6, "job_submission_policy": "all-independent-jobs" } }, + "managed_images": { + "254b98a10ea200e39f19bbe75ab9a822de2d5966": { + "controller": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "runner": "sha256:2222222222222222222222222222222222222222222222222222222222222222" + } + }, "controllers": { "sample-ci-primary": { "pool": "trusted-ci", @@ -66,19 +77,25 @@ "host_group": "development-east", "automatic": true, "requires_approval": false, - "required_secret_names": ["DEPLOY_AUTH"] + "required_secret_names": [ + "DEPLOY_AUTH" + ] }, "staging": { "host_group": "staging-east", "automatic": false, "requires_approval": true, - "required_secret_names": ["DEPLOY_AUTH"] + "required_secret_names": [ + "DEPLOY_AUTH" + ] }, "production": { "host_group": "production-primary", "automatic": false, "requires_approval": true, - "required_secret_names": ["DEPLOY_AUTH"] + "required_secret_names": [ + "DEPLOY_AUTH" + ] } }, "projects": { @@ -97,7 +114,11 @@ "max_job_minutes": 5, "shard_target_minutes": 4 }, - "deployments": ["development", "staging", "production"] + "deployments": [ + "development", + "staging", + "production" + ] }, "web": { "repository": "sample-company/web", @@ -114,7 +135,10 @@ "max_job_minutes": 5, "shard_target_minutes": 4 }, - "deployments": ["development", "production"] + "deployments": [ + "development", + "production" + ] } } } diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index bf3ccea..6108a3f 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -10,13 +10,23 @@ "runner_pools": { "trusted-ci": { "runner_group": "example-trusted-ci", - "routing_labels": ["docker-ci"], - "allowed_repositories": ["example-org/example-app"], + "routing_labels": [ + "docker-ci" + ], + "allowed_repositories": [ + "example-org/example-app" + ], "public_repositories": false, "capacity_budget": 1, "job_submission_policy": "all-independent-jobs" } }, + "managed_images": { + "254b98a10ea200e39f19bbe75ab9a822de2d5966": { + "controller": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "runner": "sha256:2222222222222222222222222222222222222222222222222222222222222222" + } + }, "controllers": { "example-ci-01": { "pool": "trusted-ci", @@ -48,13 +58,17 @@ "host_group": "development-apps", "automatic": true, "requires_approval": false, - "required_secret_names": ["DEPLOY_AUTH"] + "required_secret_names": [ + "DEPLOY_AUTH" + ] }, "production": { "host_group": "production-apps", "automatic": false, "requires_approval": true, - "required_secret_names": ["DEPLOY_AUTH"] + "required_secret_names": [ + "DEPLOY_AUTH" + ] } }, "projects": { @@ -73,7 +87,10 @@ "max_job_minutes": 5, "shard_target_minutes": 4 }, - "deployments": ["development", "production"] + "deployments": [ + "development", + "production" + ] } } } diff --git a/templates/config-repository/fleet.schema.json b/templates/config-repository/fleet.schema.json index c08e792..5d25d1a 100644 --- a/templates/config-repository/fleet.schema.json +++ b/templates/config-repository/fleet.schema.json @@ -4,7 +4,7 @@ "title": "ci-fleet organization configuration", "type": "object", "additionalProperties": false, - "required": ["schema_version", "organization", "runner_pools", "controllers", "host_groups", "environments", "projects"], + "required": ["schema_version", "organization", "runner_pools", "managed_images", "controllers", "host_groups", "environments", "projects"], "properties": { "$schema": {"type": "string"}, "schema_version": {"const": 3}, @@ -32,6 +32,12 @@ "propertyNames": {"$ref": "#/$defs/slug"}, "additionalProperties": {"$ref": "#/$defs/controller"} }, + "managed_images": { + "type": "object", + "minProperties": 1, + "propertyNames": {"type": "string", "pattern": "^(?!0{40}$)[0-9a-f]{40}$"}, + "additionalProperties": {"$ref": "#/$defs/managed_images"} + }, "host_groups": { "type": "object", "minProperties": 1, @@ -54,6 +60,16 @@ "$defs": { "slug": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,62}$"}, "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, + "image_digest": {"type": "string", "pattern": "^sha256:(?!0{64}$)[0-9a-f]{64}$"}, + "managed_images": { + "type": "object", + "additionalProperties": false, + "required": ["controller", "runner"], + "properties": { + "controller": {"$ref": "#/$defs/image_digest"}, + "runner": {"$ref": "#/$defs/image_digest"} + } + }, "runner_pool": { "type": "object", "additionalProperties": false, diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index 3d3ac48..b160f17 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -16,6 +16,7 @@ ORG_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,38}$") SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") +IMAGE_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") @@ -41,6 +42,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--runner-cpu-cores", type=positive_integer, default=2, help="CPU cores available to each runner") parser.add_argument("--runner-memory-mib", type=positive_integer, default=4096, help="memory available to each runner") parser.add_argument("--engine-ref", required=True, help="reviewed full ci-fleet commit SHA") + parser.add_argument("--controller-image-digest", required=True, help="reviewed controller image ID (sha256:...)") + parser.add_argument("--runner-image-digest", required=True, help="reviewed runner image ID (sha256:...)") parser.add_argument("--output", type=Path, default=ROOT / "fleet.json", help="output configuration path") parser.add_argument("--force", action="store_true", help="replace an existing non-example output file") return parser.parse_args() @@ -65,6 +68,9 @@ def main() -> int: fail(f"{option} must be a lowercase slug") if not COMMIT_SHA.fullmatch(args.engine_ref) or args.engine_ref == "0" * 40: fail("--engine-ref must be a nonzero full lowercase commit SHA") + for option, digest in (("--controller-image-digest", args.controller_image_digest), ("--runner-image-digest", args.runner_image_digest)): + if not IMAGE_DIGEST.fullmatch(digest) or digest == "sha256:" + "0" * 64: + fail(f"{option} must be a nonzero sha256 image digest") if args.max_runners > args.capacity_budget: fail("--max-runners must not exceed --capacity-budget") if args.runner_memory_mib < 512: @@ -101,6 +107,12 @@ def main() -> int: "job_submission_policy": "all-independent-jobs", } }, + "managed_images": { + args.engine_ref: { + "controller": args.controller_image_digest, + "runner": args.runner_image_digest, + } + }, "controllers": { args.controller: { "pool": "trusted-ci", diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 77b0ad7..e739dd4 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -59,6 +59,8 @@ def assert_rejected(self, config: dict, expected: str, *, strict: bool = False) def assert_engine_ref_contract(self, value: str, accepted: bool) -> None: config = copy.deepcopy(reference_config()) + images = next(iter(config["managed_images"].values())) + config["managed_images"] = {value: images} first_controller(config)["engine_ref"] = value self.assertEqual(schema_accepts_engine_ref(value), accepted) self.assertEqual(errors_for(config) == [], accepted) @@ -83,7 +85,10 @@ def test_initializer_omits_status_reporting_for_staged_adoption(self) -> None: subprocess.run([ sys.executable, str(ROOT / "scripts" / "init.py"), "--organization", "sample-org", "--project", "sample-app", - "--engine-ref", "1" * 40, "--output", str(output), + "--engine-ref", "1" * 40, + "--controller-image-digest", "sha256:" + "1" * 64, + "--runner-image-digest", "sha256:" + "2" * 64, + "--output", str(output), ], check=True, stdout=subprocess.DEVNULL) controller = first_controller(json.loads(output.read_text())) self.assertNotIn("status_reporting", controller) diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index db70032..903a286 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -16,6 +16,7 @@ ORG_SLUG = re.compile(r"^[a-z0-9][a-z0-9-]{0,38}$") REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") IMAGE = re.compile(r"^[a-z0-9.-]+/[a-z0-9._/-]+$") +IMAGE_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") SECRET_NAME = re.compile(r"^[A-Z][A-Z0-9_]*$") COMMIT_SHA = re.compile(r"^[0-9a-f]{40}$") HIGH_CONFIDENCE_SECRET_PATTERNS = ( @@ -181,6 +182,7 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: "schema_version", "organization", "runner_pools", + "managed_images", "controllers", "host_groups", "environments", @@ -251,6 +253,18 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: pool_capacity[name] = budget validation.require(pool.get("job_submission_policy") == "all-independent-jobs", f"{path}.job_submission_policy", "must submit every independent job and leave capacity control to infrastructure") + managed_images = config.get("managed_images") + if not isinstance(managed_images, dict) or not managed_images: + validation.errors.append("$.managed_images: must be a non-empty object") + managed_images = {} + for engine_ref, images in managed_images.items(): + path = f"$.managed_images.{engine_ref}" + validation.require(isinstance(engine_ref, str) and bool(COMMIT_SHA.fullmatch(engine_ref)) and engine_ref != "0" * 40, path, "key must be a nonzero full lowercase engine commit SHA") + if validation.exact_keys(images, path, {"controller", "runner"}): + for name in ("controller", "runner"): + digest = images.get(name) + validation.require(isinstance(digest, str) and bool(IMAGE_DIGEST.fullmatch(digest)) and digest != "sha256:" + "0" * 64, f"{path}.{name}", "must be a nonzero sha256 image digest") + controllers = config.get("controllers") if not isinstance(controllers, dict) or not controllers: validation.errors.append("$.controllers: must be a non-empty object") @@ -294,6 +308,7 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: scale_sets[scale_set] = name validation.require(lifecycle in {"experimental", "stable", "retiring"}, f"{path}.lifecycle", "must be experimental, stable, or retiring") validation.require(isinstance(engine_ref, str) and bool(COMMIT_SHA.fullmatch(engine_ref)) and engine_ref != "0" * 40, f"{path}.engine_ref", "must be a nonzero full lowercase commit SHA") + validation.require(engine_ref in managed_images, f"{path}.engine_ref", "must have reviewed controller and runner digests in $.managed_images") validation.require(type(minimum) is int and minimum >= 0, f"{path}.min_runners", "must be a non-negative integer") validation.require(type(maximum) is int and maximum > 0, f"{path}.max_runners", "must be a positive integer") if type(minimum) is int and type(maximum) is int: From 65176720af739122eb674dcafe4a06c3851d5a69 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 15 Aug 2026 22:25:07 -0500 Subject: [PATCH 02/16] test: supply managed image digests in CI --- .github/workflows/validate.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 1ea3715..e188545 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -81,6 +81,8 @@ jobs: --project test-app \ --registry BAD \ --engine-ref "$(git rev-parse HEAD)" \ + --controller-image-digest "sha256:$(printf '1%.0s' {1..64})" \ + --runner-image-digest "sha256:$(printf '2%.0s' {1..64})" \ --output "${temporary_directory}/invalid.json"; then echo 'initializer accepted an invalid registry' >&2 exit 1 @@ -90,5 +92,7 @@ jobs: --organization test-company \ --project test-app \ --engine-ref "$(git rev-parse HEAD)" \ + --controller-image-digest "sha256:$(printf '1%.0s' {1..64})" \ + --runner-image-digest "sha256:$(printf '2%.0s' {1..64})" \ --output "${temporary_directory}/fleet.json" test "$(stat -c '%a' "${temporary_directory}/fleet.json")" = 644 From 94ec00ec35cd17c50bf0ddda22a889afa4a63c92 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 01:05:49 -0500 Subject: [PATCH 03/16] fix: bind managed images to architecture --- .github/workflows/validate.yml | 2 + controller/Dockerfile | 5 +- controller/go.mod | 35 ++++++ controller/go.sum | 118 ++++++++++++++++++ docs/MANAGED-IMAGES.md | 12 +- docs/QUICKSTART.md | 1 + scripts/desired_state.py | 11 +- scripts/install-worker-controller.sh | 29 ++++- scripts/test-install-worker-controller.sh | 7 +- scripts/test_desired_state.py | 21 ++++ templates/config-repository/README.md | 1 + .../examples/multi-host/fleet.json | 6 +- templates/config-repository/fleet.json | 6 +- templates/config-repository/fleet.schema.json | 9 ++ templates/config-repository/scripts/init.py | 7 +- .../config-repository/scripts/test_policy.py | 10 ++ .../config-repository/scripts/validate.py | 15 ++- 17 files changed, 277 insertions(+), 18 deletions(-) create mode 100644 controller/go.sum diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e188545..65e62e0 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -81,6 +81,7 @@ jobs: --project test-app \ --registry BAD \ --engine-ref "$(git rev-parse HEAD)" \ + --architecture amd64 \ --controller-image-digest "sha256:$(printf '1%.0s' {1..64})" \ --runner-image-digest "sha256:$(printf '2%.0s' {1..64})" \ --output "${temporary_directory}/invalid.json"; then @@ -92,6 +93,7 @@ jobs: --organization test-company \ --project test-app \ --engine-ref "$(git rev-parse HEAD)" \ + --architecture amd64 \ --controller-image-digest "sha256:$(printf '1%.0s' {1..64})" \ --runner-image-digest "sha256:$(printf '2%.0s' {1..64})" \ --output "${temporary_directory}/fleet.json" diff --git a/controller/Dockerfile b/controller/Dockerfile index 335ff8d..64f3280 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -2,8 +2,9 @@ FROM golang:1.26.5-bookworm@sha256:53eeac89074db483fdf0ab3be1df32bf6e47562263d2d0d6baa7f26acb4957dd AS build ARG SOURCE_DATE_EPOCH=1786752000 WORKDIR /src -COPY go.mod *.go ./ -RUN go mod tidy && go mod verify +COPY go.mod go.sum ./ +RUN go mod download && go mod verify +COPY *.go ./ RUN go test ./... ARG CI_FLEET_VERSION=dev ARG CI_FLEET_COMMIT=unknown diff --git a/controller/go.mod b/controller/go.mod index 387830b..735185d 100644 --- a/controller/go.mod +++ b/controller/go.mod @@ -7,3 +7,38 @@ require ( github.com/docker/docker v28.5.2+incompatible github.com/google/uuid v1.6.0 ) + +require ( + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.7.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sys v0.47.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/controller/go.sum b/controller/go.sum new file mode 100644 index 0000000..f547d24 --- /dev/null +++ b/controller/go.sum @@ -0,0 +1,118 @@ +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/actions/scaleset v0.0.0-20260706130337-9b2803251ede h1:0epHX4I9EGqWQEG0KI+79lFpD+6kHHyYbGKJiYAGM3k= +github.com/actions/scaleset v0.0.0-20260706130337-9b2803251ede/go.mod h1:+Ylz7IYPnOTJd8dZmMziJ7J9HEfZhdoH7iliEWSb/Ms= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a h1:97PfJ4tCxY5C7NzzgGqQEMZmXbISdvSArNNEOoUGKBg= +google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a/go.mod h1:1brfde68Npq6+WA75c1EHWPijZEG1kMus61ygPZfn4A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a h1:qI/YMH1ep2qQtqcp00gMQyoU7mjvbhg88GJKCvfoLj0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/docs/MANAGED-IMAGES.md b/docs/MANAGED-IMAGES.md index 83842f8..332aa95 100644 --- a/docs/MANAGED-IMAGES.md +++ b/docs/MANAGED-IMAGES.md @@ -38,13 +38,19 @@ docker image inspect --format '{{.Id}}' \ "ci-fleet-runner:${CI_FLEET_COMMIT:0:12}" ``` -After the repeated IDs match, add one private desired-state entry: +Repeat the build on every supported host architecture. After each architecture's repeated IDs match, add them to one private desired-state entry: ```json "managed_images": { "FULL_ENGINE_COMMIT": { - "controller": "sha256:REVIEWED_CONTROLLER_IMAGE_ID", - "runner": "sha256:REVIEWED_RUNNER_IMAGE_ID" + "amd64": { + "controller": "sha256:REVIEWED_AMD64_CONTROLLER_IMAGE_ID", + "runner": "sha256:REVIEWED_AMD64_RUNNER_IMAGE_ID" + }, + "arm64": { + "controller": "sha256:REVIEWED_ARM64_CONTROLLER_IMAGE_ID", + "runner": "sha256:REVIEWED_ARM64_RUNNER_IMAGE_ID" + } } } ``` diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index a18c3c7..0bc4dcb 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -99,6 +99,7 @@ jobs through the Docker socket. --capacity-budget 1 \ --max-runners 1 \ --engine-ref PINNED_ENGINE_COMMIT \ + --architecture amd64 \ --controller-image-digest sha256:REVIEWED_CONTROLLER_IMAGE_ID \ --runner-image-digest sha256:REVIEWED_RUNNER_IMAGE_ID # Validate before committing so a rejected file or accidental value diff --git a/scripts/desired_state.py b/scripts/desired_state.py index cace686..8765d4e 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -7,6 +7,7 @@ import importlib.util import json import os +import platform import re import stat import sys @@ -155,6 +156,7 @@ def build_rendered_env( config_repository: str, config_ref: str, docker_gid: int, + machine: str | None = None, engine_capabilities: set[str] | None = None, ) -> tuple[dict[str, str], dict[str, Any]]: controller, pool = select_controller(config, controller_id) @@ -170,7 +172,13 @@ def build_rendered_env( configured_max = controller["max_runners"] effective_max = configured_max if state == "active" else 0 short_commit = engine_commit[:12] - images = config["managed_images"][engine_commit] + architecture = {"x86_64": "amd64", "aarch64": "arm64"}.get(machine or platform.machine()) + if architecture is None: + raise DesiredStateError("host architecture is unsupported for managed images") + try: + images = config["managed_images"][engine_commit][architecture] + except KeyError as error: + raise DesiredStateError(f"reviewed managed image IDs are missing for {architecture}") from error rendered = { "CI_FLEET_CAPACITY_BUDGET": str(pool["capacity_budget"]), "CI_FLEET_COMMIT": engine_commit, @@ -223,6 +231,7 @@ def build_rendered_env( "config_ref": config_ref, "engine_ref": engine_commit, "engine_repository": config["organization"]["delivery_engine"], + "architecture": architecture, "status_reporting_configured": reporting_configured, "status_reporting_required": reporting_required, } diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 93ebbaa..1d38d0b 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -686,6 +686,29 @@ build_candidate() { managed_images_match "$candidate_env" || die 'managed image digest does not match reviewed desired state' } +capture_legacy_image_ids() { + local environment=$1 controller_image runner_image controller_digest runner_digest + controller_digest=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE_DIGEST" {print $2}' "$environment") + runner_digest=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE_DIGEST" {print $2}' "$environment") + if [[ -z "$controller_digest" ]]; then + controller_image=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {print $2}' "$environment") + [[ -n "$controller_image" ]] || die 'legacy checkpoint cannot identify the installed controller image' + controller_digest=$(docker inspect --format '{{.Image}}' "$controller_container" 2>/dev/null \ + || docker image inspect --format '{{.Id}}' "$controller_image" 2>/dev/null) \ + || die 'legacy checkpoint cannot capture the installed controller image ID' + printf 'CI_FLEET_CONTROLLER_IMAGE_DIGEST=%s\n' "$controller_digest" >>"$environment" + fi + if [[ -z "$runner_digest" ]]; then + runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {print $2}' "$environment") + [[ -n "$runner_image" ]] || die 'legacy checkpoint cannot identify the installed runner image' + runner_digest=$(docker image inspect --format '{{.Id}}' "$runner_image" 2>/dev/null) \ + || die 'legacy checkpoint cannot capture the installed runner image ID' + printf 'CI_FLEET_RUNNER_IMAGE_DIGEST=%s\n' "$runner_digest" >>"$environment" + fi + [[ "$controller_digest" =~ ^sha256:[0-9a-f]{64}$ && "$runner_digest" =~ ^sha256:[0-9a-f]{64}$ ]] \ + || die 'legacy checkpoint contains an invalid managed image ID' +} + make_checkpoint() { local timestamp target unit timer final_checkpoint staged_checkpoint timestamp=$(date -u +%Y%m%dT%H%M%SZ) @@ -695,7 +718,10 @@ make_checkpoint() { staging_paths+=("$staged_checkpoint") checkpoint_dir=$staged_checkpoint install -d -m 0700 "$checkpoint_dir/systemd" - [[ ! -f "$rendered_env" ]] || install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env" + [[ ! -f "$rendered_env" ]] || { + install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env" + capture_legacy_image_ids "$checkpoint_dir/ci-fleet.env" + } [[ ! -f "$state_file" ]] || install -m 0600 "$state_file" "$checkpoint_dir/install-state.json" target=$(current_runtime_release) if [[ -n "$target" ]]; then @@ -1002,6 +1028,7 @@ restore_checkpoint() { fi else note 'ROLLBACK_IMAGE_DIGEST_UNAVAILABLE legacy_checkpoint=true' + failed=1 fi if [[ "$restored_state" == active && "$failed" == 0 ]]; then if [[ $(managed_runner_count) != 0 ]]; then diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 4b89e5c..9f6eda7 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -256,7 +256,7 @@ value = json.load(open(source, encoding="utf-8")) value["organization"]["slug"] = "fixture-org" value["runner_pools"]["trusted-ci"]["allowed_repositories"] = ["fixture-org/example-app"] value["projects"]["example-app"]["repository"] = "fixture-org/example-app" -value["managed_images"] = {engine_ref: {"controller": "sha256:" + "1" * 64, "runner": "sha256:" + "2" * 64}} +value["managed_images"] = {engine_ref: {"amd64": {"controller": "sha256:" + "1" * 64, "runner": "sha256:" + "2" * 64}}} controller = value["controllers"]["example-ci-01"] controller["engine_ref"] = engine_ref controller["state"] = state @@ -660,11 +660,14 @@ printf '%s\n' \ "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$adopt_pem" \ 'CI_FLEET_RUNNER_TTL=6h' \ 'CI_FLEET_CONTROLLER_STATE=active' \ + "CI_FLEET_CONTROLLER_IMAGE=$FAKE_CONTROLLER_IMAGE" \ + "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" \ 'CI_FLEET_INSTANCE=legacy-ci-01' >"$adopt_root/etc/ci-fleet/ci-fleet.env" chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" printf 'CI_FLEET_HEALTH_DISK_WARN_PERCENT=75\n' >"$adopt_root/etc/ci-fleet/monitoring.env" chmod 600 "$adopt_root/etc/ci-fleet/monitoring.env" : >"$FAKE_DOCKER_STATE" +printf 'sha256:%s\n' "$controller_image_digest" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" chmod 644 "$adopt_root/etc/ci-fleet/ci-fleet.env" expect_failure 'rendered environment must be owned by root with mode 0600' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" @@ -674,6 +677,8 @@ export FAKE_COMPOSE_LOG=$tmp/adopt-compose.log export FAKE_RESTART_AFTER_UP=$tmp/adopt-restart-after-up : >"$FAKE_RESTART_AFTER_UP" expect_failure 'ROLLBACK_RESTORED' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" +grep -Fxq "CI_FLEET_CONTROLLER_IMAGE_DIGEST=sha256:$controller_image_digest" "$adopt_root/etc/ci-fleet/ci-fleet.env" || fail 'legacy checkpoint did not retain the installed controller image ID' +grep -Fxq "CI_FLEET_RUNNER_IMAGE_DIGEST=sha256:$runner_image_digest" "$adopt_root/etc/ci-fleet/ci-fleet.env" || fail 'legacy checkpoint did not retain the installed runner image ID' grep -Fxq 'CI_FLEET_HEALTH_DISK_WARN_PERCENT=75' "$adopt_root/etc/ci-fleet/monitoring.env" || fail 'rollback changed host-local monitoring configuration' unset FAKE_RESTART_AFTER_UP grep -Fq "stop|$adopt_root/etc/ci-fleet/ci-fleet.env|example-ci-01" "$FAKE_COMPOSE_LOG" || fail 'rollback did not drain the candidate with its rendered environment and identity' diff --git a/scripts/test_desired_state.py b/scripts/test_desired_state.py index 2d24212..643185d 100755 --- a/scripts/test_desired_state.py +++ b/scripts/test_desired_state.py @@ -45,6 +45,7 @@ def render(self, value: dict | None = None, capabilities: set[str] | None = None config_repository="example-org/example-fleet-config", config_ref=CONFIG_COMMIT, docker_gid=998, + machine="x86_64", engine_capabilities={"status_reporting_config"} if capabilities is None else capabilities, ) @@ -58,6 +59,26 @@ def test_active_controller_renders_configured_capacity(self) -> None: self.assertEqual(environment["CI_FLEET_RUNNER_IMAGE_DIGEST"], "sha256:" + "2" * 64) self.assertEqual(metadata["controller_state"], "active") + def test_managed_image_ids_are_selected_for_the_host_architecture(self) -> None: + value = config() + images = next(iter(value["managed_images"].values())) + images["arm64"] = { + "controller": "sha256:" + "3" * 64, + "runner": "sha256:" + "4" * 64, + } + environment, _ = build_rendered_env( + value, + "example-ci-01", + host_values(), + config_repository="example-org/example-fleet-config", + config_ref=CONFIG_COMMIT, + docker_gid=998, + machine="aarch64", + engine_capabilities={"status_reporting_config"}, + ) + self.assertEqual(environment["CI_FLEET_CONTROLLER_IMAGE_DIGEST"], "sha256:" + "3" * 64) + self.assertEqual(environment["CI_FLEET_RUNNER_IMAGE_DIGEST"], "sha256:" + "4" * 64) + def test_requires_reviewed_image_digests_for_engine(self) -> None: value = config() value["managed_images"].clear() diff --git a/templates/config-repository/README.md b/templates/config-repository/README.md index 9105b58..c158b05 100644 --- a/templates/config-repository/README.md +++ b/templates/config-repository/README.md @@ -37,6 +37,7 @@ flowchart LR --capacity-budget 1 \ --max-runners 1 \ --engine-ref REVIEWED_CI_FLEET_COMMIT \ + --architecture amd64 \ --controller-image-digest sha256:REVIEWED_CONTROLLER_IMAGE_ID \ --runner-image-digest sha256:REVIEWED_RUNNER_IMAGE_ID ``` diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index d3f2845..ccfa2fd 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -24,8 +24,10 @@ }, "managed_images": { "254b98a10ea200e39f19bbe75ab9a822de2d5966": { - "controller": "sha256:1111111111111111111111111111111111111111111111111111111111111111", - "runner": "sha256:2222222222222222222222222222222222222222222222222222222222222222" + "amd64": { + "controller": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "runner": "sha256:2222222222222222222222222222222222222222222222222222222222222222" + } } }, "controllers": { diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 6108a3f..784d16a 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -23,8 +23,10 @@ }, "managed_images": { "254b98a10ea200e39f19bbe75ab9a822de2d5966": { - "controller": "sha256:1111111111111111111111111111111111111111111111111111111111111111", - "runner": "sha256:2222222222222222222222222222222222222222222222222222222222222222" + "amd64": { + "controller": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "runner": "sha256:2222222222222222222222222222222222222222222222222222222222222222" + } } }, "controllers": { diff --git a/templates/config-repository/fleet.schema.json b/templates/config-repository/fleet.schema.json index 5d25d1a..fa43af6 100644 --- a/templates/config-repository/fleet.schema.json +++ b/templates/config-repository/fleet.schema.json @@ -62,6 +62,15 @@ "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, "image_digest": {"type": "string", "pattern": "^sha256:(?!0{64}$)[0-9a-f]{64}$"}, "managed_images": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "amd64": {"$ref": "#/$defs/managed_image_ids"}, + "arm64": {"$ref": "#/$defs/managed_image_ids"} + } + }, + "managed_image_ids": { "type": "object", "additionalProperties": false, "required": ["controller", "runner"], diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index b160f17..311194d 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -42,6 +42,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--runner-cpu-cores", type=positive_integer, default=2, help="CPU cores available to each runner") parser.add_argument("--runner-memory-mib", type=positive_integer, default=4096, help="memory available to each runner") parser.add_argument("--engine-ref", required=True, help="reviewed full ci-fleet commit SHA") + parser.add_argument("--architecture", required=True, choices=("amd64", "arm64"), help="architecture of the reviewed image IDs") parser.add_argument("--controller-image-digest", required=True, help="reviewed controller image ID (sha256:...)") parser.add_argument("--runner-image-digest", required=True, help="reviewed runner image ID (sha256:...)") parser.add_argument("--output", type=Path, default=ROOT / "fleet.json", help="output configuration path") @@ -109,8 +110,10 @@ def main() -> int: }, "managed_images": { args.engine_ref: { - "controller": args.controller_image_digest, - "runner": args.runner_image_digest, + args.architecture: { + "controller": args.controller_image_digest, + "runner": args.runner_image_digest, + } } }, "controllers": { diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index e739dd4..f9e37a9 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -86,6 +86,7 @@ def test_initializer_omits_status_reporting_for_staged_adoption(self) -> None: sys.executable, str(ROOT / "scripts" / "init.py"), "--organization", "sample-org", "--project", "sample-app", "--engine-ref", "1" * 40, + "--architecture", "amd64", "--controller-image-digest", "sha256:" + "1" * 64, "--runner-image-digest", "sha256:" + "2" * 64, "--output", str(output), @@ -93,6 +94,15 @@ def test_initializer_omits_status_reporting_for_staged_adoption(self) -> None: controller = first_controller(json.loads(output.read_text())) self.assertNotIn("status_reporting", controller) + def test_managed_image_ids_require_an_architecture_key(self) -> None: + config = copy.deepcopy(reference_config()) + engine_ref = first_controller(config)["engine_ref"] + config["managed_images"][engine_ref] = { + "controller": "sha256:" + "1" * 64, + "runner": "sha256:" + "2" * 64, + } + self.assert_rejected(config, "architecture") + def test_status_reporting_requires_a_separate_engine_rollout(self) -> None: previous = reference_config() current = copy.deepcopy(previous) diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index 903a286..2813bb7 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -260,10 +260,17 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: for engine_ref, images in managed_images.items(): path = f"$.managed_images.{engine_ref}" validation.require(isinstance(engine_ref, str) and bool(COMMIT_SHA.fullmatch(engine_ref)) and engine_ref != "0" * 40, path, "key must be a nonzero full lowercase engine commit SHA") - if validation.exact_keys(images, path, {"controller", "runner"}): - for name in ("controller", "runner"): - digest = images.get(name) - validation.require(isinstance(digest, str) and bool(IMAGE_DIGEST.fullmatch(digest)) and digest != "sha256:" + "0" * 64, f"{path}.{name}", "must be a nonzero sha256 image digest") + if not isinstance(images, dict) or not images: + validation.errors.append(f"{path}: must declare at least one architecture") + continue + unsupported = sorted(set(images) - {"amd64", "arm64"}) + validation.require(not unsupported, path, "contains unsupported architecture keys") + for architecture, identifiers in images.items(): + architecture_path = f"{path}.{architecture}" + if validation.exact_keys(identifiers, architecture_path, {"controller", "runner"}): + for name in ("controller", "runner"): + digest = identifiers.get(name) + validation.require(isinstance(digest, str) and bool(IMAGE_DIGEST.fullmatch(digest)) and digest != "sha256:" + "0" * 64, f"{architecture_path}.{name}", "must be a nonzero sha256 image digest") controllers = config.get("controllers") if not isinstance(controllers, dict) or not controllers: From b3386e19b72d06b25eff92ff310ee86e7e3c2766 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 01:23:34 -0500 Subject: [PATCH 04/16] fix: checkpoint legacy engine revision --- scripts/install-worker-controller.sh | 12 +++++++++--- scripts/test-install-worker-controller.sh | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 1d38d0b..563d52f 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -687,9 +687,15 @@ build_candidate() { } capture_legacy_image_ids() { - local environment=$1 controller_image runner_image controller_digest runner_digest + local environment=$1 controller_image runner_image controller_digest runner_digest engine_ref + engine_ref=$(awk -F= '$1 == "CI_FLEET_ENGINE_REF" {print $2}' "$environment") controller_digest=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE_DIGEST" {print $2}' "$environment") runner_digest=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE_DIGEST" {print $2}' "$environment") + if [[ -z "$engine_ref" ]]; then + engine_ref=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$controller_container" 2>/dev/null) \ + || die 'legacy checkpoint cannot capture the installed engine revision' + printf 'CI_FLEET_ENGINE_REF=%s\n' "$engine_ref" >>"$environment" + fi if [[ -z "$controller_digest" ]]; then controller_image=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {print $2}' "$environment") [[ -n "$controller_image" ]] || die 'legacy checkpoint cannot identify the installed controller image' @@ -705,8 +711,8 @@ capture_legacy_image_ids() { || die 'legacy checkpoint cannot capture the installed runner image ID' printf 'CI_FLEET_RUNNER_IMAGE_DIGEST=%s\n' "$runner_digest" >>"$environment" fi - [[ "$controller_digest" =~ ^sha256:[0-9a-f]{64}$ && "$runner_digest" =~ ^sha256:[0-9a-f]{64}$ ]] \ - || die 'legacy checkpoint contains an invalid managed image ID' + [[ "$engine_ref" =~ ^[0-9a-f]{40}$ && "$controller_digest" =~ ^sha256:[0-9a-f]{64}$ && "$runner_digest" =~ ^sha256:[0-9a-f]{64}$ ]] \ + || die 'legacy checkpoint contains an invalid engine revision or managed image ID' } make_checkpoint() { diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 9f6eda7..cfd6fb8 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -677,6 +677,7 @@ export FAKE_COMPOSE_LOG=$tmp/adopt-compose.log export FAKE_RESTART_AFTER_UP=$tmp/adopt-restart-after-up : >"$FAKE_RESTART_AFTER_UP" expect_failure 'ROLLBACK_RESTORED' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" +grep -Fxq "CI_FLEET_ENGINE_REF=$engine_ref" "$adopt_root/etc/ci-fleet/ci-fleet.env" || fail 'legacy checkpoint did not retain the installed engine revision' grep -Fxq "CI_FLEET_CONTROLLER_IMAGE_DIGEST=sha256:$controller_image_digest" "$adopt_root/etc/ci-fleet/ci-fleet.env" || fail 'legacy checkpoint did not retain the installed controller image ID' grep -Fxq "CI_FLEET_RUNNER_IMAGE_DIGEST=sha256:$runner_image_digest" "$adopt_root/etc/ci-fleet/ci-fleet.env" || fail 'legacy checkpoint did not retain the installed runner image ID' grep -Fxq 'CI_FLEET_HEALTH_DISK_WARN_PERCENT=75' "$adopt_root/etc/ci-fleet/monitoring.env" || fail 'rollback changed host-local monitoring configuration' From fd2573f5b86679c446ac8c3dd5680849eb10e325 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 22:40:23 -0500 Subject: [PATCH 05/16] fix: pass reproducible timestamp to managed builds --- deploy/compose.yaml | 2 ++ docs/MANAGED-IMAGES.md | 3 ++- scripts/test-install-worker-controller.sh | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/deploy/compose.yaml b/deploy/compose.yaml index fe108ae..a7c3a71 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -7,6 +7,7 @@ services: context: ../runner args: CI_FLEET_COMMIT: ${CI_FLEET_COMMIT:-unknown} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:-1786752000} RUNNER_VERSION: ${CI_FLEET_RUNNER_VERSION:-2.335.1} command: ["/bin/true"] deploy: @@ -19,6 +20,7 @@ services: args: CI_FLEET_VERSION: ${CI_FLEET_VERSION:-dev} CI_FLEET_COMMIT: ${CI_FLEET_COMMIT:-unknown} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:-1786752000} restart: unless-stopped user: "0:0" group_add: diff --git a/docs/MANAGED-IMAGES.md b/docs/MANAGED-IMAGES.md index 332aa95..3cef397 100644 --- a/docs/MANAGED-IMAGES.md +++ b/docs/MANAGED-IMAGES.md @@ -31,7 +31,8 @@ From a reviewed engine checkout on an authorized isolated build host, build both ```bash CI_FLEET_COMMIT=$(git rev-parse 'HEAD^{commit}') CI_FLEET_VERSION=$(git rev-parse --short=12 HEAD) -export CI_FLEET_COMMIT CI_FLEET_VERSION +SOURCE_DATE_EPOCH=1786752000 +export CI_FLEET_COMMIT CI_FLEET_VERSION SOURCE_DATE_EPOCH docker compose -f deploy/compose.yaml build --no-cache runner-image controller docker image inspect --format '{{.Id}}' \ "ci-fleet-controller:${CI_FLEET_COMMIT:0:12}" \ diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index cfd6fb8..0bfcded 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -240,6 +240,7 @@ grep -Fq ' user: "0:0"' "$repo_root/deploy/compose.yaml" || fail 'controller grep -Fq 'export PYTHONDONTWRITEBYTECODE=1' "$repo_root/scripts/install-worker-controller.sh" || fail 'managed validation may write Python bytecode into the immutable manager release' grep -Fq ' trap - ERR' "$repo_root/scripts/install-worker-controller.sh" || fail 'warning health subprocess inherits the transactional rollback trap' grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' +[[ $(grep -Fc 'SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:-1786752000}' "$repo_root/deploy/compose.yaml") -eq 2 ]] || fail 'managed builds do not pass the reproducible timestamp to BuildKit' config_repo=$tmp/config-repo git init -q "$config_repo" git -C "$config_repo" config user.name fixture From 42611bc082e96d8e915090d3233f5375e2089bc4 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sun, 16 Aug 2026 22:42:37 -0500 Subject: [PATCH 06/16] test: keep reproducibility assertion shellcheck-clean --- scripts/test-install-worker-controller.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 0bfcded..5001a5d 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -240,7 +240,7 @@ grep -Fq ' user: "0:0"' "$repo_root/deploy/compose.yaml" || fail 'controller grep -Fq 'export PYTHONDONTWRITEBYTECODE=1' "$repo_root/scripts/install-worker-controller.sh" || fail 'managed validation may write Python bytecode into the immutable manager release' grep -Fq ' trap - ERR' "$repo_root/scripts/install-worker-controller.sh" || fail 'warning health subprocess inherits the transactional rollback trap' grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' -[[ $(grep -Fc 'SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:-1786752000}' "$repo_root/deploy/compose.yaml") -eq 2 ]] || fail 'managed builds do not pass the reproducible timestamp to BuildKit' +[[ $(grep -Fc "SOURCE_DATE_EPOCH: \${SOURCE_DATE_EPOCH:-1786752000}" "$repo_root/deploy/compose.yaml") -eq 2 ]] || fail 'managed builds do not pass the reproducible timestamp to BuildKit' config_repo=$tmp/config-repo git init -q "$config_repo" git -C "$config_repo" config user.name fixture From 1af7b2e5357250ee6d614a091e555b909b213e0f Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 00:11:11 -0500 Subject: [PATCH 07/16] fix: normalize managed image filesystem timestamps --- controller/Dockerfile | 6 ++++-- runner/Dockerfile | 10 +++++++--- scripts/test-install-worker-controller.sh | 2 ++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/controller/Dockerfile b/controller/Dockerfile index 64f3280..7044f1a 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -8,7 +8,8 @@ COPY *.go ./ RUN go test ./... ARG CI_FLEET_VERSION=dev ARG CI_FLEET_COMMIT=unknown -RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=${CI_FLEET_VERSION} -X main.commitSHA=${CI_FLEET_COMMIT}" -o /out/ci-fleet-controller . +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=${CI_FLEET_VERSION} -X main.commitSHA=${CI_FLEET_COMMIT}" -o /out/ci-fleet-controller . \ + && touch -d "@${SOURCE_DATE_EPOCH}" /out/ci-fleet-controller FROM debian:13.6-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dcbbee90edb3bb48c7f8e94f258 ARG SOURCE_DATE_EPOCH=1786752000 @@ -21,7 +22,8 @@ RUN rm -f /etc/apt/sources.list.d/debian.sources \ && apt-get -o Acquire::Check-Valid-Until=false update \ && apt-get install -y --no-install-recommends ca-certificates \ && install -d -o 65532 -g 65532 /run/ci-fleet \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log \ + && find / -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + COPY --from=build /out/ci-fleet-controller /usr/local/bin/ci-fleet-controller USER 65532:65532 ENTRYPOINT ["/usr/local/bin/ci-fleet-controller"] diff --git a/runner/Dockerfile b/runner/Dockerfile index e34261b..855ea8c 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -32,10 +32,12 @@ RUN rm -f /etc/apt/sources.list.d/debian.sources \ && printf '%s docker-cli.deb\n%s compose.deb\n' "$docker_sha" "$compose_sha" | sha256sum -c - \ && apt-get install -y --no-install-recommends ./docker-cli.deb ./compose.deb \ && rm docker-cli.deb compose.deb \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log \ + && find / -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + RUN useradd --create-home --uid 1001 --shell /bin/bash runner \ - && install -d -o runner -g runner /home/runner/_work + && install -d -o runner -g runner /home/runner/_work \ + && find / -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + WORKDIR /home/runner RUN case "${TARGETARCH}" in \ @@ -48,7 +50,9 @@ RUN case "${TARGETARCH}" in \ && tar xzf actions-runner.tar.gz \ && rm actions-runner.tar.gz \ && ./bin/installdependencies.sh \ - && chown -R runner:runner /home/runner + && chown -R runner:runner /home/runner \ + && rm -rf /var/log/apt/* /var/log/dpkg.log \ + && find / -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + ENV RUNNER_ALLOW_RUNASROOT=0 USER runner diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 5001a5d..c3d6454 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -241,6 +241,8 @@ grep -Fq 'export PYTHONDONTWRITEBYTECODE=1' "$repo_root/scripts/install-worker-c grep -Fq ' trap - ERR' "$repo_root/scripts/install-worker-controller.sh" || fail 'warning health subprocess inherits the transactional rollback trap' grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' [[ $(grep -Fc "SOURCE_DATE_EPOCH: \${SOURCE_DATE_EPOCH:-1786752000}" "$repo_root/deploy/compose.yaml") -eq 2 ]] || fail 'managed builds do not pass the reproducible timestamp to BuildKit' +grep -Fq "touch -d \"@\${SOURCE_DATE_EPOCH}\" /out/ci-fleet-controller" "$repo_root/controller/Dockerfile" || fail 'controller binary timestamp is not normalized' +[[ $(grep -Fc "find / -xdev -newermt \"@\${SOURCE_DATE_EPOCH}\"" "$repo_root/runner/Dockerfile") -eq 3 ]] || fail 'runner filesystem timestamps are not normalized after every mutating step' config_repo=$tmp/config-repo git init -q "$config_repo" git -C "$config_repo" config user.name fixture From 84d2c701519bc3ff3d34651388a43f32a41dcc54 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 00:17:28 -0500 Subject: [PATCH 08/16] fix: avoid immutable build mounts during normalization --- controller/Dockerfile | 2 +- runner/Dockerfile | 6 +++--- scripts/test-install-worker-controller.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/controller/Dockerfile b/controller/Dockerfile index 7044f1a..f3305a2 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -23,7 +23,7 @@ RUN rm -f /etc/apt/sources.list.d/debian.sources \ && apt-get install -y --no-install-recommends ca-certificates \ && install -d -o 65532 -g 65532 /run/ci-fleet \ && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log \ - && find / -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + && find /etc /home /run /usr /var -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + COPY --from=build /out/ci-fleet-controller /usr/local/bin/ci-fleet-controller USER 65532:65532 ENTRYPOINT ["/usr/local/bin/ci-fleet-controller"] diff --git a/runner/Dockerfile b/runner/Dockerfile index 855ea8c..93cc1f3 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -33,11 +33,11 @@ RUN rm -f /etc/apt/sources.list.d/debian.sources \ && apt-get install -y --no-install-recommends ./docker-cli.deb ./compose.deb \ && rm docker-cli.deb compose.deb \ && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log \ - && find / -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + && find /etc /home /run /usr /var -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + RUN useradd --create-home --uid 1001 --shell /bin/bash runner \ && install -d -o runner -g runner /home/runner/_work \ - && find / -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + && find /etc /home /run /usr /var -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + WORKDIR /home/runner RUN case "${TARGETARCH}" in \ @@ -52,7 +52,7 @@ RUN case "${TARGETARCH}" in \ && ./bin/installdependencies.sh \ && chown -R runner:runner /home/runner \ && rm -rf /var/log/apt/* /var/log/dpkg.log \ - && find / -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + && find /etc /home /run /usr /var -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + ENV RUNNER_ALLOW_RUNASROOT=0 USER runner diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index c3d6454..1356e94 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -242,7 +242,7 @@ grep -Fq ' trap - ERR' "$repo_root/scripts/install-worker-controller.sh" || f grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' [[ $(grep -Fc "SOURCE_DATE_EPOCH: \${SOURCE_DATE_EPOCH:-1786752000}" "$repo_root/deploy/compose.yaml") -eq 2 ]] || fail 'managed builds do not pass the reproducible timestamp to BuildKit' grep -Fq "touch -d \"@\${SOURCE_DATE_EPOCH}\" /out/ci-fleet-controller" "$repo_root/controller/Dockerfile" || fail 'controller binary timestamp is not normalized' -[[ $(grep -Fc "find / -xdev -newermt \"@\${SOURCE_DATE_EPOCH}\"" "$repo_root/runner/Dockerfile") -eq 3 ]] || fail 'runner filesystem timestamps are not normalized after every mutating step' +[[ $(grep -Fc "find /etc /home /run /usr /var -xdev -newermt \"@\${SOURCE_DATE_EPOCH}\"" "$repo_root/runner/Dockerfile") -eq 3 ]] || fail 'runner filesystem timestamps are not normalized after every mutating step' config_repo=$tmp/config-repo git init -q "$config_repo" git -C "$config_repo" config user.name fixture From 376a13c3f3bab78a34ef744c6cded956bae83f4d Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 00:20:53 -0500 Subject: [PATCH 09/16] fix: exclude runtime bind files from normalization --- controller/Dockerfile | 2 +- runner/Dockerfile | 6 +++--- scripts/test-install-worker-controller.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/controller/Dockerfile b/controller/Dockerfile index f3305a2..c884593 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -23,7 +23,7 @@ RUN rm -f /etc/apt/sources.list.d/debian.sources \ && apt-get install -y --no-install-recommends ca-certificates \ && install -d -o 65532 -g 65532 /run/ci-fleet \ && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log \ - && find /etc /home /run /usr /var -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + && find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + COPY --from=build /out/ci-fleet-controller /usr/local/bin/ci-fleet-controller USER 65532:65532 ENTRYPOINT ["/usr/local/bin/ci-fleet-controller"] diff --git a/runner/Dockerfile b/runner/Dockerfile index 93cc1f3..435d8d8 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -33,11 +33,11 @@ RUN rm -f /etc/apt/sources.list.d/debian.sources \ && apt-get install -y --no-install-recommends ./docker-cli.deb ./compose.deb \ && rm docker-cli.deb compose.deb \ && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log \ - && find /etc /home /run /usr /var -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + && find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + RUN useradd --create-home --uid 1001 --shell /bin/bash runner \ && install -d -o runner -g runner /home/runner/_work \ - && find /etc /home /run /usr /var -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + && find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + WORKDIR /home/runner RUN case "${TARGETARCH}" in \ @@ -52,7 +52,7 @@ RUN case "${TARGETARCH}" in \ && ./bin/installdependencies.sh \ && chown -R runner:runner /home/runner \ && rm -rf /var/log/apt/* /var/log/dpkg.log \ - && find /etc /home /run /usr /var -xdev -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + && find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + ENV RUNNER_ALLOW_RUNASROOT=0 USER runner diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 1356e94..aa90bf9 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -242,7 +242,7 @@ grep -Fq ' trap - ERR' "$repo_root/scripts/install-worker-controller.sh" || f grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' [[ $(grep -Fc "SOURCE_DATE_EPOCH: \${SOURCE_DATE_EPOCH:-1786752000}" "$repo_root/deploy/compose.yaml") -eq 2 ]] || fail 'managed builds do not pass the reproducible timestamp to BuildKit' grep -Fq "touch -d \"@\${SOURCE_DATE_EPOCH}\" /out/ci-fleet-controller" "$repo_root/controller/Dockerfile" || fail 'controller binary timestamp is not normalized' -[[ $(grep -Fc "find /etc /home /run /usr /var -xdev -newermt \"@\${SOURCE_DATE_EPOCH}\"" "$repo_root/runner/Dockerfile") -eq 3 ]] || fail 'runner filesystem timestamps are not normalized after every mutating step' +[[ $(grep -Fc "find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt \"@\${SOURCE_DATE_EPOCH}\"" "$repo_root/runner/Dockerfile") -eq 3 ]] || fail 'runner filesystem timestamps are not normalized after every mutating step' config_repo=$tmp/config-repo git init -q "$config_repo" git -C "$config_repo" config user.name fixture From 489eca1dbdc7b4f9f8b35dd326546e43df03ebcb Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 01:35:14 -0500 Subject: [PATCH 10/16] fix: normalize controller copy in one runtime layer --- controller/Dockerfile | 5 +++-- scripts/test-install-worker-controller.sh | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/controller/Dockerfile b/controller/Dockerfile index c884593..6ecc1b4 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -17,13 +17,14 @@ ARG DEBIAN_SNAPSHOT=20260815T000000Z ARG CI_FLEET_COMMIT=unknown LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" \ io.randomdevelopment.ci-fleet.managed="true" -RUN rm -f /etc/apt/sources.list.d/debian.sources \ +RUN --mount=type=bind,from=build,source=/out,target=/out \ + rm -f /etc/apt/sources.list.d/debian.sources \ && printf 'deb http://snapshot.debian.org/archive/debian/%s trixie main\n' "$DEBIAN_SNAPSHOT" >/etc/apt/sources.list \ && apt-get -o Acquire::Check-Valid-Until=false update \ && apt-get install -y --no-install-recommends ca-certificates \ + && install -m 0755 /out/ci-fleet-controller /usr/local/bin/ci-fleet-controller \ && install -d -o 65532 -g 65532 /run/ci-fleet \ && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log \ && find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + -COPY --from=build /out/ci-fleet-controller /usr/local/bin/ci-fleet-controller USER 65532:65532 ENTRYPOINT ["/usr/local/bin/ci-fleet-controller"] diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index aa90bf9..dcfd90e 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -242,6 +242,7 @@ grep -Fq ' trap - ERR' "$repo_root/scripts/install-worker-controller.sh" || f grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/compose.yaml" || fail 'runner build lacks engine provenance argument' [[ $(grep -Fc "SOURCE_DATE_EPOCH: \${SOURCE_DATE_EPOCH:-1786752000}" "$repo_root/deploy/compose.yaml") -eq 2 ]] || fail 'managed builds do not pass the reproducible timestamp to BuildKit' grep -Fq "touch -d \"@\${SOURCE_DATE_EPOCH}\" /out/ci-fleet-controller" "$repo_root/controller/Dockerfile" || fail 'controller binary timestamp is not normalized' +grep -Fq 'RUN --mount=type=bind,from=build,source=/out,target=/out' "$repo_root/controller/Dockerfile" || fail 'controller binary is not installed in the normalized runtime layer' [[ $(grep -Fc "find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt \"@\${SOURCE_DATE_EPOCH}\"" "$repo_root/runner/Dockerfile") -eq 3 ]] || fail 'runner filesystem timestamps are not normalized after every mutating step' config_repo=$tmp/config-repo git init -q "$config_repo" From 427fa2067c468161ca89b925a106d0362e2c9d5d Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 04:05:26 -0500 Subject: [PATCH 11/16] fix: remove nondeterministic runner linker cache --- runner/Dockerfile | 4 ++-- scripts/test-install-worker-controller.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/runner/Dockerfile b/runner/Dockerfile index 435d8d8..9a86fef 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -32,7 +32,7 @@ RUN rm -f /etc/apt/sources.list.d/debian.sources \ && printf '%s docker-cli.deb\n%s compose.deb\n' "$docker_sha" "$compose_sha" | sha256sum -c - \ && apt-get install -y --no-install-recommends ./docker-cli.deb ./compose.deb \ && rm docker-cli.deb compose.deb \ - && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log \ + && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log /var/cache/ldconfig/aux-cache \ && find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + RUN useradd --create-home --uid 1001 --shell /bin/bash runner \ @@ -51,7 +51,7 @@ RUN case "${TARGETARCH}" in \ && rm actions-runner.tar.gz \ && ./bin/installdependencies.sh \ && chown -R runner:runner /home/runner \ - && rm -rf /var/log/apt/* /var/log/dpkg.log \ + && rm -rf /var/log/apt/* /var/log/dpkg.log /var/cache/ldconfig/aux-cache \ && find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + ENV RUNNER_ALLOW_RUNASROOT=0 diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index dcfd90e..e23a6b0 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -244,6 +244,7 @@ grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/comp grep -Fq "touch -d \"@\${SOURCE_DATE_EPOCH}\" /out/ci-fleet-controller" "$repo_root/controller/Dockerfile" || fail 'controller binary timestamp is not normalized' grep -Fq 'RUN --mount=type=bind,from=build,source=/out,target=/out' "$repo_root/controller/Dockerfile" || fail 'controller binary is not installed in the normalized runtime layer' [[ $(grep -Fc "find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt \"@\${SOURCE_DATE_EPOCH}\"" "$repo_root/runner/Dockerfile") -eq 3 ]] || fail 'runner filesystem timestamps are not normalized after every mutating step' +[[ $(grep -Fc '/var/cache/ldconfig/aux-cache' "$repo_root/runner/Dockerfile") -eq 2 ]] || fail 'runner build preserves nondeterministic ldconfig cache content' config_repo=$tmp/config-repo git init -q "$config_repo" git -C "$config_repo" config user.name fixture From 8664d64bd20eebae91454587897bb43bc6eff407 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 06:43:35 -0500 Subject: [PATCH 12/16] fix: pin remaining managed image build inputs --- controller/Dockerfile | 2 +- runner/Dockerfile | 8 +++++--- scripts/test-install-worker-controller.sh | 4 ++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/controller/Dockerfile b/controller/Dockerfile index 6ecc1b4..40f6b3c 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.18 +# syntax=docker/dockerfile:1.18@sha256:dabfc0969b935b2080555ace70ee69a5261af8a8f1b4df97b9e7fbcf6722eddf FROM golang:1.26.5-bookworm@sha256:53eeac89074db483fdf0ab3be1df32bf6e47562263d2d0d6baa7f26acb4957dd AS build ARG SOURCE_DATE_EPOCH=1786752000 WORKDIR /src diff --git a/runner/Dockerfile b/runner/Dockerfile index 9a86fef..44b91b7 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.18 +# syntax=docker/dockerfile:1.18@sha256:dabfc0969b935b2080555ace70ee69a5261af8a8f1b4df97b9e7fbcf6722eddf FROM debian:13.6-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dcbbee90edb3bb48c7f8e94f258 ARG TARGETARCH @@ -20,7 +20,8 @@ LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN rm -f /etc/apt/sources.list.d/debian.sources \ && printf 'deb http://snapshot.debian.org/archive/debian/%s trixie main\n' "$DEBIAN_SNAPSHOT" >/etc/apt/sources.list \ - && apt-get -o Acquire::Check-Valid-Until=false update \ + && printf 'Acquire::Check-Valid-Until "false";\n' >/etc/apt/apt.conf.d/99-ci-fleet-snapshot \ + && apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl git jq sudo \ && case "$TARGETARCH" in \ amd64) docker_sha="$DOCKER_CLI_SHA256_AMD64"; compose_sha="$COMPOSE_SHA256_AMD64" ;; \ @@ -35,7 +36,8 @@ RUN rm -f /etc/apt/sources.list.d/debian.sources \ && rm -rf /var/lib/apt/lists/* /var/log/apt/* /var/log/dpkg.log /var/cache/ldconfig/aux-cache \ && find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + -RUN useradd --create-home --uid 1001 --shell /bin/bash runner \ +RUN useradd --create-home --no-log-init --uid 1001 --shell /bin/bash runner \ + && chage -d "$((SOURCE_DATE_EPOCH / 86400))" runner \ && install -d -o runner -g runner /home/runner/_work \ && find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt "@${SOURCE_DATE_EPOCH}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index e23a6b0..247d370 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -243,6 +243,10 @@ grep -Fq "CI_FLEET_COMMIT: \${CI_FLEET_COMMIT:-unknown}" "$repo_root/deploy/comp [[ $(grep -Fc "SOURCE_DATE_EPOCH: \${SOURCE_DATE_EPOCH:-1786752000}" "$repo_root/deploy/compose.yaml") -eq 2 ]] || fail 'managed builds do not pass the reproducible timestamp to BuildKit' grep -Fq "touch -d \"@\${SOURCE_DATE_EPOCH}\" /out/ci-fleet-controller" "$repo_root/controller/Dockerfile" || fail 'controller binary timestamp is not normalized' grep -Fq 'RUN --mount=type=bind,from=build,source=/out,target=/out' "$repo_root/controller/Dockerfile" || fail 'controller binary is not installed in the normalized runtime layer' +[[ $(grep -Fhc '# syntax=docker/dockerfile:1.18@sha256:dabfc0969b935b2080555ace70ee69a5261af8a8f1b4df97b9e7fbcf6722eddf' "$repo_root/controller/Dockerfile" "$repo_root/runner/Dockerfile" | grep -Fc 1) -eq 2 ]] || fail 'Dockerfile frontend is not digest-pinned' +grep -Fq 'Acquire::Check-Valid-Until "false";' "$repo_root/runner/Dockerfile" || fail 'runner snapshot validity override is not persistent' +grep -Fq 'useradd --create-home --no-log-init' "$repo_root/runner/Dockerfile" || fail 'runner account initialization writes time-dependent login logs' +grep -Fq 'chage -d "$((SOURCE_DATE_EPOCH / 86400))" runner' "$repo_root/runner/Dockerfile" || fail 'runner shadow date is not deterministic' [[ $(grep -Fc "find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt \"@\${SOURCE_DATE_EPOCH}\"" "$repo_root/runner/Dockerfile") -eq 3 ]] || fail 'runner filesystem timestamps are not normalized after every mutating step' [[ $(grep -Fc '/var/cache/ldconfig/aux-cache' "$repo_root/runner/Dockerfile") -eq 2 ]] || fail 'runner build preserves nondeterministic ldconfig cache content' config_repo=$tmp/config-repo From 942f0307d09ae0ecc5e143eea762953a07c3af3f Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 06:46:19 -0500 Subject: [PATCH 13/16] test: keep deterministic shadow assertion literal --- scripts/test-install-worker-controller.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 247d370..b475dd4 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -246,7 +246,7 @@ grep -Fq 'RUN --mount=type=bind,from=build,source=/out,target=/out' "$repo_root/ [[ $(grep -Fhc '# syntax=docker/dockerfile:1.18@sha256:dabfc0969b935b2080555ace70ee69a5261af8a8f1b4df97b9e7fbcf6722eddf' "$repo_root/controller/Dockerfile" "$repo_root/runner/Dockerfile" | grep -Fc 1) -eq 2 ]] || fail 'Dockerfile frontend is not digest-pinned' grep -Fq 'Acquire::Check-Valid-Until "false";' "$repo_root/runner/Dockerfile" || fail 'runner snapshot validity override is not persistent' grep -Fq 'useradd --create-home --no-log-init' "$repo_root/runner/Dockerfile" || fail 'runner account initialization writes time-dependent login logs' -grep -Fq 'chage -d "$((SOURCE_DATE_EPOCH / 86400))" runner' "$repo_root/runner/Dockerfile" || fail 'runner shadow date is not deterministic' +grep -Fq "chage -d \"\$((SOURCE_DATE_EPOCH / 86400))\" runner" "$repo_root/runner/Dockerfile" || fail 'runner shadow date is not deterministic' [[ $(grep -Fc "find /etc /home /run /usr /var -xdev ! -path /etc/hostname ! -path /etc/hosts ! -path /etc/resolv.conf -newermt \"@\${SOURCE_DATE_EPOCH}\"" "$repo_root/runner/Dockerfile") -eq 3 ]] || fail 'runner filesystem timestamps are not normalized after every mutating step' [[ $(grep -Fc '/var/cache/ldconfig/aux-cache' "$repo_root/runner/Dockerfile") -eq 2 ]] || fail 'runner build preserves nondeterministic ldconfig cache content' config_repo=$tmp/config-repo From 1c4dcf3f2e9bbfbca2df87c0c90cbff663cfb98a Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 09:40:16 -0500 Subject: [PATCH 14/16] fix: stage managed image enforcement safely --- docs/MANAGED-IMAGES.md | 37 ++++++++++++++++--- scripts/desired_state.py | 19 +++++++--- scripts/install-worker-controller.sh | 28 ++++++++++++-- scripts/test-install-worker-controller.sh | 13 +++++-- scripts/test_desired_state.py | 9 ++++- templates/config-repository/fleet.schema.json | 2 +- .../config-repository/scripts/test_policy.py | 14 ++++++- .../config-repository/scripts/validate.py | 12 ++++-- 8 files changed, 109 insertions(+), 25 deletions(-) diff --git a/docs/MANAGED-IMAGES.md b/docs/MANAGED-IMAGES.md index 3cef397..215abd0 100644 --- a/docs/MANAGED-IMAGES.md +++ b/docs/MANAGED-IMAGES.md @@ -24,21 +24,48 @@ Reviewed on 2026-08-16 from official upstream endpoints: The Dockerfiles fail closed if an upstream artifact no longer matches. Updating any pin requires a reviewed source change and new managed image IDs. +## Two-step desired-state rollout + +Older schema-v3 engines reject `managed_images` as an unknown top-level key. Upgrade without deadlocking reconciliation: + +1. Merge and reconcile a staging desired-state revision that changes only `engine_ref` and omits `managed_images`. +2. After every target controller reports the new engine revision, merge a second reviewed revision that adds `managed_images` and its exact IDs. + +This engine accepts the omitted key only for that staging transition. Once `managed_images` is present, every controller engine reference must have reviewed IDs; do not combine the two revisions. + ## Record image IDs for an engine commit -From a reviewed engine checkout on an authorized isolated build host, build both images twice from clean BuildKit state and compare IDs: +From a reviewed engine checkout on an authorized isolated build host, build each host architecture twice from clean state using the reviewed BuildKit backend: ```bash CI_FLEET_COMMIT=$(git rev-parse 'HEAD^{commit}') -CI_FLEET_VERSION=$(git rev-parse --short=12 HEAD) +CI_FLEET_VERSION=${CI_FLEET_COMMIT:0:12} SOURCE_DATE_EPOCH=1786752000 +BUILDKIT_IMAGE=moby/buildkit:v0.32.2@sha256:28a898719c18a33f4e8000685287fa36fd0dd9560c6440227d3a732d79bb41d8 +BUILDER=ci-fleet-managed-review export CI_FLEET_COMMIT CI_FLEET_VERSION SOURCE_DATE_EPOCH -docker compose -f deploy/compose.yaml build --no-cache runner-image controller +cleanup() { docker buildx rm --force "$BUILDER" >/dev/null 2>&1 || true; } +trap cleanup EXIT +cleanup +docker buildx create --name "$BUILDER" --driver docker-container \ + --driver-opt "image=$BUILDKIT_IMAGE" --bootstrap + +docker buildx build --builder "$BUILDER" --no-cache --load \ + --provenance=false --sbom=false \ + --build-arg CI_FLEET_COMMIT --build-arg SOURCE_DATE_EPOCH \ + --tag "ci-fleet-runner:$CI_FLEET_VERSION" runner +docker buildx build --builder "$BUILDER" --no-cache --load \ + --provenance=false --sbom=false \ + --build-arg CI_FLEET_COMMIT --build-arg CI_FLEET_VERSION \ + --build-arg SOURCE_DATE_EPOCH \ + --tag "ci-fleet-controller:$CI_FLEET_VERSION" controller docker image inspect --format '{{.Id}}' \ - "ci-fleet-controller:${CI_FLEET_COMMIT:0:12}" \ - "ci-fleet-runner:${CI_FLEET_COMMIT:0:12}" + "ci-fleet-controller:$CI_FLEET_VERSION" \ + "ci-fleet-runner:$CI_FLEET_VERSION" ``` +Managed installation also rejects BuildKit backends older than v0.11, the first release that implements `SOURCE_DATE_EPOCH` output timestamp rewriting. + Repeat the build on every supported host architecture. After each architecture's repeated IDs match, add them to one private desired-state entry: ```json diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 8765d4e..4d5dfff 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -175,10 +175,14 @@ def build_rendered_env( architecture = {"x86_64": "amd64", "aarch64": "arm64"}.get(machine or platform.machine()) if architecture is None: raise DesiredStateError("host architecture is unsupported for managed images") - try: - images = config["managed_images"][engine_commit][architecture] - except KeyError as error: - raise DesiredStateError(f"reviewed managed image IDs are missing for {architecture}") from error + managed_images = config.get("managed_images") + if managed_images is None: + images = None + else: + try: + images = managed_images[engine_commit][architecture] + except KeyError as error: + raise DesiredStateError(f"reviewed managed image IDs are missing for {architecture}") from error rendered = { "CI_FLEET_CAPACITY_BUDGET": str(pool["capacity_budget"]), "CI_FLEET_COMMIT": engine_commit, @@ -186,7 +190,7 @@ def build_rendered_env( "CI_FLEET_CONFIG_REF": config_ref, "CI_FLEET_CONFIG_REPOSITORY": config_repository, "CI_FLEET_CONTROLLER_IMAGE": f"ci-fleet-controller:{short_commit}", - "CI_FLEET_CONTROLLER_IMAGE_DIGEST": images["controller"], + "CI_FLEET_CONTROLLER_STATE": state, "CI_FLEET_DESIRED_STATE_SCHEMA": "3", "CI_FLEET_DOCKER_GID": str(docker_gid), @@ -199,12 +203,15 @@ def build_rendered_env( "CI_FLEET_RUNNER_CPUS": str(controller["runner_resources"]["cpu_cores"]), "CI_FLEET_RUNNER_GROUP": pool["runner_group"], "CI_FLEET_RUNNER_IMAGE": f"ci-fleet-runner:{short_commit}", - "CI_FLEET_RUNNER_IMAGE_DIGEST": images["runner"], + "CI_FLEET_RUNNER_MEMORY_MIB": str(controller["runner_resources"]["memory_mib"]), "CI_FLEET_SCALE_SET_NAME": controller["scale_set_name"], "CI_FLEET_VERSION": short_commit, **validate_host_values(host_values), } + if images is not None: + rendered["CI_FLEET_CONTROLLER_IMAGE_DIGEST"] = images["controller"] + rendered["CI_FLEET_RUNNER_IMAGE_DIGEST"] = images["runner"] reporting_configured = "status_reporting" in controller reporting_required = (controller.get("status_reporting") or {}).get("enabled") is True if reporting_required and REQUIRED_STATUS_CAPABILITY not in (engine_capabilities or set()): diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 563d52f..aa2220a 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -135,12 +135,22 @@ cleanup_temporary() { trap cleanup_temporary EXIT require_commands() { - local command + local command buildkit_version for command in git python3 docker tar install cmp readlink systemctl stat awk grep date flock mktemp; do command -v "$command" >/dev/null || die "$command is required" done docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' docker compose version >/dev/null 2>&1 || die 'Docker Compose v2 is unavailable' + docker buildx version >/dev/null 2>&1 || die 'Docker Buildx is unavailable' + buildkit_version=$(docker buildx inspect --bootstrap 2>/dev/null | awk '$1 == "BuildKit" && $2 == "version:" {sub(/^v/, "", $3); print $3; exit}') + [[ -n "$buildkit_version" ]] || die 'BuildKit backend version is unavailable' + python3 - "$buildkit_version" <<'PY' || die 'BuildKit v0.11 or newer is required' +import re +import sys + +match = re.match(r"^(\d+)\.(\d+)", sys.argv[1]) +raise SystemExit(0 if match and tuple(map(int, match.groups())) >= (0, 11) else 1) +PY } validate_common_arguments() { @@ -548,6 +558,12 @@ managed_images_match() { done } +managed_image_ids_configured() { + local environment=${1:-$candidate_env} + grep -Eq '^CI_FLEET_CONTROLLER_IMAGE_DIGEST=sha256:[0-9a-f]{64}$' "$environment" \ + && grep -Eq '^CI_FLEET_RUNNER_IMAGE_DIGEST=sha256:[0-9a-f]{64}$' "$environment" +} + systemd_matches() { local expected_manager unit expected_manager=$manager_releases/$engine_ref @@ -581,7 +597,9 @@ drift_count() { note 'DRIFT managed_runners' count=$((count + 1)) fi - managed_images_match || { note 'DRIFT managed_images'; count=$((count + 1)); } + if managed_image_ids_configured; then + managed_images_match || { note 'DRIFT managed_images'; count=$((count + 1)); } + fi systemd_matches || { note 'DRIFT maintenance_timers'; count=$((count + 1)); } DRIFT_COUNT=$count } @@ -683,7 +701,11 @@ build_candidate() { run_candidate_preflight compose "$release_dir" "$candidate_env" config --quiet compose "$release_dir" "$candidate_env" build runner-image controller - managed_images_match "$candidate_env" || die 'managed image digest does not match reviewed desired state' + if managed_image_ids_configured "$candidate_env"; then + managed_images_match "$candidate_env" || die 'managed image digest does not match reviewed desired state' + else + note 'MANAGED_IMAGES_STAGING no_reviewed_ids=true' + fi } capture_legacy_image_ids() { diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index b475dd4..bdea717 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -25,6 +25,13 @@ status_file=${FAKE_CONTROLLER_STATUS_FILE:-} paused_state=${FAKE_PAUSED_STATE:-} case "${1:-}" in info) exit 0 ;; + buildx) + case "${2:-}" in + version) printf 'github.com/docker/buildx v0.32.1\n' ;; + inspect) printf 'Name: default\nDriver: docker\nNodes:\nBuildKit version: v0.27.0\n' ;; + *) exit 1 ;; + esac + ;; inspect) [[ -f "$state" ]] || exit 1 if [[ "$*" == *'.Config.Env'* ]]; then @@ -223,8 +230,8 @@ expect_command_failure() { } engine_ref=$(git -C "$repo_root" rev-parse 'HEAD^{commit}') -controller_image_digest=$(printf '1%.0s' {1..64}) -runner_image_digest=$(printf '2%.0s' {1..64}) +controller_image_digest=$(printf '3%.0s' {1..64}) +runner_image_digest=$(printf '4%.0s' {1..64}) export FAKE_ENGINE_REF=$engine_ref runner_image="ci-fleet-runner:${engine_ref:0:12}" export FAKE_RUNNER_IMAGE=$runner_image @@ -265,7 +272,7 @@ value = json.load(open(source, encoding="utf-8")) value["organization"]["slug"] = "fixture-org" value["runner_pools"]["trusted-ci"]["allowed_repositories"] = ["fixture-org/example-app"] value["projects"]["example-app"]["repository"] = "fixture-org/example-app" -value["managed_images"] = {engine_ref: {"amd64": {"controller": "sha256:" + "1" * 64, "runner": "sha256:" + "2" * 64}}} +value["managed_images"] = {engine_ref: {"amd64": {"controller": "sha256:" + "3" * 64, "runner": "sha256:" + "4" * 64}}} controller = value["controllers"]["example-ci-01"] controller["engine_ref"] = engine_ref controller["state"] = state diff --git a/scripts/test_desired_state.py b/scripts/test_desired_state.py index 643185d..c896854 100755 --- a/scripts/test_desired_state.py +++ b/scripts/test_desired_state.py @@ -79,13 +79,20 @@ def test_managed_image_ids_are_selected_for_the_host_architecture(self) -> None: self.assertEqual(environment["CI_FLEET_CONTROLLER_IMAGE_DIGEST"], "sha256:" + "3" * 64) self.assertEqual(environment["CI_FLEET_RUNNER_IMAGE_DIGEST"], "sha256:" + "4" * 64) + def test_staged_engine_upgrade_omits_image_digest_enforcement(self) -> None: + value = config() + value.pop("managed_images") + environment, _ = self.render(value) + self.assertNotIn("CI_FLEET_CONTROLLER_IMAGE_DIGEST", environment) + self.assertNotIn("CI_FLEET_RUNNER_IMAGE_DIGEST", environment) + def test_requires_reviewed_image_digests_for_engine(self) -> None: value = config() value["managed_images"].clear() with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "fleet.json" path.write_text(json.dumps(value), encoding="utf-8") - with self.assertRaisesRegex(DesiredStateError, "reviewed controller and runner digests"): + with self.assertRaisesRegex(DesiredStateError, "non-empty"): load_and_validate_config(path) def test_status_reporting_requires_fixed_host_local_configuration(self) -> None: diff --git a/templates/config-repository/fleet.schema.json b/templates/config-repository/fleet.schema.json index fa43af6..bae8f9f 100644 --- a/templates/config-repository/fleet.schema.json +++ b/templates/config-repository/fleet.schema.json @@ -4,7 +4,7 @@ "title": "ci-fleet organization configuration", "type": "object", "additionalProperties": false, - "required": ["schema_version", "organization", "runner_pools", "managed_images", "controllers", "host_groups", "environments", "projects"], + "required": ["schema_version", "organization", "runner_pools", "controllers", "host_groups", "environments", "projects"], "properties": { "$schema": {"type": "string"}, "schema_version": {"const": 3}, diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index f9e37a9..0a92486 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -74,6 +74,16 @@ def assert_delivery_engine_contract(self, value: str, accepted: bool) -> None: def test_reference_configuration_is_valid(self) -> None: self.assertEqual(errors_for(reference_config()), []) + def test_managed_images_may_be_omitted_for_staged_engine_upgrade(self) -> None: + config = copy.deepcopy(reference_config()) + config.pop("managed_images") + self.assertEqual(errors_for(config), []) + self.assertNotIn("managed_images", contract_schema()["required"]) + + def test_strict_validation_rejects_fixture_image_ids(self) -> None: + config = copy.deepcopy(reference_config()) + self.assert_rejected(config, "replace the fixture image digest", strict=True) + def test_status_reporting_null_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) first_controller(config)["status_reporting"] = None @@ -87,8 +97,8 @@ def test_initializer_omits_status_reporting_for_staged_adoption(self) -> None: "--organization", "sample-org", "--project", "sample-app", "--engine-ref", "1" * 40, "--architecture", "amd64", - "--controller-image-digest", "sha256:" + "1" * 64, - "--runner-image-digest", "sha256:" + "2" * 64, + "--controller-image-digest", "sha256:" + "3" * 64, + "--runner-image-digest", "sha256:" + "4" * 64, "--output", str(output), ], check=True, stdout=subprocess.DEVNULL) controller = first_controller(json.loads(output.read_text())) diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index 2813bb7..c42adab 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -182,13 +182,12 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: "schema_version", "organization", "runner_pools", - "managed_images", "controllers", "host_groups", "environments", "projects", } - if not validation.exact_keys(config, "$", required_top, {"$schema"}): + if not validation.exact_keys(config, "$", required_top, {"$schema", "managed_images"}): return validation.require(config.get("schema_version") == 3, "$.schema_version", "must equal 3") @@ -254,7 +253,9 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: validation.require(pool.get("job_submission_policy") == "all-independent-jobs", f"{path}.job_submission_policy", "must submit every independent job and leave capacity control to infrastructure") managed_images = config.get("managed_images") - if not isinstance(managed_images, dict) or not managed_images: + if managed_images is None: + managed_images = {} + elif not isinstance(managed_images, dict) or not managed_images: validation.errors.append("$.managed_images: must be a non-empty object") managed_images = {} for engine_ref, images in managed_images.items(): @@ -271,6 +272,8 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: for name in ("controller", "runner"): digest = identifiers.get(name) validation.require(isinstance(digest, str) and bool(IMAGE_DIGEST.fullmatch(digest)) and digest != "sha256:" + "0" * 64, f"{architecture_path}.{name}", "must be a nonzero sha256 image digest") + if strict: + validation.require(digest not in {"sha256:" + "1" * 64, "sha256:" + "2" * 64}, f"{architecture_path}.{name}", "replace the fixture image digest before use") controllers = config.get("controllers") if not isinstance(controllers, dict) or not controllers: @@ -315,7 +318,8 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: scale_sets[scale_set] = name validation.require(lifecycle in {"experimental", "stable", "retiring"}, f"{path}.lifecycle", "must be experimental, stable, or retiring") validation.require(isinstance(engine_ref, str) and bool(COMMIT_SHA.fullmatch(engine_ref)) and engine_ref != "0" * 40, f"{path}.engine_ref", "must be a nonzero full lowercase commit SHA") - validation.require(engine_ref in managed_images, f"{path}.engine_ref", "must have reviewed controller and runner digests in $.managed_images") + if managed_images: + validation.require(engine_ref in managed_images, f"{path}.engine_ref", "must have reviewed controller and runner digests in $.managed_images") validation.require(type(minimum) is int and minimum >= 0, f"{path}.min_runners", "must be a non-negative integer") validation.require(type(maximum) is int and maximum > 0, f"{path}.max_runners", "must be a positive integer") if type(minimum) is int and type(maximum) is int: From 9ba44a6068c58b55fcf61731fe51239326b7e58e Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 09:45:05 -0500 Subject: [PATCH 15/16] test: use non-placeholder image IDs in CI --- .github/workflows/validate.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 65e62e0..9648bd2 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -82,8 +82,8 @@ jobs: --registry BAD \ --engine-ref "$(git rev-parse HEAD)" \ --architecture amd64 \ - --controller-image-digest "sha256:$(printf '1%.0s' {1..64})" \ - --runner-image-digest "sha256:$(printf '2%.0s' {1..64})" \ + --controller-image-digest "sha256:$(printf '3%.0s' {1..64})" \ + --runner-image-digest "sha256:$(printf '4%.0s' {1..64})" \ --output "${temporary_directory}/invalid.json"; then echo 'initializer accepted an invalid registry' >&2 exit 1 @@ -94,7 +94,7 @@ jobs: --project test-app \ --engine-ref "$(git rev-parse HEAD)" \ --architecture amd64 \ - --controller-image-digest "sha256:$(printf '1%.0s' {1..64})" \ - --runner-image-digest "sha256:$(printf '2%.0s' {1..64})" \ + --controller-image-digest "sha256:$(printf '3%.0s' {1..64})" \ + --runner-image-digest "sha256:$(printf '4%.0s' {1..64})" \ --output "${temporary_directory}/fleet.json" test "$(stat -c '%a' "${temporary_directory}/fleet.json")" = 644 From 7ca616fb9efe8cc2c3c87102ea20dfe721bf90d7 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Mon, 17 Aug 2026 10:21:24 -0500 Subject: [PATCH 16/16] fix: keep image staging manager-only --- docs/MANAGED-IMAGES.md | 8 +- scripts/install-worker-controller.sh | 74 +++++++++++++------ scripts/test-install-worker-controller.sh | 12 ++- .../config-repository/scripts/test_policy.py | 19 +++++ .../config-repository/scripts/validate.py | 23 ++++++ 5 files changed, 110 insertions(+), 26 deletions(-) diff --git a/docs/MANAGED-IMAGES.md b/docs/MANAGED-IMAGES.md index 215abd0..b678f7a 100644 --- a/docs/MANAGED-IMAGES.md +++ b/docs/MANAGED-IMAGES.md @@ -28,10 +28,10 @@ The Dockerfiles fail closed if an upstream artifact no longer matches. Updating Older schema-v3 engines reject `managed_images` as an unknown top-level key. Upgrade without deadlocking reconciliation: -1. Merge and reconcile a staging desired-state revision that changes only `engine_ref` and omits `managed_images`. -2. After every target controller reports the new engine revision, merge a second reviewed revision that adds `managed_images` and its exact IDs. +1. Merge and reconcile a staging desired-state revision that changes only `engine_ref` and omits `managed_images`. Upgrade mode installs and activates only the new manager; it leaves the running controller and image unchanged. +2. After every target reports `MANAGER_STAGED` for the new engine revision, merge a second reviewed revision that adds `managed_images` and its exact IDs; only this revision may build and activate the new runtime. -This engine accepts the omitted key only for that staging transition. Once `managed_images` is present, every controller engine reference must have reviewed IDs; do not combine the two revisions. +Fresh installation, field removal after enforcement, unrelated staging changes, and runtime activation without reviewed IDs all fail closed. ## Record image IDs for an engine commit @@ -64,7 +64,7 @@ docker image inspect --format '{{.Id}}' \ "ci-fleet-runner:$CI_FLEET_VERSION" ``` -Managed installation also rejects BuildKit backends older than v0.11, the first release that implements `SOURCE_DATE_EPOCH` output timestamp rewriting. +Managed installation creates the same digest-pinned v0.32.2 backend for every rebuild, verifies its reported version exactly, and removes the temporary builder afterward. Repeat the build on every supported host architecture. After each architecture's repeated IDs match, add them to one private desired-state entry: diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index aa2220a..66bef9d 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -123,6 +123,8 @@ timer_names=(ci-fleet-health.timer ci-fleet-cleanup.timer ci-fleet-drift.timer) optional_unit_names=( ci-fleet-reconcile.service ci-fleet-reconcile.timer ) +managed_builder=ci-fleet-managed +managed_buildkit='moby/buildkit:v0.32.2@sha256:28a898719c18a33f4e8000685287fa36fd0dd9560c6440227d3a732d79bb41d8' temporary=$(mktemp -d) cleanup_temporary() { @@ -135,22 +137,13 @@ cleanup_temporary() { trap cleanup_temporary EXIT require_commands() { - local command buildkit_version + local command for command in git python3 docker tar install cmp readlink systemctl stat awk grep date flock mktemp; do command -v "$command" >/dev/null || die "$command is required" done docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' docker compose version >/dev/null 2>&1 || die 'Docker Compose v2 is unavailable' docker buildx version >/dev/null 2>&1 || die 'Docker Buildx is unavailable' - buildkit_version=$(docker buildx inspect --bootstrap 2>/dev/null | awk '$1 == "BuildKit" && $2 == "version:" {sub(/^v/, "", $3); print $3; exit}') - [[ -n "$buildkit_version" ]] || die 'BuildKit backend version is unavailable' - python3 - "$buildkit_version" <<'PY' || die 'BuildKit v0.11 or newer is required' -import re -import sys - -match = re.match(r"^(\d+)\.(\d+)", sys.argv[1]) -raise SystemExit(0 if match and tuple(map(int, match.groups())) >= (0, 11) else 1) -PY } validate_common_arguments() { @@ -353,6 +346,29 @@ compose() { "${clean_environment[@]}" docker compose --project-name ci-fleet --env-file "$env_file" -f "$release/deploy/compose.yaml" "$@" } +build_managed_images() { + local release=$1 environment=$2 + ( + set -a + # shellcheck disable=SC1090 + . "$environment" + set +a + cleanup_builder() { docker buildx rm --force "$managed_builder" >/dev/null 2>&1 || true; } + trap cleanup_builder EXIT + cleanup_builder + docker buildx create --name "$managed_builder" --driver docker-container \ + --driver-opt "image=$managed_buildkit" --bootstrap >/dev/null + [[ $(docker buildx inspect "$managed_builder" | awk '$1 == "BuildKit" && $2 == "version:" {print $3; exit}') == v0.32.2 ]] \ + || die 'reviewed BuildKit backend v0.32.2 is unavailable' + docker buildx build --builder "$managed_builder" --no-cache --load --provenance=false --sbom=false \ + --build-arg CI_FLEET_COMMIT --build-arg SOURCE_DATE_EPOCH \ + --tag "$CI_FLEET_RUNNER_IMAGE" "$release/runner" + docker buildx build --builder "$managed_builder" --no-cache --load --provenance=false --sbom=false \ + --build-arg CI_FLEET_COMMIT --build-arg CI_FLEET_VERSION --build-arg SOURCE_DATE_EPOCH \ + --tag "$CI_FLEET_CONTROLLER_IMAGE" "$release/controller" + ) +} + controller_status() { docker inspect --format '{{.State.Status}}' "$controller_container" 2>/dev/null || true } @@ -559,9 +575,8 @@ managed_images_match() { } managed_image_ids_configured() { - local environment=${1:-$candidate_env} - grep -Eq '^CI_FLEET_CONTROLLER_IMAGE_DIGEST=sha256:[0-9a-f]{64}$' "$environment" \ - && grep -Eq '^CI_FLEET_RUNNER_IMAGE_DIGEST=sha256:[0-9a-f]{64}$' "$environment" + grep -Eq '^CI_FLEET_CONTROLLER_IMAGE_DIGEST=sha256:[0-9a-f]{64}$' "$candidate_env" \ + && grep -Eq '^CI_FLEET_RUNNER_IMAGE_DIGEST=sha256:[0-9a-f]{64}$' "$candidate_env" } systemd_matches() { @@ -700,12 +715,8 @@ run_candidate_preflight() { build_candidate() { run_candidate_preflight compose "$release_dir" "$candidate_env" config --quiet - compose "$release_dir" "$candidate_env" build runner-image controller - if managed_image_ids_configured "$candidate_env"; then - managed_images_match "$candidate_env" || die 'managed image digest does not match reviewed desired state' - else - note 'MANAGED_IMAGES_STAGING no_reviewed_ids=true' - fi + build_managed_images "$release_dir" "$candidate_env" + managed_images_match "$candidate_env" || die 'managed image digest does not match reviewed desired state' } capture_legacy_image_ids() { @@ -1051,7 +1062,7 @@ restore_checkpoint() { [[ "$restored_state" == active || "$restored_state" == drained || "$restored_state" == disabled ]] || failed=1 if ((failed == 0)) && grep -Eq '^CI_FLEET_(CONTROLLER|RUNNER)_IMAGE_DIGEST=sha256:[0-9a-f]{64}$' "$rendered_env"; then if ! managed_images_match "$rendered_env"; then - compose "$release_dir" "$rendered_env" build runner-image controller || failed=1 + build_managed_images "$release_dir" "$rendered_env" || failed=1 if ((failed == 0)); then managed_images_match "$rendered_env" || failed=1; fi fi else @@ -1088,6 +1099,13 @@ trap on_error ERR perform_check() { local count display_last_health + if ! managed_image_ids_configured; then + [[ -L "$manager_current" && $(readlink -f "$manager_current") == "$manager_releases/$engine_ref" ]] \ + || die 'manager-only staging is incomplete' + [[ $(current_runtime_release) != "$release_dir" ]] || die 'manager-only staging activated an unreviewed runtime' + note "CHECK_STAGED manager_ref=$engine_ref runtime_unchanged=true" + return + fi drift_count count=$DRIFT_COUNT if ((count > 0)); then @@ -1098,7 +1116,7 @@ perform_check() { } perform_converge() { - local count existing_status desired_controller_id=$controller_id + local count existing_status installed_engine desired_controller_id=$controller_id if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then die '--upgrade requires an existing managed installation; use --install or --adopt' fi @@ -1109,6 +1127,20 @@ perform_converge() { if [[ "$mode" == install && -f "$rendered_env" && ! -f "$state_file" ]]; then die 'an unmanaged controller configuration exists; use --adopt' fi + if ! managed_image_ids_configured; then + [[ "$mode" == upgrade && -f "$state_file" && -f "$rendered_env" ]] \ + || die 'managed_images may be omitted only for an existing managed engine upgrade' + ! grep -Eq '^CI_FLEET_(CONTROLLER|RUNNER)_IMAGE_DIGEST=sha256:[0-9a-f]{64}$' "$rendered_env" \ + || die 'managed_images cannot be removed after image enforcement' + installed_engine=$(awk -F= '$1 == "CI_FLEET_ENGINE_REF" {print $2}' "$rendered_env") + [[ "$installed_engine" =~ ^[0-9a-f]{40}$ && "$installed_engine" != "$engine_ref" ]] \ + || die 'manager-only staging requires a new engine revision' + install_release + install_manager + install_systemd_units "$(readlink -f "$manager_current")" + note "MANAGER_STAGED engine_ref=$engine_ref runtime_ref=$installed_engine" + return + fi drift_count count=$DRIFT_COUNT if ((count == 0)); then diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index bdea717..6bb2310 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -28,7 +28,17 @@ case "${1:-}" in buildx) case "${2:-}" in version) printf 'github.com/docker/buildx v0.32.1\n' ;; - inspect) printf 'Name: default\nDriver: docker\nNodes:\nBuildKit version: v0.27.0\n' ;; + inspect) printf 'Name: ci-fleet-managed\nDriver: docker-container\nNodes:\nBuildKit version: v0.32.2\n' ;; + create|rm) ;; + build) + if [[ "$*" == *ci-fleet-runner:* ]]; then + [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${CI_FLEET_RUNNER_IMAGE_DIGEST#sha256:}" >"$FAKE_RUNNER_IMAGE_STATE" + elif [[ "$*" == *ci-fleet-controller:* ]]; then + [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${CI_FLEET_CONTROLLER_IMAGE_DIGEST#sha256:}" >"$FAKE_CONTROLLER_IMAGE_STATE" + else + exit 1 + fi + ;; *) exit 1 ;; esac ;; diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 0a92486..e952dfd 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -80,6 +80,25 @@ def test_managed_images_may_be_omitted_for_staged_engine_upgrade(self) -> None: self.assertEqual(errors_for(config), []) self.assertNotIn("managed_images", contract_schema()["required"]) + def test_manager_only_staging_transition_is_engine_ref_only(self) -> None: + previous = copy.deepcopy(reference_config()) + previous.pop("managed_images") + current = copy.deepcopy(previous) + first_controller(current)["engine_ref"] = "3" * 40 + validation = Validation() + validate_transition(previous, current, {}, validation) + self.assertEqual(validation.errors, []) + + current["organization"]["slug"] = "other-org" + validation = Validation() + validate_transition(previous, current, {}, validation) + self.assertTrue(any("may change only" in error for error in validation.errors), validation.errors) + + enforced = copy.deepcopy(reference_config()) + validation = Validation() + validate_transition(enforced, previous, {}, validation) + self.assertTrue(any("cannot be removed" in error for error in validation.errors), validation.errors) + def test_strict_validation_rejects_fixture_image_ids(self) -> None: config = copy.deepcopy(reference_config()) self.assert_rejected(config, "replace the fixture image digest", strict=True) diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index c42adab..4db5a51 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -505,6 +505,29 @@ def validate_transition( new_controllers = current.get("controllers") if not isinstance(old_controllers, dict) or not isinstance(new_controllers, dict): return + if "managed_images" not in current: + validation.require("managed_images" not in previous, "$.managed_images", "cannot be removed after image enforcement") + validation.require( + {key: value for key, value in previous.items() if key != "controllers"} + == {key: value for key, value in current.items() if key != "controllers"}, + "$", + "manager-only staging may change only controller engine_ref values", + ) + validation.require(set(old_controllers) == set(new_controllers), "$.controllers", "manager-only staging cannot add or remove controllers") + changed_engine = False + for name in set(old_controllers) & set(new_controllers): + old = old_controllers[name] + new = new_controllers[name] + if not isinstance(old, dict) or not isinstance(new, dict): + continue + validation.require( + {key: value for key, value in old.items() if key != "engine_ref"} + == {key: value for key, value in new.items() if key != "engine_ref"}, + f"$.controllers.{name}", + "manager-only staging may change only engine_ref", + ) + changed_engine = changed_engine or old.get("engine_ref") != new.get("engine_ref") + validation.require(changed_engine, "$.controllers", "manager-only staging requires an engine_ref change") for name, new in new_controllers.items(): old = old_controllers.get(name) if not isinstance(new, dict):