diff --git a/.github/actions/detect-change-scope/ci-scope-paths.sh b/.github/actions/detect-change-scope/ci-scope-paths.sh index 0a970c7972..39d9eb13d8 100644 --- a/.github/actions/detect-change-scope/ci-scope-paths.sh +++ b/.github/actions/detect-change-scope/ci-scope-paths.sh @@ -47,8 +47,9 @@ package_declared_build_input_changed_files() { package_publish_flow_changed_files() { grep -E \ -e '^\.github/actions/detect-change-scope/(ci-scope-paths|test-ci-scope-paths)\.sh$' \ - -e '^\.github/workflows/(staging-build|prepare-merge|activate-merge-candidate|recover-rejected-merge-candidate|staging-cleanup|force-rebuild|reusable-package-source-publish)\.yml$' \ - -e '^\.github/scripts/(activate-merge-candidate|cleanup-merge-candidates|clone-rejected-merge-candidate|compose-staging-release-snapshots|download-verified-release-asset|fetch-canonical-index|github-api-get|init-merge-candidate|latest-merge-gate-status|mark-merge-candidate-ready|reconcile-merge-candidates|recover-canonical-indexes|require-exact-head-approval|select-package-archive-source|state-lock|test-activate-merge-candidate|test-cleanup-merge-candidates|test-clone-rejected-merge-candidate|test-download-verified-release-asset|test-fetch-canonical-index|test-init-merge-candidate|test-latest-merge-gate-status|test-merge-candidate-workflows|test-reconcile-merge-candidates|test-recover-canonical-indexes|test-require-exact-head-approval|test-select-package-archive-source|test-state-lock|test-validate-staging-release|test-verify-merge-candidate|validate-staging-release|verify-merge-candidate)\.sh$' \ + -e '^\.github/workflows/(staging-build|prepare-merge|activate-merge-candidate|recover-rejected-merge-candidate|staging-cleanup|force-rebuild|reusable-package-source-publish|verify-integration-batch|finalize-integration-batch)\.yml$' \ + -e '^\.github/scripts/(activate-merge-candidate|cleanup-merge-candidates|clone-rejected-merge-candidate|compose-staging-release-snapshots|download-verified-release-asset|fetch-canonical-index|finalize-integration-batch|github-api-get|init-merge-candidate|latest-merge-gate-status|mark-merge-candidate-ready|reconcile-merge-candidates|recover-canonical-indexes|require-exact-head-approval|select-package-archive-source|state-lock|test-activate-merge-candidate|test-cleanup-merge-candidates|test-clone-rejected-merge-candidate|test-download-verified-release-asset|test-fetch-canonical-index|test-finalize-integration-batch|test-init-merge-candidate|test-latest-merge-gate-status|test-merge-candidate-workflows|test-reconcile-merge-candidates|test-recover-canonical-indexes|test-require-exact-head-approval|test-select-package-archive-source|test-state-lock|test-validate-staging-release|test-verify-merge-candidate|validate-staging-release|verify-merge-candidate)\.sh$' \ + -e '^\.github/integration-batches/' \ -e '^tools/xtask/src/(build_index|bundle_program|index_candidate|index_toml|index_update|staging_reuse|update_pkg_manifest)\.rs$' \ -e '^scripts/(compose-initial-index|index-has-current-entry|index-update|prepare-sdk-package|publish-package-source|release-index-state|sync-package-source)\.sh$' \ -e '^tests/scripts/(index-update|package-publish-flow|release-index-state)\.sh$' \ @@ -74,8 +75,9 @@ kernel_runtime_changed_files() { ci_control_changed_files() { grep -E \ - -e '^\.github/workflows/(staging-build|prepare-merge|activate-merge-candidate|recover-rejected-merge-candidate|staging-cleanup|force-rebuild|reusable-package-source-publish)\.yml$' \ - -e '^\.github/scripts/(activate-merge-candidate|cleanup-merge-candidates|clone-rejected-merge-candidate|compose-staging-release-snapshots|download-verified-release-asset|fetch-canonical-index|github-api-get|init-merge-candidate|latest-merge-gate-status|mark-merge-candidate-ready|reconcile-merge-candidates|recover-canonical-indexes|require-exact-head-approval|select-package-archive-source|state-lock|test-activate-merge-candidate|test-cleanup-merge-candidates|test-clone-rejected-merge-candidate|test-download-verified-release-asset|test-fetch-canonical-index|test-init-merge-candidate|test-latest-merge-gate-status|test-merge-candidate-workflows|test-reconcile-merge-candidates|test-recover-canonical-indexes|test-require-exact-head-approval|test-select-package-archive-source|test-state-lock|test-validate-staging-release|test-verify-merge-candidate|validate-staging-release|verify-merge-candidate)\.sh$' \ + -e '^\.github/workflows/(staging-build|prepare-merge|activate-merge-candidate|recover-rejected-merge-candidate|staging-cleanup|force-rebuild|reusable-package-source-publish|verify-integration-batch|finalize-integration-batch)\.yml$' \ + -e '^\.github/scripts/(activate-merge-candidate|cleanup-merge-candidates|clone-rejected-merge-candidate|compose-staging-release-snapshots|download-verified-release-asset|fetch-canonical-index|finalize-integration-batch|github-api-get|init-merge-candidate|latest-merge-gate-status|mark-merge-candidate-ready|reconcile-merge-candidates|recover-canonical-indexes|require-exact-head-approval|select-package-archive-source|state-lock|test-activate-merge-candidate|test-cleanup-merge-candidates|test-clone-rejected-merge-candidate|test-download-verified-release-asset|test-fetch-canonical-index|test-finalize-integration-batch|test-init-merge-candidate|test-latest-merge-gate-status|test-merge-candidate-workflows|test-reconcile-merge-candidates|test-recover-canonical-indexes|test-require-exact-head-approval|test-select-package-archive-source|test-state-lock|test-validate-staging-release|test-verify-merge-candidate|validate-staging-release|verify-merge-candidate)\.sh$' \ + -e '^\.github/integration-batches/' \ -e '^scripts/(compose-initial-index|index-update|release-index-state)\.sh$' \ -e '^tests/scripts/(index-update|package-publish-flow|release-index-state)\.sh$' \ -e '^\.github/actions/detect-change-scope/' \ diff --git a/.github/actions/detect-change-scope/test-ci-scope-paths.sh b/.github/actions/detect-change-scope/test-ci-scope-paths.sh index fae7ea959f..80944fb90c 100755 --- a/.github/actions/detect-change-scope/test-ci-scope-paths.sh +++ b/.github/actions/detect-change-scope/test-ci-scope-paths.sh @@ -114,6 +114,15 @@ assert_matches package_publish_flow_changed_files \ assert_matches package_publish_flow_changed_files \ ".github/scripts/test-activate-merge-candidate.sh" \ ".github/scripts/test-activate-merge-candidate.sh" +assert_matches package_publish_flow_changed_files \ + ".github/scripts/finalize-integration-batch.sh" \ + ".github/scripts/finalize-integration-batch.sh" +assert_matches package_publish_flow_changed_files \ + ".github/scripts/test-finalize-integration-batch.sh" \ + ".github/scripts/test-finalize-integration-batch.sh" +assert_matches package_publish_flow_changed_files \ + ".github/integration-batches/batch-1001.json" \ + ".github/integration-batches/batch-1001.json" assert_matches package_publish_flow_changed_files \ ".github/scripts/clone-rejected-merge-candidate.sh" \ ".github/scripts/clone-rejected-merge-candidate.sh" @@ -201,6 +210,12 @@ assert_matches package_publish_flow_changed_files \ assert_matches package_publish_flow_changed_files \ ".github/workflows/reusable-package-source-publish.yml" \ ".github/workflows/reusable-package-source-publish.yml" +assert_matches package_publish_flow_changed_files \ + ".github/workflows/verify-integration-batch.yml" \ + ".github/workflows/verify-integration-batch.yml" +assert_matches package_publish_flow_changed_files \ + ".github/workflows/finalize-integration-batch.yml" \ + ".github/workflows/finalize-integration-batch.yml" assert_not_matches package_publish_flow_changed_files \ "tools/xtask/src/remote_fetch.rs" \ "tools/xtask/src/remote_fetch.rs" @@ -233,6 +248,18 @@ assert_matches ci_control_changed_files \ assert_matches ci_control_changed_files \ ".github/workflows/recover-rejected-merge-candidate.yml" \ ".github/workflows/recover-rejected-merge-candidate.yml" +assert_matches ci_control_changed_files \ + ".github/workflows/verify-integration-batch.yml" \ + ".github/workflows/verify-integration-batch.yml" +assert_matches ci_control_changed_files \ + ".github/workflows/finalize-integration-batch.yml" \ + ".github/workflows/finalize-integration-batch.yml" +assert_matches ci_control_changed_files \ + ".github/scripts/finalize-integration-batch.sh" \ + ".github/scripts/finalize-integration-batch.sh" +assert_matches ci_control_changed_files \ + ".github/integration-batches/batch-1001.json" \ + ".github/integration-batches/batch-1001.json" assert_matches ci_control_changed_files \ ".github/scripts/activate-merge-candidate.sh" \ ".github/scripts/activate-merge-candidate.sh" diff --git a/.github/integration-batches/README.md b/.github/integration-batches/README.md new file mode 100644 index 0000000000..03f14f6415 --- /dev/null +++ b/.github/integration-batches/README.md @@ -0,0 +1,12 @@ +# Integration Batch Manifests + +Each `batch-.json` file is an immutable receipt for one integration batch. +The batch may contain unrelated fixes. What it must share is one declared +validation treatment whose required suites are the union of every absorbed +source PR's requirements. + +Receipts are append-only. A batch PR adds exactly its own receipt and cannot +change the validation authority that verifies it. + +See [`docs/integration-batches.md`](../../docs/integration-batches.md) for the +manifest schema and maintainer workflow. diff --git a/.github/scripts/finalize-integration-batch.sh b/.github/scripts/finalize-integration-batch.sh new file mode 100755 index 0000000000..3624aaee38 --- /dev/null +++ b/.github/scripts/finalize-integration-batch.sh @@ -0,0 +1,573 @@ +#!/usr/bin/env bash +set -euo pipefail + +MANIFEST="" +BATCH_PR="" +MODE="proposed" +APPLY=false +MAX_COMMIT_PAGES=3 +PER_PAGE=100 +RETRY_DELAY_SECONDS="${INTEGRATION_BATCH_RETRY_DELAY_SECONDS:-2}" + +while [ "$#" -gt 0 ]; do + case "$1" in + --manifest) MANIFEST="$2"; shift 2 ;; + --batch-pr) BATCH_PR="$2"; shift 2 ;; + --mode) MODE="$2"; shift 2 ;; + --apply) APPLY=true; shift ;; + *) echo "finalize-integration-batch: unknown flag $1" >&2; exit 2 ;; + esac +done + +die() { + echo "finalize-integration-batch: $*" >&2 + exit 1 +} + +usage_error() { + echo "finalize-integration-batch: $*" >&2 + exit 2 +} + +is_sha() { + [[ "$1" =~ ^[0-9a-f]{40}$ ]] +} + +if ! [[ "$BATCH_PR" =~ ^[1-9][0-9]*$ ]]; then + usage_error "--batch-pr must be a positive integer" +fi +case "$MODE" in + proposed|finalize) ;; + *) usage_error "--mode must be proposed or finalize" ;; +esac +if [ "$APPLY" = true ] && [ "$MODE" != finalize ]; then + usage_error "--apply is valid only with --mode finalize" +fi +if ! [[ "$RETRY_DELAY_SECONDS" =~ ^[0-9]+$ ]]; then + usage_error "INTEGRATION_BATCH_RETRY_DELAY_SECONDS must be non-negative" +fi + +REPOSITORY="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY required}" +DEFAULT_BRANCH="${GITHUB_DEFAULT_BRANCH:?GITHUB_DEFAULT_BRANCH required}" +EXPECTED_MANIFEST=".github/integration-batches/batch-${BATCH_PR}.json" +if [ "$MANIFEST" != "$EXPECTED_MANIFEST" ]; then + usage_error "manifest for batch PR #$BATCH_PR must be $EXPECTED_MANIFEST" +fi +if ! git check-ref-format "refs/heads/$DEFAULT_BRANCH" >/dev/null 2>&1; then + usage_error "GITHUB_DEFAULT_BRANCH is not a valid branch name" +fi + +REPO_ROOT=$(git rev-parse --show-toplevel) +cd "$REPO_ROOT" +[ -f "$MANIFEST" ] && [ ! -L "$MANIFEST" ] || die "manifest is missing or is a symlink: $MANIFEST" + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=.github/scripts/github-api-get.sh +source "$SCRIPT_DIR/github-api-get.sh" +export GITHUB_API_CONTEXT=finalize-integration-batch +export GITHUB_API_RETRY_DELAY_SECONDS="$RETRY_DELAY_SECONDS" + +TMP_ROOT=$(mktemp -d) +trap 'rm -rf "$TMP_ROOT"' EXIT + +gh_retry() { + local attempt=1 delay="$RETRY_DELAY_SECONDS" + while true; do + if "$@"; then + return 0 + fi + if [ "$attempt" -ge 4 ]; then + return 1 + fi + echo "finalize-integration-batch: GitHub command failed; retrying in ${delay}s: $*" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done +} + +fetch_retry() { + local attempt=1 delay="$RETRY_DELAY_SECONDS" + while true; do + if git fetch --quiet --no-tags origin "$@"; then + return 0 + fi + if [ "$attempt" -ge 4 ]; then + return 1 + fi + echo "finalize-integration-batch: git fetch failed; retrying in ${delay}s" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done +} + +patch_id() { + local commit="$1" output + [ "$(git rev-list --parents -n 1 "$commit" | wc -w | tr -d ' ')" = 2 ] || + die "commit $commit is not a non-merge commit" + output=$(git show --format= --full-index --binary --unified=0 "$commit" | git patch-id --stable) || + die "could not compute patch ID for $commit" + [ "$(printf '%s\n' "$output" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 ] || + die "commit $commit has an empty or ambiguous stable patch ID" + awk '{print $1}' <<<"$output" +} + +require_source_replay_tree() { + local source_commit="$1" batch_commit="$2" + local source_parent batch_parent replay_output replay_tree batch_tree + + source_parent=$(git rev-parse "${source_commit}^") || + die "cannot read parent of source commit $source_commit" + batch_parent=$(git rev-parse "${batch_commit}^") || + die "cannot read parent of batch commit $batch_commit" + replay_output=$(git merge-tree --write-tree --no-messages \ + --merge-base "$source_parent" "$batch_parent" "$source_commit") || + die "source commit $source_commit does not replay cleanly onto parent of batch commit $batch_commit" + [ "$(printf '%s\n' "$replay_output" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 ] || + die "source replay for $source_commit produced an ambiguous tree" + replay_tree=$(printf '%s\n' "$replay_output" | sed '/^$/d') + is_sha "$replay_tree" || die "source replay for $source_commit produced an invalid tree" + git cat-file -e "${replay_tree}^{tree}" 2>/dev/null || + die "source replay for $source_commit did not produce a tree" + batch_tree=$(git rev-parse "${batch_commit}^{tree}") || + die "cannot read tree of batch commit $batch_commit" + [ "$replay_tree" = "$batch_tree" ] || + die "batch commit $batch_commit is not an exact replay of source commit $source_commit" +} + +abi_version_at() { + local commit="$1" versions + versions=$(git show "${commit}:crates/shared/src/lib.rs" 2>/dev/null | + sed -nE 's/^pub const ABI_VERSION: u32 = ([0-9]+);$/\1/p') || + die "cannot read ABI_VERSION at $commit" + [ "$(printf '%s\n' "$versions" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 ] || + die "ABI_VERSION at $commit is missing or ambiguous" + printf '%s\n' "$versions" +} + +list_pr_commits() { + local pr="$1" output="$2" page response count reached_end=false + : > "$output" + for ((page = 1; page <= MAX_COMMIT_PAGES; page++)); do + response=$(gh_retry gh api "/repos/${REPOSITORY}/pulls/${pr}/commits?per_page=${PER_PAGE}&page=${page}") || + die "cannot list commits for PR #$pr" + jq -e 'type == "array" and all(.[]; (keys | index("sha")) != null and + (.sha | type == "string" and test("^[0-9a-f]{40}$")))' <<<"$response" >/dev/null || + die "commit response for PR #$pr is malformed" + jq -r '.[].sha' <<<"$response" >> "$output" + count=$(jq 'length' <<<"$response") + if [ "$count" -lt "$PER_PAGE" ]; then + reached_end=true + break + fi + done + [ "$reached_end" = true ] || die "commit scan for PR #$pr reached the 300-commit safety bound" + [ -s "$output" ] || die "PR #$pr has no commits" +} + +require_latest_merge_gate_success() { + local head_sha="$1" page response count reached_end=false + local statuses_file="$TMP_ROOT/merge-gate-statuses.jsonl" + : > "$statuses_file" + for ((page = 1; page <= 50; page++)); do + response=$(gh_retry gh api "/repos/${REPOSITORY}/commits/${head_sha}/statuses?per_page=100&page=${page}") || + die "cannot read validation statuses for batch head" + jq -e ' + type == "array" and all(.[]; + (.id | type == "number" and . > 0) and + (.context | type == "string") and + (.state | type == "string") and + (.created_at | type == "string" and + test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"))) + ' <<<"$response" >/dev/null || die "validation status page $page is malformed" + count=$(jq 'length' <<<"$response") + jq -c '.[]' <<<"$response" >> "$statuses_file" + if [ "$count" -lt 100 ]; then + reached_end=true + break + fi + done + [ "$reached_end" = true ] || die "validation status scan reached its safety bound" + if [ -s "$statuses_file" ] && + [ -n "$(jq -sr 'group_by(.id)[] | select(length > 1) | .[0].id' "$statuses_file")" ]; then + die "duplicate validation status IDs make pagination uncertain" + fi + jq -es ' + map(select(.context == "merge-gate")) | + sort_by(.created_at, .id) | + (last // null) as $latest | + $latest != null and $latest.state == "success" + ' "$statuses_file" >/dev/null || die "batch head does not have a latest successful merge-gate status" +} + +require_manifest_schema() { + jq -e --argjson batch_pr "$BATCH_PR" --arg default_branch "$DEFAULT_BRANCH" ' + def exact_keys($wanted): (keys | sort) == ($wanted | sort); + def sha: type == "string" and test("^[0-9a-f]{40}$"); + def slug: type == "string" and test("^[a-z0-9]+(?:-[a-z0-9]+)*$"); + def effect: . == "none" or . == "compatible" or . == "breaking"; + def validation_id: + . == "abi-snapshot" or . == "kernel-unit" or . == "fork-instrument" or + . == "host-integration" or . == "browser" or . == "browser-assets" or + . == "libc" or . == "posix" or . == "sortix" or + . == "package-publish" or . == "package-universe" or + . == "homebrew-pilot" or . == "vfs-node" or . == "vfs-browser" or . == "docs"; + def sorted_unique: . == (sort | unique); + def effect_rank: if . == "none" then 0 elif . == "compatible" then 1 else 2 end; + + exact_keys(["schema_version", "batch", "sources"]) and + .schema_version == 1 and + (.batch | exact_keys(["pull_request", "base_ref", "base_sha", "merge_method", "abi", "validation"])) and + .batch.pull_request == $batch_pr and + .batch.base_ref == $default_branch and + (.batch.base_sha | sha) and + .batch.merge_method == "rebase" and + (.batch.abi | exact_keys(["effect", "from_version", "to_version"])) and + (.batch.abi.effect | effect) and + (.batch.abi.from_version | type == "number" and floor == . and . >= 0) and + (.batch.abi.to_version | type == "number" and floor == . and . >= 0) and + (if .batch.abi.effect == "breaking" + then .batch.abi.to_version > .batch.abi.from_version + else .batch.abi.to_version == .batch.abi.from_version end) and + (.batch.validation | exact_keys(["treatment", "required"])) and + (.batch.validation.treatment | slug) and + (.batch.validation.required | type == "array" and length > 0 and sorted_unique and all(.[]; validation_id)) and + (.sources | type == "array" and length > 0) and + (.sources | map(.pull_request) | sorted_unique) and + ([.sources[].head_ref] | length == (unique | length)) and + (.sources | all(.[]; + exact_keys(["pull_request", "head_sha", "head_ref", "abi_effect", "required_validation", "commits"]) and + (.pull_request | type == "number" and floor == . and . > 0 and . != $batch_pr) and + (.head_sha | sha) and (.head_ref | type == "string" and length > 0) and (.head_ref != $default_branch) and + (.abi_effect | effect) and + (.required_validation | type == "array" and length > 0 and sorted_unique and all(.[]; validation_id)) and + (.commits | type == "array" and length > 0) and + (.commits | all(.[]; exact_keys(["source_sha", "batch_sha", "patch_id"]) and + (.source_sha | sha) and (.batch_sha | sha) and (.patch_id | sha))) and + .head_sha == .commits[-1].source_sha + )) and + ([.sources[].commits[].source_sha] | length == (unique | length)) and + ([.sources[].commits[].batch_sha] | length == (unique | length)) and + ([.sources[].commits[].patch_id] | length == (unique | length)) and + (.batch.validation.required == ([.sources[].required_validation[]] | sort | unique)) and + ((.batch.abi.effect | effect_rank) == ([.sources[].abi_effect | effect_rank] | max)) + ' "$MANIFEST" >/dev/null || die "manifest does not satisfy schema version 1" +} + +require_pr_shape() { + local json="$1" pr="$2" + jq -e --argjson pr "$pr" ' + .number == $pr and + (.state == "open" or .state == "closed") and + (.merged | type == "boolean") and + (.merged_at == null or (.merged_at | type == "string")) and + (.merge_commit_sha == null or (.merge_commit_sha | type == "string" and test("^[0-9a-f]{40}$"))) and + (.commits | type == "number" and floor == . and . > 0) and + (.head.sha | type == "string" and test("^[0-9a-f]{40}$")) and + (.head.ref | type == "string" and length > 0) and + (.head.repo.full_name | type == "string" and length > 0) and + (.base.ref | type == "string" and length > 0) + ' <<<"$json" >/dev/null || die "PR #$pr response is malformed" +} + +require_source_pr() { + local source_json="$1" pr="$2" head_sha="$3" head_ref="$4" + require_pr_shape "$source_json" "$pr" + jq -e --arg repo "$REPOSITORY" --arg base "$DEFAULT_BRANCH" \ + --arg sha "$head_sha" --arg ref "$head_ref" ' + .merged == false and .merged_at == null and + .head.repo.full_name == $repo and .base.ref == $base and + .head.sha == $sha and .head.ref == $ref + ' <<<"$source_json" >/dev/null || + die "source PR #$pr is merged, cross-repository, retargeted, or no longer at its recorded head" +} + +inspect_branch() { + local pr="$1" ref="$2" sha="$3" state="$4" output="$5" encoded status + encoded=$(jq -rn --arg value "$ref" '$value | @uri') + set +e + github_api_get_json "/repos/${REPOSITORY}/branches/${encoded}" "$output" + status=$? + set -e + case "$status" in + 0) + jq -e --arg ref "$ref" --arg sha "$sha" ' + .name == $ref and .commit.sha == $sha and .protected == false + ' "$output" >/dev/null || + die "source PR #$pr branch is protected, malformed, or no longer at $sha" + printf 'present\n' + ;; + 44) + if [ "$MODE" = proposed ]; then + die "source PR #$pr branch $ref is unexpectedly absent" + fi + printf 'absent\n' + ;; + *) die "could not determine branch state for source PR #$pr" ;; + esac +} + +require_exclusive_source_branch() { + local pr="$1" ref="$2" state="$3" owner query response expected + owner="${REPOSITORY%%/*}" + query=$(jq -rn --arg value "${owner}:${ref}" '$value | @uri') + response=$(gh_retry gh api "/repos/${REPOSITORY}/pulls?state=open&head=${query}&per_page=100") || + die "cannot determine whether source PR #$pr branch is shared" + jq -e --arg repo "$REPOSITORY" --arg ref "$ref" ' + type == "array" and length < 100 and all(.[]; + (.number | type == "number" and floor == . and . > 0) and + .head.repo.full_name == $repo and .head.ref == $ref) + ' <<<"$response" >/dev/null || die "open-PR response for source branch $ref is malformed or truncated" + if [ "$state" = open ]; then + expected="[$pr]" + else + expected='[]' + fi + [ "$(jq -c '[.[].number] | sort' <<<"$response")" = "$expected" ] || + die "source PR #$pr branch $ref is shared by another open PR" +} + +require_manifest_schema + +repo_json=$(gh_retry gh api "/repos/${REPOSITORY}") || die "cannot read repository metadata" +actual_default=$(jq -er '.default_branch | select(type == "string" and length > 0)' <<<"$repo_json") || + die "repository metadata is malformed" +[ "$actual_default" = "$DEFAULT_BRANCH" ] || die "repository default branch changed to $actual_default" + +BATCH_JSON=$(gh_retry gh api "/repos/${REPOSITORY}/pulls/${BATCH_PR}") || die "cannot read batch PR #$BATCH_PR" +require_pr_shape "$BATCH_JSON" "$BATCH_PR" +BATCH_HEAD=$(jq -r .head.sha <<<"$BATCH_JSON") +BATCH_HEAD_REF=$(jq -r .head.ref <<<"$BATCH_JSON") +BATCH_BASE=$(jq -r .base.ref <<<"$BATCH_JSON") +BATCH_COMMIT_COUNT=$(jq -r .commits <<<"$BATCH_JSON") +RECORDED_BASE_SHA=$(jq -r .batch.base_sha "$MANIFEST") +jq -e --arg repo "$REPOSITORY" --arg base "$DEFAULT_BRANCH" ' + .head.repo.full_name == $repo and .base.ref == $base +' <<<"$BATCH_JSON" >/dev/null || die "batch PR must be a same-repository PR to the default branch" +[ "$BATCH_BASE" = "$DEFAULT_BRANCH" ] || die "batch PR target changed" + +if [ "$MODE" = proposed ]; then + jq -e '.state == "open" and .merged == false and .merged_at == null' <<<"$BATCH_JSON" >/dev/null || + die "proposed validation requires an open, unmerged batch PR" + [ "$(git rev-parse HEAD)" = "$BATCH_HEAD" ] || die "checkout is not the exact batch PR head" + fetch_retry "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}" || + die "cannot fetch current default branch" + [ "$(git rev-parse "refs/remotes/origin/${DEFAULT_BRANCH}")" = "$RECORDED_BASE_SHA" ] || + die "batch is not based on the current default-branch tip" +else + jq -e '.state == "closed" and .merged == true and .merged_at != null and .merge_commit_sha != null' \ + <<<"$BATCH_JSON" >/dev/null || die "finalization requires a merged batch PR" + CHECKED_OUT_SHA=$(git rev-parse HEAD) + fetch_retry "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}" || + die "cannot fetch current default branch" + LIVE_DEFAULT_SHA=$(git rev-parse "refs/remotes/origin/${DEFAULT_BRANCH}") + [ "$CHECKED_OUT_SHA" = "$LIVE_DEFAULT_SHA" ] || + die "finalization must run from the current default-branch tip" +fi + +BATCH_REF="refs/remotes/integration-batch/batch-${BATCH_PR}" +fetch_retry "+refs/pull/${BATCH_PR}/head:${BATCH_REF}" || die "cannot fetch batch PR head" +[ "$(git rev-parse "$BATCH_REF")" = "$BATCH_HEAD" ] || die "batch PR ref disagrees with GitHub API head" + +ORIGINAL_MANIFEST="$TMP_ROOT/original-manifest.json" +git show "${BATCH_HEAD}:${MANIFEST}" > "$ORIGINAL_MANIFEST" 2>/dev/null || + die "batch PR head does not contain $MANIFEST" +cmp -s "$MANIFEST" "$ORIGINAL_MANIFEST" || die "checked-out manifest differs from the batch PR head" + +BATCH_COMMITS="$TMP_ROOT/batch-commits" +list_pr_commits "$BATCH_PR" "$BATCH_COMMITS" +[ "$(wc -l < "$BATCH_COMMITS" | tr -d ' ')" = "$BATCH_COMMIT_COUNT" ] || + die "batch PR commit count disagrees with its commit listing" +[ "$(tail -n 1 "$BATCH_COMMITS")" = "$BATCH_HEAD" ] || die "batch PR commit list does not end at its head" + +declare -A BATCH_PATCH_TO_SHA=() +declare -A BATCH_SHA_SET=() +previous_batch_commit="$RECORDED_BASE_SHA" +git cat-file -e "${RECORDED_BASE_SHA}^{commit}" 2>/dev/null || die "recorded batch base $RECORDED_BASE_SHA is missing" +git merge-base --is-ancestor "$RECORDED_BASE_SHA" "$BATCH_HEAD" || + die "recorded batch base is not an ancestor of the batch head" +[ "$(git merge-base "$RECORDED_BASE_SHA" "$BATCH_HEAD")" = "$RECORDED_BASE_SHA" ] || + die "recorded batch base is not the batch merge-base" + +RECEIPT_CHANGES="$TMP_ROOT/receipt-changes" +EXPECTED_RECEIPT_CHANGE="$TMP_ROOT/expected-receipt-change" +git diff --name-status "$RECORDED_BASE_SHA" "$BATCH_HEAD" -- .github/integration-batches > "$RECEIPT_CHANGES" +printf 'A\t%s\n' "$MANIFEST" > "$EXPECTED_RECEIPT_CHANGE" +cmp -s "$EXPECTED_RECEIPT_CHANGE" "$RECEIPT_CHANGES" || + die "batch must add only its own immutable integration-batch receipt" + +AUTHORITY_CHANGES="$TMP_ROOT/authority-changes" +git diff --name-only "$RECORDED_BASE_SHA" "$BATCH_HEAD" -- \ + .github/actions/detect-change-scope \ + .github/scripts/finalize-integration-batch.sh \ + .github/scripts/github-api-get.sh \ + .github/scripts/test-finalize-integration-batch.sh \ + .github/workflows/finalize-integration-batch.yml \ + .github/workflows/prepare-merge.yml \ + .github/workflows/verify-integration-batch.yml \ + scripts/dev-shell.sh \ + tests/scripts/package-publish-flow.sh \ + > "$AUTHORITY_CHANGES" +if [ -s "$AUTHORITY_CHANGES" ]; then + echo "finalize-integration-batch: manifest-bearing batches cannot change their validation authority:" >&2 + sed 's/^/ /' "$AUTHORITY_CHANGES" >&2 + exit 1 +fi + +while IFS= read -r sha; do + git cat-file -e "${sha}^{commit}" 2>/dev/null || die "batch commit $sha is missing" + [ "$(git rev-parse "${sha}^")" = "$previous_batch_commit" ] || + die "batch commit sequence is not linear from recorded base $RECORDED_BASE_SHA" + id=$(patch_id "$sha") + [ -z "${BATCH_PATCH_TO_SHA[$id]+x}" ] || + die "batch PR has ambiguous stable patch ID $id at ${BATCH_PATCH_TO_SHA[$id]} and $sha" + BATCH_PATCH_TO_SHA[$id]="$sha" + BATCH_SHA_SET[$sha]=1 + previous_batch_commit="$sha" +done < "$BATCH_COMMITS" + +RECORDED_ABI_FROM=$(jq -r .batch.abi.from_version "$MANIFEST") +RECORDED_ABI_TO=$(jq -r .batch.abi.to_version "$MANIFEST") +ACTUAL_ABI_FROM=$(abi_version_at "$RECORDED_BASE_SHA") +ACTUAL_ABI_TO=$(abi_version_at "$BATCH_HEAD") +[ "$RECORDED_ABI_FROM" = "$ACTUAL_ABI_FROM" ] || + die "manifest ABI from_version $RECORDED_ABI_FROM does not match base ABI $ACTUAL_ABI_FROM" +[ "$RECORDED_ABI_TO" = "$ACTUAL_ABI_TO" ] || + die "manifest ABI to_version $RECORDED_ABI_TO does not match batch ABI $ACTUAL_ABI_TO" + +SOURCE_STATE_FILE="$TMP_ROOT/source-state.tsv" +: > "$SOURCE_STATE_FILE" +source_count=$(jq '.sources | length' "$MANIFEST") +for ((source_index = 0; source_index < source_count; source_index++)); do + pr=$(jq -r ".sources[$source_index].pull_request" "$MANIFEST") + head_sha=$(jq -r ".sources[$source_index].head_sha" "$MANIFEST") + head_ref=$(jq -r ".sources[$source_index].head_ref" "$MANIFEST") + [ "$head_ref" != "$BATCH_HEAD_REF" ] || die "source PR #$pr reuses the batch branch" + git check-ref-format "refs/heads/$head_ref" >/dev/null 2>&1 || die "source PR #$pr has an invalid head ref" + + source_json=$(gh_retry gh api "/repos/${REPOSITORY}/pulls/${pr}") || die "cannot read source PR #$pr" + require_source_pr "$source_json" "$pr" "$head_sha" "$head_ref" + state=$(jq -r .state <<<"$source_json") + if [ "$MODE" = proposed ] && [ "$state" != open ]; then + die "proposed batch source PR #$pr is not open" + fi + require_exclusive_source_branch "$pr" "$head_ref" "$state" + + source_ref="refs/remotes/integration-batch/source-${pr}" + fetch_retry "+refs/pull/${pr}/head:${source_ref}" || die "cannot fetch source PR #$pr head" + [ "$(git rev-parse "$source_ref")" = "$head_sha" ] || die "source PR #$pr ref disagrees with its recorded head" + + listed="$TMP_ROOT/source-${pr}-commits" + expected="$TMP_ROOT/source-${pr}-expected" + list_pr_commits "$pr" "$listed" + jq -r ".sources[$source_index].commits[].source_sha" "$MANIFEST" > "$expected" + cmp -s "$listed" "$expected" || die "manifest does not account for every commit in source PR #$pr" + + mapping_count=$(jq ".sources[$source_index].commits | length" "$MANIFEST") + for ((mapping_index = 0; mapping_index < mapping_count; mapping_index++)); do + source_sha=$(jq -r ".sources[$source_index].commits[$mapping_index].source_sha" "$MANIFEST") + batch_sha=$(jq -r ".sources[$source_index].commits[$mapping_index].batch_sha" "$MANIFEST") + recorded_patch=$(jq -r ".sources[$source_index].commits[$mapping_index].patch_id" "$MANIFEST") + [ -n "${BATCH_SHA_SET[$batch_sha]+x}" ] || die "mapped batch commit $batch_sha is not in batch PR #$BATCH_PR" + source_patch=$(patch_id "$source_sha") + batch_patch=$(patch_id "$batch_sha") + [ "$source_patch" = "$recorded_patch" ] || die "source commit $source_sha patch ID changed" + [ "$batch_patch" = "$recorded_patch" ] || die "batch commit $batch_sha is not patch-identical to source commit $source_sha" + require_source_replay_tree "$source_sha" "$batch_sha" + done + + branch_state=$(inspect_branch "$pr" "$head_ref" "$head_sha" "$state" "$TMP_ROOT/branch-${pr}.json") + printf '%s\t%s\t%s\t%s\n' "$pr" "$head_ref" "$head_sha" "$branch_state" >> "$SOURCE_STATE_FILE" +done + +if [ "$MODE" = finalize ]; then + MERGE_SHA=$(jq -r .merge_commit_sha <<<"$BATCH_JSON") + is_sha "$MERGE_SHA" || die "merged batch has no valid merge commit SHA" + git cat-file -e "${MERGE_SHA}^{commit}" 2>/dev/null || die "merged batch commit $MERGE_SHA is missing from default history" + git merge-base --is-ancestor "$MERGE_SHA" "refs/remotes/origin/${DEFAULT_BRANCH}" || + die "batch merge commit is not on current default-branch history" + git rev-list --first-parent "refs/remotes/origin/${DEFAULT_BRANCH}" | grep -Fx "$MERGE_SHA" >/dev/null || + die "batch merge commit is not on the default branch's first-parent history" + [ "$(git rev-list --parents -n 1 "$MERGE_SHA" | wc -w | tr -d ' ')" = 2 ] || + die "batch PR was not landed as a linear rebase sequence" + + SPAN_PARENT=$(git rev-parse "${MERGE_SHA}~${BATCH_COMMIT_COUNT}") || + die "cannot locate the landed batch span" + [ "$SPAN_PARENT" = "$RECORDED_BASE_SHA" ] || + die "landed batch was rebased onto $SPAN_PARENT, not recorded base $RECORDED_BASE_SHA" + LANDED_COMMITS="$TMP_ROOT/landed-commits" + git rev-list --first-parent --reverse "${SPAN_PARENT}..${MERGE_SHA}" > "$LANDED_COMMITS" + [ "$(wc -l < "$LANDED_COMMITS" | tr -d ' ')" = "$BATCH_COMMIT_COUNT" ] || + die "landed batch span has the wrong length" + [ "$(git rev-parse "${BATCH_HEAD}^{tree}")" = "$(git rev-parse "${MERGE_SHA}^{tree}")" ] || + die "landed batch tree differs from the reviewed batch head" + paste "$BATCH_COMMITS" "$LANDED_COMMITS" | while IFS=$'\t' read -r original landed; do + original_patch=$(patch_id "$original") + landed_patch=$(patch_id "$landed") + [ "$original_patch" = "$landed_patch" ] || + die "landed commit $landed is not patch-identical to batch commit $original" + [ "$(git rev-parse "${original}^{tree}")" = "$(git rev-parse "${landed}^{tree}")" ] || + die "landed commit $landed has a different tree from batch commit $original" + done + + LANDED_MANIFEST="$TMP_ROOT/landed-manifest.json" + git show "${MERGE_SHA}:${MANIFEST}" > "$LANDED_MANIFEST" 2>/dev/null || die "landed batch does not contain its manifest" + cmp -s "$MANIFEST" "$LANDED_MANIFEST" || die "default-branch manifest differs from the exact landed batch" + + require_latest_merge_gate_success "$BATCH_HEAD" + + fetch_retry "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}" || + die "cannot recheck current default branch" + [ "$(git rev-parse "refs/remotes/origin/${DEFAULT_BRANCH}")" = "$LIVE_DEFAULT_SHA" ] || + die "default branch advanced during finalization validation" +fi + +echo "Verified batch PR #$BATCH_PR: $source_count source PR(s), $BATCH_COMMIT_COUNT batch commit(s), treatment $(jq -r .batch.validation.treatment "$MANIFEST")." +while IFS=$'\t' read -r pr ref sha branch_state; do + echo "PLAN source PR #$pr: close if open; branch $ref at $sha is $branch_state." +done < "$SOURCE_STATE_FILE" + +if [ "$MODE" != finalize ] || [ "$APPLY" = false ]; then + if [ "$MODE" = finalize ]; then + echo "Dry run only; pass --apply to close source PRs and delete their exact recorded branches." + fi + exit 0 +fi + +while IFS=$'\t' read -r pr ref sha _branch_state; do + current_json=$(gh_retry gh api "/repos/${REPOSITORY}/pulls/${pr}") || die "cannot recheck source PR #$pr" + require_source_pr "$current_json" "$pr" "$sha" "$ref" + state=$(jq -r .state <<<"$current_json") + require_exclusive_source_branch "$pr" "$ref" "$state" + branch_state=$(inspect_branch "$pr" "$ref" "$sha" "$state" "$TMP_ROOT/apply-branch-${pr}.json") + + if [ "$branch_state" = present ]; then + git push --quiet --force-with-lease="refs/heads/${ref}:${sha}" origin ":refs/heads/${ref}" || + die "conditional deletion of source PR #$pr branch $ref failed" + if git ls-remote --exit-code origin "refs/heads/${ref}" > "$TMP_ROOT/remaining-${pr}" 2>/dev/null; then + die "source PR #$pr branch $ref still exists after deletion" + else + ls_status=$? + [ "$ls_status" = 2 ] || die "could not verify deletion of source PR #$pr branch $ref" + fi + echo "Deleted source PR #$pr branch $ref at its recorded head." + fi + + current_json=$(gh_retry gh api "/repos/${REPOSITORY}/pulls/${pr}") || die "cannot recheck source PR #$pr after branch deletion" + require_source_pr "$current_json" "$pr" "$sha" "$ref" + if [ "$(jq -r .state <<<"$current_json")" = open ]; then + closed_json=$(gh_retry gh api --method PATCH "/repos/${REPOSITORY}/pulls/${pr}" -f state=closed) || + die "could not close absorbed source PR #$pr" + require_source_pr "$closed_json" "$pr" "$sha" "$ref" + [ "$(jq -r .state <<<"$closed_json")" = closed ] || die "GitHub did not close source PR #$pr" + echo "Closed absorbed source PR #$pr." + else + echo "Source PR #$pr was already closed." + fi + final_branch_state=$(inspect_branch "$pr" "$ref" "$sha" closed "$TMP_ROOT/final-branch-${pr}.json") + [ "$final_branch_state" = absent ] || + die "source PR #$pr branch $ref reappeared during finalization" +done < "$SOURCE_STATE_FILE" + +echo "Integration batch #$BATCH_PR finalization complete." diff --git a/.github/scripts/test-finalize-integration-batch.sh b/.github/scripts/test-finalize-integration-batch.sh new file mode 100755 index 0000000000..0e7ee1398b --- /dev/null +++ b/.github/scripts/test-finalize-integration-batch.sh @@ -0,0 +1,569 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +FINALIZER="$SCRIPT_DIR/finalize-integration-batch.sh" +WORKFLOW_VERIFY="$SCRIPT_DIR/../workflows/verify-integration-batch.yml" +WORKFLOW_FINALIZE="$SCRIPT_DIR/../workflows/finalize-integration-batch.yml" +TMP_ROOT=$(mktemp -d) +if [ "${KEEP_INTEGRATION_BATCH_FIXTURES:-false}" = true ]; then + trap 'echo "kept integration batch fixtures at $TMP_ROOT" >&2' EXIT +else + trap 'rm -rf "$TMP_ROOT"' EXIT +fi + +FIXTURE_NUMBER=0 + +fixture_patch_id() { + local repo="$1" commit="$2" + git -C "$repo" show --format= --full-index --binary --unified=0 "$commit" | + git patch-id --stable | awk '{print $1}' +} + +make_fixture() { + local variant="${1:-valid}" + FIXTURE_NUMBER=$((FIXTURE_NUMBER + 1)) + FIXTURE="$TMP_ROOT/fixture-$FIXTURE_NUMBER" + REMOTE="$FIXTURE/remote.git" + REPO="$FIXTURE/repo" + DATA="$FIXTURE/data" + BIN="$FIXTURE/bin" + LOG="$FIXTURE/gh.log" + MUTATIONS="$FIXTURE/mutations.log" + mkdir -p "$DATA" "$BIN" + : > "$LOG" + : > "$MUTATIONS" + + git init --quiet --bare "$REMOTE" + git init --quiet -b main "$REPO" + git -C "$REPO" config user.name "Batch Test" + git -C "$REPO" config user.email batch@example.invalid + git -C "$REPO" remote add origin "$REMOTE" + + printf 'base\n' > "$REPO/base.txt" + printf 'first context\nold value\ncontext a\ncontext b\nlast context\nold value\n' \ + > "$REPO/source-one.txt" + mkdir -p "$REPO/crates/shared/src" + mkdir -p "$REPO/.github/integration-batches" "$REPO/.github/workflows" + printf 'pub const ABI_VERSION: u32 = 39;\n' > "$REPO/crates/shared/src/lib.rs" + printf '{"historical":"receipt"}\n' > "$REPO/.github/integration-batches/batch-5.json" + printf 'name: trusted prepare merge\n' > "$REPO/.github/workflows/prepare-merge.yml" + git -C "$REPO" add base.txt source-one.txt crates/shared/src/lib.rs .github + git -C "$REPO" commit --quiet -m base + BASE_SHA=$(git -C "$REPO" rev-parse HEAD) + + git -C "$REPO" checkout --quiet -b fix/source-one + printf 'first context\nnew value\ncontext a\ncontext b\nlast context\nold value\n' \ + > "$REPO/source-one.txt" + git -C "$REPO" add source-one.txt + git -C "$REPO" commit --quiet -m 'kernel: source one' + SOURCE_ONE=$(git -C "$REPO" rev-parse HEAD) + + git -C "$REPO" checkout --quiet -b fix/source-two main + printf 'source two a\n' > "$REPO/source-two-a.txt" + git -C "$REPO" add source-two-a.txt + git -C "$REPO" commit --quiet -m 'host: source two part one' + SOURCE_TWO_A=$(git -C "$REPO" rev-parse HEAD) + printf 'source two b\n' > "$REPO/source-two-b.txt" + git -C "$REPO" add source-two-b.txt + git -C "$REPO" commit --quiet -m 'host: source two part two' + SOURCE_TWO_B=$(git -C "$REPO" rev-parse HEAD) + + git -C "$REPO" checkout --quiet -b integration/batch-10 main + case "$variant" in + context-rebase) + printf 'first context\nold value\ncontext a\ncontext b\nrebased context\nold value\n' \ + > "$REPO/source-one.txt" + git -C "$REPO" commit --quiet -am 'ci: adjust nearby batch context' + git -C "$REPO" cherry-pick --quiet "$SOURCE_ONE" >/dev/null + ;; + moved-delta) + printf 'first context\nold value\ncontext a\ncontext b\nlast context\nnew value\n' \ + > "$REPO/source-one.txt" + git -C "$REPO" commit --quiet -am 'kernel: source one' + ;; + changed-delta) + printf 'first context\na genuinely different value\ncontext a\ncontext b\nlast context\nold value\n' \ + > "$REPO/source-one.txt" + git -C "$REPO" commit --quiet -am 'kernel: source one' + ;; + *) + git -C "$REPO" cherry-pick --quiet "$SOURCE_ONE" >/dev/null + ;; + esac + BATCH_ONE=$(git -C "$REPO" rev-parse HEAD) + git -C "$REPO" cherry-pick --quiet "$SOURCE_TWO_A" >/dev/null + BATCH_TWO_A=$(git -C "$REPO" rev-parse HEAD) + git -C "$REPO" cherry-pick --quiet "$SOURCE_TWO_B" >/dev/null + BATCH_TWO_B=$(git -C "$REPO" rev-parse HEAD) + printf 'batch validation notes\n' > "$REPO/batch-notes.txt" + if [ "$variant" = actual-abi-drift ]; then + printf 'pub const ABI_VERSION: u32 = 40;\n' > "$REPO/crates/shared/src/lib.rs" + fi + if [ "$variant" = prior-receipt-mutation ]; then + printf '{"historical":"rewritten"}\n' > "$REPO/.github/integration-batches/batch-5.json" + fi + if [ "$variant" = authority-change ]; then + printf 'name: batch-controlled prepare merge\n' > "$REPO/.github/workflows/prepare-merge.yml" + fi + git -C "$REPO" add batch-notes.txt crates/shared/src/lib.rs .github + git -C "$REPO" commit --quiet -m 'ci: record batch validation notes' + + patch_one=$(fixture_patch_id "$REPO" "$SOURCE_ONE") + patch_two_a=$(fixture_patch_id "$REPO" "$SOURCE_TWO_A") + patch_two_b=$(fixture_patch_id "$REPO" "$SOURCE_TWO_B") + if [ "$variant" = context-rebase ]; then + contextual_source=$(git -C "$REPO" show --format= --full-index --binary --unified=3 "$SOURCE_ONE" | + git patch-id --stable | awk '{print $1}') + contextual_batch=$(git -C "$REPO" show --format= --full-index --binary --unified=3 "$BATCH_ONE" | + git patch-id --stable | awk '{print $1}') + [ "$contextual_source" != "$contextual_batch" ] || { + echo "context-rebase fixture does not change ordinary diff context" >&2 + exit 1 + } + fi + batch_effect=compatible + recorded_base_sha="$BASE_SHA" + from_version=39 + to_version=39 + batch_required='["docs","host-integration","kernel-unit","posix"]' + source_two_commits=$(jq -n \ + --arg source_a "$SOURCE_TWO_A" --arg batch_a "$BATCH_TWO_A" --arg patch_a "$patch_two_a" \ + --arg source_b "$SOURCE_TWO_B" --arg batch_b "$BATCH_TWO_B" --arg patch_b "$patch_two_b" \ + '[{source_sha:$source_a,batch_sha:$batch_a,patch_id:$patch_a}, + {source_sha:$source_b,batch_sha:$batch_b,patch_id:$patch_b}]') + + case "$variant" in + ambiguous) + source_two_commits=$(jq --arg duplicate "$patch_one" '.[0].patch_id = $duplicate' <<<"$source_two_commits") + ;; + wrong-patch) + source_two_commits=$(jq '.[0].patch_id = "0000000000000000000000000000000000000000"' <<<"$source_two_commits") + ;; + validation-gap) + batch_required='["docs","host-integration","kernel-unit"]' + ;; + bad-abi) + batch_effect=none + ;; + wrong-abi-version) + from_version=38 + to_version=38 + ;; + wrong-base) + recorded_base_sha="$BATCH_ONE" + ;; + valid|squash|base-drift|actual-abi-drift|prior-receipt-mutation|authority-change|whitespace-landed|context-rebase|moved-delta|changed-delta) ;; + *) echo "unknown fixture variant $variant" >&2; exit 2 ;; + esac + + mkdir -p "$REPO/.github/integration-batches" + jq -n \ + --arg effect "$batch_effect" \ + --argjson required "$batch_required" \ + --arg base_sha "$recorded_base_sha" \ + --argjson from_version "$from_version" --argjson to_version "$to_version" \ + --arg source_one "$SOURCE_ONE" --arg batch_one "$BATCH_ONE" --arg patch_one "$patch_one" \ + --arg source_two_head "$SOURCE_TWO_B" --argjson source_two_commits "$source_two_commits" ' + { + schema_version: 1, + batch: { + pull_request: 10, + base_ref: "main", + base_sha: $base_sha, + merge_method: "rebase", + abi: {effect: $effect, from_version: $from_version, to_version: $to_version}, + validation: {treatment: "broad-runtime", required: $required} + }, + sources: [ + { + pull_request: 1, + head_sha: $source_one, + head_ref: "fix/source-one", + abi_effect: "compatible", + required_validation: ["kernel-unit", "posix"], + commits: [{source_sha:$source_one,batch_sha:$batch_one,patch_id:$patch_one}] + }, + { + pull_request: 2, + head_sha: $source_two_head, + head_ref: "fix/source-two", + abi_effect: "none", + required_validation: ["docs", "host-integration"], + commits: $source_two_commits + } + ] + } + ' > "$REPO/.github/integration-batches/batch-10.json" + git -C "$REPO" add .github/integration-batches/batch-10.json + git -C "$REPO" commit --quiet -m 'ci: bind source fixes to integration batch' + BATCH_HEAD=$(git -C "$REPO" rev-parse HEAD) + + mapfile -t BATCH_COMMITS < <(git -C "$REPO" rev-list --reverse "${BASE_SHA}..${BATCH_HEAD}") + + git -C "$REPO" push --quiet origin \ + "$SOURCE_ONE:refs/heads/fix/source-one" \ + "$SOURCE_TWO_B:refs/heads/fix/source-two" \ + "$BATCH_HEAD:refs/heads/integration/batch-10" \ + "$SOURCE_ONE:refs/pull/1/head" \ + "$SOURCE_TWO_B:refs/pull/2/head" \ + "$BATCH_HEAD:refs/pull/10/head" \ + "$BASE_SHA:refs/heads/main" + + git -C "$REPO" checkout --quiet main + if [ "$variant" = squash ]; then + git -C "$REPO" merge --quiet --squash integration/batch-10 >/dev/null + git -C "$REPO" commit --quiet -m 'squashed batch' + elif [ "$variant" = whitespace-landed ]; then + git -C "$REPO" cherry-pick --quiet --no-commit "${BATCH_COMMITS[0]}" >/dev/null + printf 'source one\n' > "$REPO/source-one.txt" + git -C "$REPO" add source-one.txt + git -C "$REPO" commit --quiet -C "${BATCH_COMMITS[0]}" + git -C "$REPO" cherry-pick --quiet "${BATCH_COMMITS[@]:1}" >/dev/null + else + if [ "$variant" = base-drift ]; then + printf 'intervening default commit\n' > "$REPO/intervening.txt" + git -C "$REPO" add intervening.txt + git -C "$REPO" commit --quiet -m 'intervening default commit' + fi + git -C "$REPO" cherry-pick --quiet "${BATCH_COMMITS[@]}" >/dev/null + fi + MERGE_SHA=$(git -C "$REPO" rev-parse HEAD) + git -C "$REPO" push --quiet --force origin "$MERGE_SHA:refs/heads/main" + + printf 'advance\n' > "$REPO/advance.txt" + git -C "$REPO" add advance.txt + git -C "$REPO" commit --quiet -m 'later default commit' + ADVANCED_SHA=$(git -C "$REPO" rev-parse HEAD) + git -C "$REPO" push --quiet origin "$ADVANCED_SHA:refs/heads/advanced-fixture" + git -C "$REPO" reset --quiet --hard "$MERGE_SHA" + + jq -n --arg sha "$SOURCE_ONE" ' + {number:1,state:"open",merged:false,merged_at:null,merge_commit_sha:null,commits:1, + head:{sha:$sha,ref:"fix/source-one",repo:{full_name:"example/repo"}},base:{ref:"main"}} + ' > "$DATA/pr-1.json" + jq -n --arg sha "$SOURCE_TWO_B" ' + {number:2,state:"open",merged:false,merged_at:null,merge_commit_sha:null,commits:2, + head:{sha:$sha,ref:"fix/source-two",repo:{full_name:"example/repo"}},base:{ref:"main"}} + ' > "$DATA/pr-2.json" + jq -n --arg head "$BATCH_HEAD" --arg merge "$MERGE_SHA" --argjson count "${#BATCH_COMMITS[@]}" ' + {number:10,state:"closed",merged:true,merged_at:"2026-07-15T12:00:00Z",merge_commit_sha:$merge,commits:$count, + head:{sha:$head,ref:"integration/batch-10",repo:{full_name:"example/repo"}},base:{ref:"main"}} + ' > "$DATA/pr-10.json" + jq -n --arg sha "$SOURCE_ONE" '[{sha:$sha}]' > "$DATA/commits-1.json" + jq -n --arg a "$SOURCE_TWO_A" --arg b "$SOURCE_TWO_B" '[{sha:$a},{sha:$b}]' > "$DATA/commits-2.json" + printf '%s\n' "${BATCH_COMMITS[@]}" | jq -R '{sha:.}' | jq -s . > "$DATA/commits-10.json" + jq -n '[ + {id:1,context:"merge-gate",state:"success",created_at:"2026-07-15T11:00:00Z"}, + {id:2,context:"merge-gate",state:"failure",created_at:"2026-07-15T10:00:00Z"} + ]' > "$DATA/status.json" + + cat > "$BIN/gh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "$GH_STUB_LOG" +[ "${1:-}" = api ] || exit 99 +shift +include=false +method=GET +endpoint="" +while [ "$#" -gt 0 ]; do + case "$1" in + --include) include=true; shift ;; + --method) method="$2"; shift 2 ;; + -f|-F|-H) shift 2 ;; + /repos/*) endpoint="$1"; shift ;; + *) shift ;; + esac +done +[ -n "$endpoint" ] || exit 98 + +emit() { + if [ "$include" = true ]; then + printf 'HTTP/2.0 200 OK\n\n' + fi + cat "$1" +} + +case "$endpoint" in + /repos/example/repo) + if [ "$include" = true ]; then printf 'HTTP/2.0 200 OK\n\n'; fi + printf '{"default_branch":"main"}\n' + ;; + /repos/example/repo/pulls\?state=open\&head=*\&per_page=100) + query="${endpoint#*head=}" + query="${query%&per_page=100}" + head=$(jq -rn --arg value "$query" '$value | @urid') + ref="${head#*:}" + if [ "$ref" = fix/source-one ]; then pr=1; else pr=2; fi + if [ "$(jq -r .state "$GH_STUB_DATA/pr-$pr.json")" = open ]; then + jq -n --argjson pr "$pr" --arg ref "$ref" \ + '[{number:$pr,head:{ref:$ref,repo:{full_name:"example/repo"}}}]' + else + printf '[]\n' + fi > "$GH_STUB_DATA/open-prs.tmp" + if [ "${GH_STUB_SHARED_BRANCH_PR:-false}" = true ]; then + jq --arg ref "$ref" '. + [{number:99,head:{ref:$ref,repo:{full_name:"example/repo"}}}]' \ + "$GH_STUB_DATA/open-prs.tmp" + else + cat "$GH_STUB_DATA/open-prs.tmp" + fi + ;; + /repos/example/repo/pulls/[0-9]*) + tail="${endpoint#/repos/example/repo/pulls/}" + if [[ "$tail" == */commits\?* ]]; then + pr="${tail%%/*}" + page="${tail##*page=}" + if [ "$page" = 1 ]; then cat "$GH_STUB_DATA/commits-$pr.json"; else printf '[]\n'; fi + exit 0 + fi + pr="${tail%%\?*}" + file="$GH_STUB_DATA/pr-$pr.json" + [ -f "$file" ] || exit 97 + if [ "$method" = PATCH ]; then + jq '.state = "closed"' "$file" > "$file.tmp" + mv "$file.tmp" "$file" + printf 'close %s\n' "$pr" >> "$GH_STUB_MUTATIONS" + if [ "$pr" = 1 ] && [ "${GH_STUB_RECREATE_BRANCH_AFTER_CLOSE:-false}" = true ]; then + recreate_ref=$(jq -r .head.ref "$file") + recreate_sha=$(jq -r .head.sha "$file") + git --git-dir="$GH_STUB_REMOTE" update-ref "refs/heads/$recreate_ref" "$recreate_sha" + fi + fi + if [ "$pr" = 10 ] && [ "${GH_STUB_BATCH_PROPOSED:-false}" = true ]; then + jq '.state = "open" | .merged = false | .merged_at = null | .merge_commit_sha = null' "$file" + elif [ "${GH_STUB_FORK_PR:-}" = "$pr" ]; then + jq '.head.repo.full_name = "someone/fork"' "$file" + elif [ "${GH_STUB_DRIFT_PR:-}" = "$pr" ]; then + jq '.head.sha = "ffffffffffffffffffffffffffffffffffffffff"' "$file" + else + cat "$file" + fi + ;; + /repos/example/repo/commits/*/statuses\?per_page=100\&page=*) + page="${endpoint##*page=}" + if [ "${GH_STUB_MERGE_GATE_FAIL:-false}" = true ]; then + if [ "$page" = 1 ]; then jq '.[0].state = "failure"' "$GH_STUB_DATA/status.json"; else printf '[]\n'; fi + else + if [ "$page" = 1 ]; then cat "$GH_STUB_DATA/status.json"; else printf '[]\n'; fi + fi + if [ "${GH_STUB_ADVANCE_DEFAULT:-false}" = true ]; then + git --git-dir="$GH_STUB_REMOTE" update-ref refs/heads/main "$GH_STUB_ADVANCED_SHA" + fi + ;; + /repos/example/repo/branches/*) + if [ "${GH_STUB_BRANCH_API_FAIL:-false}" = true ]; then + [ "$include" = false ] || printf 'HTTP/2.0 500 Internal Server Error\n\n{}\n' + exit 1 + fi + encoded="${endpoint#/repos/example/repo/branches/}" + ref=$(jq -rn --arg value "$encoded" '$value | @urid') + if ! sha=$(git --git-dir="$GH_STUB_REMOTE" rev-parse --verify "refs/heads/$ref" 2>/dev/null); then + [ "$include" = false ] || printf 'HTTP/2.0 404 Not Found\n\n{}\n' + exit 1 + fi + protected=false + [ "${GH_STUB_PROTECTED_REF:-}" != "$ref" ] || protected=true + if [ "$include" = true ]; then printf 'HTTP/2.0 200 OK\n\n'; fi + jq -n --arg name "$ref" --arg sha "$sha" --argjson protected "$protected" \ + '{name:$name,commit:{sha:$sha},protected:$protected}' + ;; + *) exit 96 ;; +esac +EOF + chmod +x "$BIN/gh" + + export PATH="$BIN:$ORIGINAL_PATH" + export GITHUB_REPOSITORY=example/repo + export GITHUB_DEFAULT_BRANCH=main + export INTEGRATION_BATCH_RETRY_DELAY_SECONDS=0 + export GH_STUB_DATA="$DATA" + export GH_STUB_LOG="$LOG" + export GH_STUB_MUTATIONS="$MUTATIONS" + export GH_STUB_REMOTE="$REMOTE" + export GH_STUB_ADVANCED_SHA="$ADVANCED_SHA" + unset GH_STUB_FORK_PR GH_STUB_DRIFT_PR GH_STUB_PROTECTED_REF GH_STUB_BRANCH_API_FAIL + unset GH_STUB_MERGE_GATE_FAIL GH_STUB_ADVANCE_DEFAULT GH_STUB_SHARED_BRANCH_PR + unset GH_STUB_RECREATE_BRANCH_AFTER_CLOSE +} + +ORIGINAL_PATH="$PATH" + +run_proposed() { + if [ "${RUN_PROPOSED_PRESERVE_DEFAULT:-false}" != true ]; then + git --git-dir="$REMOTE" update-ref refs/heads/main "$BASE_SHA" + fi + git -C "$REPO" checkout --quiet integration/batch-10 + ( + cd "$REPO" + GH_STUB_BATCH_PROPOSED=true \ + bash "$FINALIZER" --manifest .github/integration-batches/batch-10.json --batch-pr 10 --mode proposed + ) +} + +run_finalize() { + git --git-dir="$REMOTE" update-ref refs/heads/main "$MERGE_SHA" + git -C "$REPO" checkout --quiet main + git -C "$REPO" reset --quiet --hard "$MERGE_SHA" + ( + cd "$REPO" + bash "$FINALIZER" --manifest .github/integration-batches/batch-10.json --batch-pr 10 --mode finalize "$@" + ) +} + +expect_failure() { + local pattern="$1" + shift + local output status + set +e + output=$("$@" 2>&1) + status=$? + set -e + [ "$status" -ne 0 ] || { + echo "expected failure matching '$pattern'" >&2 + exit 1 + } + grep -Fq "$pattern" <<<"$output" || { + echo "failure did not match '$pattern':" >&2 + echo "$output" >&2 + exit 1 + } +} + +expect_success() { + local pattern="$1" + shift + local output status + set +e + output=$("$@" 2>&1) + status=$? + set -e + [ "$status" -eq 0 ] || { + echo "expected success containing '$pattern':" >&2 + echo "$output" >&2 + exit 1 + } + grep -Fq "$pattern" <<<"$output" || { + echo "success output did not contain '$pattern':" >&2 + echo "$output" >&2 + exit 1 + } +} + +make_fixture valid +expect_success 'Verified batch PR #10' run_proposed +expect_success 'Dry run only' run_finalize +[ ! -s "$MUTATIONS" ] || { echo "dry run mutated GitHub state" >&2; exit 1; } +git --git-dir="$REMOTE" rev-parse --verify refs/heads/fix/source-one >/dev/null + +expect_success 'finalization complete' run_finalize --apply +[ "$(jq -r .state "$DATA/pr-1.json")" = closed ] +[ "$(jq -r .state "$DATA/pr-2.json")" = closed ] +! git --git-dir="$REMOTE" rev-parse --verify refs/heads/fix/source-one >/dev/null 2>&1 +! git --git-dir="$REMOTE" rev-parse --verify refs/heads/fix/source-two >/dev/null 2>&1 +expect_success 'finalization complete' run_finalize --apply + +make_fixture context-rebase +expect_success 'Verified batch PR #10' run_proposed +expect_success 'Dry run only' run_finalize + +make_fixture moved-delta +expect_failure 'is not an exact replay of source commit' run_proposed + +make_fixture changed-delta +expect_failure 'is not patch-identical to source commit' run_proposed + +make_fixture valid +export GH_STUB_RECREATE_BRANCH_AFTER_CLOSE=true +expect_failure 'reappeared during finalization' run_finalize --apply + +make_fixture valid +export GH_STUB_DRIFT_PR=1 +expect_failure 'no longer at its recorded head' run_proposed + +make_fixture valid +git --git-dir="$REMOTE" update-ref refs/heads/main "$ADVANCED_SHA" +export RUN_PROPOSED_PRESERVE_DEFAULT=true +expect_failure 'not based on the current default-branch tip' run_proposed +unset RUN_PROPOSED_PRESERVE_DEFAULT + +make_fixture valid +export GH_STUB_FORK_PR=2 +expect_failure 'cross-repository' run_proposed + +make_fixture valid +export GH_STUB_PROTECTED_REF=fix/source-one +expect_failure 'branch is protected' run_proposed + +make_fixture valid +export GH_STUB_BRANCH_API_FAIL=true +expect_failure 'could not determine branch state' run_proposed + +make_fixture valid +export GH_STUB_SHARED_BRANCH_PR=true +expect_failure 'shared by another open PR' run_proposed + +make_fixture ambiguous +expect_failure 'manifest does not satisfy schema version 1' run_proposed + +make_fixture wrong-patch +expect_failure 'patch ID changed' run_proposed + +make_fixture validation-gap +expect_failure 'manifest does not satisfy schema version 1' run_proposed + +make_fixture bad-abi +expect_failure 'manifest does not satisfy schema version 1' run_proposed + +make_fixture wrong-abi-version +expect_failure 'does not match base ABI' run_proposed + +make_fixture wrong-base +[ "$(jq -r .batch.base_sha "$REPO/.github/integration-batches/batch-10.json")" = "$BATCH_ONE" ] +git -C "$REPO" merge-base --is-ancestor "$BATCH_ONE" "$BATCH_HEAD" +expect_failure 'batch commit sequence is not linear from recorded base' run_finalize + +make_fixture actual-abi-drift +expect_failure 'does not match batch ABI' run_proposed + +make_fixture prior-receipt-mutation +expect_failure 'must add only its own immutable integration-batch receipt' run_proposed + +make_fixture authority-change +expect_failure 'cannot change their validation authority' run_proposed + +make_fixture squash +expect_failure 'cannot locate the landed batch span' run_finalize + +make_fixture base-drift +expect_failure 'not recorded base' run_finalize + +make_fixture whitespace-landed +expect_failure 'landed batch tree differs from the reviewed batch head' run_finalize + +make_fixture valid +export GH_STUB_MERGE_GATE_FAIL=true +expect_failure 'does not have a latest successful merge-gate status' run_finalize + +make_fixture valid +export GH_STUB_ADVANCE_DEFAULT=true +expect_failure 'default branch advanced during finalization validation' run_finalize + +# Workflow contracts: proposed verification is read-only; finalization is +# maintainer-triggered, default-branch-only, and dry-run by default. +grep -Fq 'pull_request_target:' "$WORKFLOW_VERIFY" +! grep -Eq '^ pull_request:$' "$WORKFLOW_VERIFY" +grep -Fq 'HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }}' "$WORKFLOW_VERIFY" +grep -Fq 'persist-credentials: false' "$WORKFLOW_VERIFY" +grep -Fq 'permissions:' "$WORKFLOW_VERIFY" +! grep -Fq 'contents: write' "$WORKFLOW_VERIFY" +grep -Fq 'Integration batch receipts are append-only' "$WORKFLOW_VERIFY" +grep -Fq 'git worktree add --quiet --detach "$trusted_worktree" "$trusted_base"' "$WORKFLOW_VERIFY" +grep -Fq '"$trusted_worktree/.github/scripts/finalize-integration-batch.sh"' "$WORKFLOW_VERIFY" +grep -Fq 'workflow_dispatch:' "$WORKFLOW_FINALIZE" +grep -Fq 'default: dry-run' "$WORKFLOW_FINALIZE" +grep -Fq "github.ref_name == github.event.repository.default_branch" "$WORKFLOW_FINALIZE" +grep -Fq -- '--mode finalize' "$WORKFLOW_FINALIZE" +grep -Fq 'args+=(--apply)' "$WORKFLOW_FINALIZE" + +echo "integration batch finalizer tests passed" diff --git a/.github/workflows/finalize-integration-batch.yml b/.github/workflows/finalize-integration-batch.yml new file mode 100644 index 0000000000..c1c114d514 --- /dev/null +++ b/.github/workflows/finalize-integration-batch.yml @@ -0,0 +1,76 @@ +name: Finalize integration batch + +# This maintainer-triggered workflow is intentionally separate from the batch +# merge. It defaults to a read-only plan and mutates source PRs only after the +# merged rebase sequence and its merge-gate authority have been reverified. +on: + workflow_dispatch: + inputs: + batch_pr: + description: Merged integration batch pull request number + required: true + type: string + mode: + description: Dry-run plans actions; apply closes absorbed PRs and deletes exact recorded branches + required: true + default: dry-run + type: choice + options: + - dry-run + - apply + +concurrency: + group: integration-batch-finalize-${{ inputs.batch_pr }} + cancel-in-progress: false + +jobs: + finalize: + if: github.ref_type == 'branch' && github.ref_name == github.event.repository.default_branch + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + statuses: read + env: + GH_TOKEN: ${{ github.token }} + GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + BATCH_PR: ${{ inputs.batch_pr }} + APPLY_MODE: ${{ inputs.mode }} + steps: + - uses: actions/checkout@v7.0.0 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 + with: + github-token: "" + + - name: Cache Nix store + flake eval + uses: DeterminateSystems/magic-nix-cache-action@908b263ff629f4cc17666315b7fd3ec127c6244d # v14 + with: + use-gha-cache: false + use-flakehub: false + + - name: Reverify and finalize absorbed source PRs + run: | + set -euo pipefail + [[ "$BATCH_PR" =~ ^[1-9][0-9]*$ ]] || { + echo "::error::batch_pr must be a positive integer" + exit 1 + } + args=( + --manifest ".github/integration-batches/batch-${BATCH_PR}.json" + --batch-pr "$BATCH_PR" + --mode finalize + ) + if [ "$APPLY_MODE" = apply ]; then + args+=(--apply) + elif [ "$APPLY_MODE" != dry-run ]; then + echo "::error::unknown finalization mode: $APPLY_MODE" + exit 1 + fi + bash scripts/dev-shell.sh env \ + GITHUB_DEFAULT_BRANCH="$GITHUB_DEFAULT_BRANCH" \ + bash .github/scripts/finalize-integration-batch.sh "${args[@]}" diff --git a/.github/workflows/verify-integration-batch.yml b/.github/workflows/verify-integration-batch.yml new file mode 100644 index 0000000000..30332f5066 --- /dev/null +++ b/.github/workflows/verify-integration-batch.yml @@ -0,0 +1,95 @@ +name: Verify integration batch + +# A batch may combine unrelated fixes. Its coherence is the validation +# treatment recorded in the manifest, not a shared product theme. This check +# proves that every still-open source PR is represented exactly once in the +# batch before the broad Prepare merge gate is run. +on: + # The workflow definition must come from the default branch. A batch cannot + # be allowed to weaken the check that certifies its own manifest. + pull_request_target: + paths: + - '.github/integration-batches/**' + - '.github/scripts/finalize-integration-batch.sh' + - '.github/scripts/test-finalize-integration-batch.sh' + - '.github/workflows/verify-integration-batch.yml' + - '.github/workflows/finalize-integration-batch.yml' + +jobs: + verify: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + env: + GH_TOKEN: ${{ github.token }} + GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + BATCH_PR: ${{ github.event.pull_request.number }} + BATCH_HEAD: ${{ github.event.pull_request.head.sha }} + steps: + - name: Reject forked batch candidates + env: + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + BASE_REPOSITORY: ${{ github.repository }} + run: | + [ "$HEAD_REPOSITORY" = "$BASE_REPOSITORY" ] || { + echo "::error::Integration batches must use a branch in $BASE_REPOSITORY." + exit 1 + } + + - uses: actions/checkout@v7.0.0 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 + with: + github-token: "" + + - name: Cache Nix store + flake eval + uses: DeterminateSystems/magic-nix-cache-action@908b263ff629f4cc17666315b7fd3ec127c6244d # v14 + with: + use-gha-cache: false + use-flakehub: false + + - name: Verify source-to-batch equivalence + run: | + set -euo pipefail + manifest=".github/integration-batches/batch-${BATCH_PR}.json" + git fetch --quiet --no-tags origin \ + "+refs/heads/${GITHUB_DEFAULT_BRANCH}:refs/remotes/origin/${GITHUB_DEFAULT_BRANCH}" + trusted_base=$(git rev-parse "refs/remotes/origin/${GITHUB_DEFAULT_BRANCH}") + receipt_changes="$RUNNER_TEMP/integration-batch-receipt-changes" + git diff --name-status "$trusted_base" "$BATCH_HEAD" -- .github/integration-batches \ + > "$receipt_changes" + if [ ! -f "$manifest" ]; then + if awk '$2 ~ /^\.github\/integration-batches\/batch-[1-9][0-9]*\.json$/ { found = 1 } END { exit !found }' \ + "$receipt_changes" + then + echo "::error::Integration batch receipts are append-only; this PR changes a receipt without adding $manifest." + cat "$receipt_changes" + exit 1 + fi + echo "No integration batch manifest for PR #${BATCH_PR}; script-only change." + exit 0 + fi + [ "$(git rev-parse HEAD)" = "$BATCH_HEAD" ] || { + echo "::error::checkout is not the exact batch PR head" + exit 1 + } + trusted_worktree="$RUNNER_TEMP/integration-batch-authority" + git worktree add --quiet --detach "$trusted_worktree" "$trusted_base" + ( + cd "$trusted_worktree" + bash scripts/dev-shell.sh env \ + GITHUB_DEFAULT_BRANCH="$GITHUB_DEFAULT_BRANCH" \ + GITHUB_WORKSPACE="$GITHUB_WORKSPACE" \ + bash -c 'cd "$1"; exec bash "$2" --manifest "$3" --batch-pr "$4" --mode proposed' \ + integration-batch-verifier \ + "$GITHUB_WORKSPACE" \ + "$trusted_worktree/.github/scripts/finalize-integration-batch.sh" \ + "$manifest" \ + "$BATCH_PR" + ) diff --git a/docs/integration-batches.md b/docs/integration-batches.md new file mode 100644 index 0000000000..49195130d9 --- /dev/null +++ b/docs/integration-batches.md @@ -0,0 +1,183 @@ +# Integration Batches + +Integration batches let maintainers test and rebase-merge many independently +reviewable fixes together. The fixes do not need a common theme. The batch is +coherent when its validation treatment covers the union of the source changes, +its ABI effect is truthful, and every source commit remains independently +bisectable after the rebase merge. + +Release-control, approval, or permission changes that define the batch's own +validation authority should remain standalone when including them would make +their validation circular. Ordinary unrelated kernel, host, package, and build +fixes may share a batch. + +## Manifest Contract + +A batch PR adds `.github/integration-batches/batch-.json`. The file records: + +- the batch PR, exact base commit, default branch, required `rebase` merge + method, and aggregate ABI effect; +- one validation treatment and the sorted union of required validation IDs; +- every absorbed source PR's exact same-repository head and branch; +- every commit in each source PR, mapped to one unique pre-merge batch commit + with their shared context-free stable Git patch ID. + +Example: + +```json +{ + "schema_version": 1, + "batch": { + "pull_request": 1001, + "base_ref": "main", + "base_sha": "0000000000000000000000000000000000000000", + "merge_method": "rebase", + "abi": { + "effect": "breaking", + "from_version": 39, + "to_version": 40 + }, + "validation": { + "treatment": "abi-kernel-train", + "required": [ + "abi-snapshot", + "browser", + "host-integration", + "kernel-unit", + "libc", + "package-universe", + "posix", + "sortix" + ] + } + }, + "sources": [ + { + "pull_request": 864, + "head_sha": "1111111111111111111111111111111111111111", + "head_ref": "fix/fifo-named-pipes", + "abi_effect": "breaking", + "required_validation": [ + "abi-snapshot", + "host-integration", + "kernel-unit", + "libc", + "package-universe", + "posix", + "sortix" + ], + "commits": [ + { + "source_sha": "1111111111111111111111111111111111111111", + "batch_sha": "2222222222222222222222222222222222222222", + "patch_id": "3333333333333333333333333333333333333333" + } + ] + } + ] +} +``` + +Allowed ABI effects are `none`, `compatible`, and `breaking`. A non-breaking +batch keeps the same ABI version. A breaking batch must increase it. The batch +effect must equal the strongest source effect. The verifier reads +`ABI_VERSION` from `base_sha` and the batch head, so the recorded versions +cannot disagree with the code. This mechanical check does not classify +semantic ABI changes; reviewers and the ordinary ABI snapshot gate still own +that decision. + +Allowed validation IDs are `abi-snapshot`, `kernel-unit`, `fork-instrument`, +`host-integration`, `browser`, `browser-assets`, `libc`, `posix`, `sortix`, +`package-publish`, `package-universe`, `homebrew-pilot`, `vfs-node`, +`vfs-browser`, and `docs`. Lists are sorted and duplicate-free. Add a new ID to +the validator and this reference together when the validation contract grows. + +The manifest is added after the source commits have been rebased or +cherry-picked into the batch, so its own commit can refer to their stable SHAs. +`base_sha` is the parent of the batch's first commit. Do not amend mapped batch +commits or rebase the batch after recording them; regenerate the manifest +instead. + +Generate each `patch_id` from a zero-context binary diff: + +```bash +git show --format= --full-index --binary --unified=0 | git patch-id --stable +``` + +The zero-context form identifies the lines and file metadata the commit changes +without binding the ID to nearby unchanged lines. This lets a source commit and +its rebased batch commit retain one manifest identity when only their +surrounding context moved. A non-whitespace change to an added or removed line +produces a different ID. Git patch IDs normalize whitespace, so the exact +replay and tree check below remains authoritative for every byte-level change, +including whitespace. + +A context-free patch ID is an identifier, not sufficient proof that a change +was applied in the right place. For every source-to-batch mapping, the verifier +also performs a clean three-way replay of the source commit onto the mapped +batch commit's parent. The replayed tree must exactly equal the mapped batch +commit's tree. This rejects an identical textual change moved to a different +location, as well as conflict resolutions or other edits that are not the +source change rebased onto the batch history. + +Receipts are append-only. A manifest-bearing PR must add exactly its own +`batch-.json` and cannot modify, delete, or add another receipt. It also +cannot change the scripts, workflows, dev shell, scope classifier, or publish +flow that supply its validation authority. Land those control-plane changes as +standalone PRs first. + +## Verification And Merge + +`Verify integration batch` checks the proposed batch against live GitHub state. +It fails closed unless every source PR is open, same-repository, based on the +default branch, still at the recorded head, and represented completely. The +recorded batch base must also still be the live default-branch tip. It +recomputes every context-free stable patch ID, proves every mapped commit by +clean three-way replay and exact tree equality, and rejects empty commits, +merge commits, duplicate patch identities, omitted source commits, protected +branches, or API uncertainty. + +The verifier is a read-only `pull_request_target` workflow, so its definition +comes from the default branch rather than the candidate batch. It rejects fork +heads, checks out the exact untrusted head without persisting credentials, and +runs the trusted default branch's verifier and dev shell against that checkout. +This prevents a batch from weakening the check that certifies it. + +The ordinary Prepare merge workflow remains the validation authority. Run the +union of the manifest's suites, obtain `merge-gate=success` on the exact batch +head, and rebase-merge the batch. Do not squash or create a merge commit. + +## Closing Absorbed PRs + +After the batch merges, dispatch `Finalize integration batch` from the default +branch with its PR number. The default `dry-run` mode only prints the plan. + +Before either planning or applying, the finalizer freshly proves that: + +- the batch PR merged from this repository into the default branch; +- its complete original commit sequence maps in order to one contiguous linear + sequence ending at GitHub's recorded merge SHA; +- every corresponding original and landed commit has both the same context-free + stable patch ID and exact tree, and the final landed tree equals the reviewed + batch head; +- the original and landed sequences both start at the manifest's exact base, + which is also the batch head's merge base; +- the immutable manifest in the original head, landed sequence, and current + default branch is byte-identical; +- the exact original batch head's latest `merge-gate` status is successful; +- every source PR still has its recorded head and complete commit list; +- every source mapping and branch remains exact and unprotected; +- no recorded source branch is also the head of another open pull request; +- the default branch did not advance during verification. + +In `apply` mode, the workflow rechecks each source immediately before mutation. +It deletes a present source branch with Git's exact-SHA `force-with-lease`, then +closes the unmerged source PR. An absent branch or already-closed PR is accepted +only during post-merge finalization, which makes a partially completed run safe +to resume. Fork branches, changed heads, protected/default branches, merged +source PRs, ambiguous patch IDs, unexpected API data, and network uncertainty +remain hard failures. + +The finalizer never merges the batch, changes its manifest, rewrites commits, +or claims that a listed validation suite ran. The merge gate supplies that +evidence; the manifest records which evidence the batch was required to earn. diff --git a/docs/repository-organization.md b/docs/repository-organization.md index dd5e6b82d9..95376c8c86 100644 --- a/docs/repository-organization.md +++ b/docs/repository-organization.md @@ -76,3 +76,10 @@ The layout is designed so later CI path filters can make conservative, explainab | `images/**`, `tools/mkrootfs/**` | Rootfs/VFS image checks and consumers of those images | These are intended categories, not a CI implementation. The current PR only keeps the paths clean enough for a future CI-filter PR to use them. + +## Integration Batches + +Reviewable fixes may be combined into a broad integration batch so their union +of required suites runs once before a rebase merge. The machine-checked +manifest, patch-identity rules, ABI classification, and safe post-merge source +PR cleanup are documented in [Integration Batches](integration-batches.md). diff --git a/tests/scripts/package-publish-flow.sh b/tests/scripts/package-publish-flow.sh index 910c472488..e4db163580 100755 --- a/tests/scripts/package-publish-flow.sh +++ b/tests/scripts/package-publish-flow.sh @@ -24,5 +24,6 @@ bash "$REPO_ROOT/.github/scripts/test-reconcile-merge-candidates.sh" bash "$REPO_ROOT/.github/scripts/test-cleanup-merge-candidates.sh" bash "$REPO_ROOT/.github/scripts/test-verify-merge-candidate.sh" bash "$REPO_ROOT/.github/scripts/test-activate-merge-candidate.sh" +bash "$REPO_ROOT/.github/scripts/test-finalize-integration-batch.sh" echo "package publish flow tests passed"