diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 9bb869d1..a42e488f 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -47,6 +47,9 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Check testsuite workflow contract + run: python3 scripts/check-testsuite-workflow-ref.py + - name: Check for undeclared gitlinks shell: bash env: @@ -228,7 +231,7 @@ jobs: permissions: contents: read packages: write # testsuite pushes screenshot OCI artifacts to GHCR - # Keep the testsuite SHA centralized in run-testsuite.yml. + # Keep the testsuite workflow ref centralized in run-testsuite.yml. uses: ./.github/workflows/run-testsuite.yml with: image: ghcr.io/projectbluefin/bluefin:testing diff --git a/docs/skills/ci/SKILL.md b/docs/skills/ci/SKILL.md index 067f575d..55d150d0 100644 --- a/docs/skills/ci/SKILL.md +++ b/docs/skills/ci/SKILL.md @@ -127,12 +127,12 @@ collects coverage but does not own pass/fail. - Preserve action pinning and workflow permissions. - Do not add PAT-based authentication. - Keep end-to-end suites on their configured event. -- Reference `projectbluefin/testsuite`'s reusable E2E workflow through its - managed `@v1` tag, never an immutable digest; testsuite advances `v1` after - each successful main-branch merge. `config:best-practices` pins action refs - to digests, so `projectbluefin/testsuite` is disabled for the - `github-actions` manager in `.github/renovate.json5`; without that rule - Renovate re-pins the ref and freezes the gate on a stale test tree. +- Reference `projectbluefin/testsuite`'s reusable E2E workflow by its managed + `@v1` tag, never a digest; testsuite advances `v1` after each successful main + merge. It is disabled for the `github-actions` Renovate manager in + `.github/renovate.json5`, or re-pinning freezes the gate on a stale test tree. +- Route testsuite calls through `.github/workflows/run-testsuite.yml`; + `scripts/check-testsuite-workflow-ref.py` enforces that and `test_ref: v1`. - Update this skill when workflow behavior changes. ## Verification diff --git a/docs/skills/ci/references/workflow-map.md b/docs/skills/ci/references/workflow-map.md index eace9c5f..b8e2f2a9 100644 --- a/docs/skills/ci/references/workflow-map.md +++ b/docs/skills/ci/references/workflow-map.md @@ -9,3 +9,27 @@ For current workflows: find .github/workflows -maxdepth 1 -type f -name '*.yml' -o -name '*.yaml' | sort git grep -n '^name:\|^on:' .github/workflows ``` + +## The testsuite reference contract + +`.github/workflows/run-testsuite.yml` is the only workflow that may reference +`projectbluefin/testsuite/.github/workflows/e2e.yml` directly. Every other +caller goes through that wrapper, so the ref and `test_ref` are set in exactly +one place. + +Two invariants, both enforced by `scripts/check-testsuite-workflow-ref.py` in +the `validate` job: + +- the reference is `@v1` — testsuite advances that tag after each successful + main-branch merge, so a digest pin silently freezes the gate on a stale test + tree (this is the #929 regression); +- the wrapper passes `test_ref: v1`. + +Renovate would otherwise undo the first one: `config:best-practices` pins +action refs to digests, so `projectbluefin/testsuite` is excluded from the +`github-actions` manager in `.github/renovate.json5`. Removing that exclusion +re-pins the ref and reintroduces the same freeze. + +```bash +python3 scripts/check-testsuite-workflow-ref.py +``` diff --git a/scripts/check-testsuite-workflow-ref.py b/scripts/check-testsuite-workflow-ref.py new file mode 100644 index 00000000..1bc0ba6d --- /dev/null +++ b/scripts/check-testsuite-workflow-ref.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Enforce the Bluefin-to-testsuite reusable-workflow contract.""" + +from pathlib import Path +import re +import sys + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW_DIR = ROOT / ".github" / "workflows" +WRAPPER = WORKFLOW_DIR / "run-testsuite.yml" +WORKFLOW_REF = re.compile( + r"^\s+uses:\s+projectbluefin/testsuite/\.github/workflows/e2e\.yml@([^\s#]+)", + re.MULTILINE, +) +TEST_REF = re.compile(r"^\s+test_ref:\s+([^\s#]+)", re.MULTILINE) + + +def main() -> int: + errors: list[str] = [] + wrapper_refs = WORKFLOW_REF.findall(WRAPPER.read_text(encoding="utf-8")) + + if wrapper_refs != ["v1"]: + errors.append( + f"{WRAPPER.relative_to(ROOT)} must contain exactly one direct testsuite " + f"workflow reference at @v1; found {wrapper_refs!r}" + ) + + wrapper_test_refs = TEST_REF.findall(WRAPPER.read_text(encoding="utf-8")) + if wrapper_test_refs != ["v1"]: + errors.append( + f"{WRAPPER.relative_to(ROOT)} must pass exactly one test_ref: v1; " + f"found {wrapper_test_refs!r}" + ) + + for workflow in sorted(WORKFLOW_DIR.glob("*.y*ml")): + if workflow == WRAPPER: + continue + refs = WORKFLOW_REF.findall(workflow.read_text(encoding="utf-8")) + if refs: + errors.append( + f"{workflow.relative_to(ROOT)} calls testsuite e2e directly; " + "call the local run-testsuite wrapper instead" + ) + + if errors: + print("Testsuite workflow contract failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + print("Testsuite workflow contract passed: canonical wrapper uses @v1 with test_ref: v1") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())