diff --git a/.gitlab/collect-result/CollectResults.java b/.gitlab/collect-result/CollectResults.java index ad7c6429cda..ac8d1cea04a 100644 --- a/.gitlab/collect-result/CollectResults.java +++ b/.gitlab/collect-result/CollectResults.java @@ -3,12 +3,15 @@ class CollectResults { public static void main(String[] args) throws Exception { + // Detect flaky jobs + var continueOnFailure = Boolean.parseBoolean(System.getenv("CONTINUE_ON_FAILURE")); + // Run collector var collector = new ResultCollector( Path.of("results"), Path.of("workspace"), - List.of(Path.of("workspace"), Path.of("buildSrc"))); - + List.of(Path.of("workspace"), Path.of("buildSrc")), + continueOnFailure); collector.collect(); } } diff --git a/.gitlab/collect-result/JUnitReport.java b/.gitlab/collect-result/JUnitReport.java index b79b8a551ad..a77a54af4b3 100644 --- a/.gitlab/collect-result/JUnitReport.java +++ b/.gitlab/collect-result/JUnitReport.java @@ -153,6 +153,29 @@ void tagFinalStatuses() { } } + /// Marks every testcase as `test.final_status=skip` for jobs that tolerate failures. + /// + /// The flaky test jobs (`test_flaky`, `test_flaky_inst`) run with `CONTINUE_ON_FAILURE=true`, so + /// their result never gates the pipeline: it runs to completion regardless of pass or fail. + /// `test.final_status` records that CI impact rather than the raw outcome, so every test in these + /// jobs is `skip` — a failure is a non-blocking failure and a pass is a non-blocking pass. This + /// keeps flaky failures from creating false-positive notifications and skewing SLIs, while the + /// real per-test outcome stays available in `test.status` (derived from the ``, + /// ``, and `` children, which are left in place). Always-green tests that could + /// leave the flaky pipeline are then found with `@test.status:pass @test.final_status:skip`. + /// + /// **Must run before {@link #tagFinalStatuses()}** so the natural pass/fail status is never + /// assigned. Testcases already tagged by {@link #tagRetriedAttempts()} or + /// {@link #tagSyntheticFailures()} are left untouched (already `skip`). + void tagAllAsSkipped() { + for (var testcase : testcases()) { + if (hasFinalStatusProperty(testcase)) { + continue; + } + addFinalStatusProperty(testcase, "skip", MissingPropertiesPlacement.FIRST_CHILD); + } + } + void write(Path xmlFile) throws Exception { Files.createDirectories(xmlFile.getParent()); var tmpFile = Files.createTempFile(xmlFile.getParent(), "collect-results-", ".xml"); diff --git a/.gitlab/collect-result/ResultCollector.java b/.gitlab/collect-result/ResultCollector.java index 423b44343d8..c851e95cdc2 100644 --- a/.gitlab/collect-result/ResultCollector.java +++ b/.gitlab/collect-result/ResultCollector.java @@ -13,12 +13,15 @@ final class ResultCollector { private final Path resultsDir; private final Path workspaceDir; private final List searchDirs; + private final boolean continueOnFailure; private final SourceFileResolver sourceFileResolver; - ResultCollector(Path resultsDir, Path workspaceDir, List searchDirs) { + ResultCollector( + Path resultsDir, Path workspaceDir, List searchDirs, boolean continueOnFailure) { this.resultsDir = resultsDir; this.workspaceDir = workspaceDir; this.searchDirs = searchDirs; + this.continueOnFailure = continueOnFailure; this.sourceFileResolver = new SourceFileResolver(workspaceDir); } @@ -32,6 +35,10 @@ void collect() throws Exception { return; } + if (continueOnFailure) { + System.out.println("CONTINUE_ON_FAILURE=true: reporting all tests as skip"); + } + System.out.println("Saving test results:"); for (var sourceXml : findXmlFiles(testResultDirs)) { collect(sourceXml); @@ -51,6 +58,11 @@ private void collect(Path sourceXml) throws Exception { report.tagRetriedAttempts(); reportChangedBeforeFinalStatus |= report.normalizeStableTestNames(); report.tagSyntheticFailures(); + // Flaky jobs (CONTINUE_ON_FAILURE=true) never gate CI, so record every test as skip before + // assigning natural statuses (APMLP-1267). + if (continueOnFailure) { + report.tagAllAsSkipped(); + } report.tagFinalStatuses(); report.write(targetXml); diff --git a/AGENTS.md b/AGENTS.md index 69e6e48aac3..4eee2471c3d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,7 +68,7 @@ docs/ Developer documentation (see below) - Title: imperative verb sentence describing user-visible change (e.g. "Fix span sampling rule parsing") - Labels: always add `tag: ai generated` and at least one `comp:` or `inst:` label + one `type:` label -- Use `tag: no release note` for internal/refactoring changes +- Use `tag: no release notes` for internal/refactoring changes - Open as draft first, convert to ready when reviewable ## Bootstrap constraints (critical) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f5f9bdd15ab..4dde1824dd7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -118,7 +118,7 @@ Both pull requests and issues should be labelled with at least a component or an Labels are used to not only categorize but also alter the continuous integration behavior: -* `tag: no release note` to exclude a pull request from the next release changelog. Use it when changes are not relevant to the users like: +* `tag: no release notes` to exclude a pull request from the next release changelog. Use it when changes are not relevant to the users like: * Internal features changes * Refactoring pull requests * CI and build tools improvements