Report failed flaky tests as skip

.gitlab/collect-result/CollectResults.java

@@ -3,12 +3,15 @@
 
 class CollectResults {
   public static void main(String[] args) throws Exception {
+    // Detect flaky jobs
+    var continueOnFailure = Boolean.parseBoolean(System.getenv("CONTINUE_ON_FAILURE"));
+    // Run collector
     var collector =
         new ResultCollector(
             Path.of("results"),
             Path.of("workspace"),
-            List.of(Path.of("workspace"), Path.of("buildSrc")));
-
+            List.of(Path.of("workspace"), Path.of("buildSrc")),
+            continueOnFailure);
     collector.collect();
   }
 }

.gitlab/collect-result/JUnitReport.java

@@ -153,6 +153,26 @@ void tagFinalStatuses() {
     }
   }
 
+  /// Downgrades failing testcases to `test.final_status=skip` for jobs that tolerate failures.
+  ///
+  /// The flaky test jobs (`test_flaky`, `test_flaky_inst`) run with `CONTINUE_ON_FAILURE=true`, so
+  /// Gradle test failures are swallowed and a failing flaky test does not fail the job. Reporting
+  /// those failures to Test Optimization as `fail` produces false-positive failure notifications
+  /// and skews SLIs, so we record them as `skip` instead. Passing and skipped tests keep their 
+  /// natural status, so the tests stay visible in Test Optimization.
+  ///
+  /// **Must run before {@link #tagFinalStatuses()}** so the natural `fail` status is never assigned
+  /// to these testcases. Testcases already tagged by {@link #tagRetriedAttempts()} or
+  /// {@link #tagSyntheticFailures()} are left untouched.
+  void tagFailuresAsSkipped() {
+    for (var testcase : testcases()) {
+      if (hasFinalStatusProperty(testcase) || !isFailed(testcase)) {
+        continue;
+      }
+      addFinalStatusProperty(testcase, "skip", MissingPropertiesPlacement.FIRST_CHILD);
+    }
+  }
+
   void write(Path xmlFile) throws Exception {
     Files.createDirectories(xmlFile.getParent());
     var tmpFile = Files.createTempFile(xmlFile.getParent(), "collect-results-", ".xml");
@@ -239,7 +259,7 @@ private static boolean propertiesHasFinalStatusProperty(Element properties) {
   }
 
   private static String finalStatus(Element testcase) {
-    if (hasChildElement(testcase, "failure") || hasChildElement(testcase, "error")) {
+    if (isFailed(testcase)) {
       return "fail";
     }
     if (hasChildElement(testcase, "skipped")) {
@@ -248,6 +268,10 @@ private static String finalStatus(Element testcase) {
     return "pass";
   }
 
+  private static boolean isFailed(Element testcase) {
+    return hasChildElement(testcase, "failure") || hasChildElement(testcase, "error");
+  }
+
   private static Element firstChildElement(Element parent, String tagName) {
     var children = parent.getChildNodes();
     for (var i = 0; i < children.getLength(); i++) {

.gitlab/collect-result/ResultCollector.java

@@ -13,12 +13,15 @@ final class ResultCollector {
   private final Path resultsDir;
   private final Path workspaceDir;
   private final List<Path> searchDirs;
+  private final boolean continueOnFailure;
   private final SourceFileResolver sourceFileResolver;
 
-  ResultCollector(Path resultsDir, Path workspaceDir, List<Path> searchDirs) {
+  ResultCollector(
+      Path resultsDir, Path workspaceDir, List<Path> searchDirs, boolean continueOnFailure) {
     this.resultsDir = resultsDir;
     this.workspaceDir = workspaceDir;
     this.searchDirs = searchDirs;
+    this.continueOnFailure = continueOnFailure;
     this.sourceFileResolver = new SourceFileResolver(workspaceDir);
   }
 
@@ -32,6 +35,11 @@ void collect() throws Exception {
       return;
     }
 
+    if (continueOnFailure) {
+      System.out.println(
+          "CONTINUE_ON_FAILURE=true: reporting failed tests as skip");
+    }
+
     System.out.println("Saving test results:");
     for (var sourceXml : findXmlFiles(testResultDirs)) {
       collect(sourceXml);
@@ -51,6 +59,10 @@ private void collect(Path sourceXml) throws Exception {
     report.tagRetriedAttempts();
     reportChangedBeforeFinalStatus |= report.normalizeStableTestNames();
     report.tagSyntheticFailures();
+    // On flaky jobs, downgrade failures to skip before assigning natural statuses (APMLP-1267).
+    if (continueOnFailure) {
+      report.tagFailuresAsSkipped();
+    }
     report.tagFinalStatuses();
     report.write(targetXml);
 

AGENTS.md

@@ -68,7 +68,7 @@ docs/                     Developer documentation (see below)
 
 - Title: imperative verb sentence describing user-visible change (e.g. "Fix span sampling rule parsing")
 - Labels: always add `tag: ai generated` and at least one `comp:` or `inst:` label + one `type:` label
-- Use `tag: no release note` for internal/refactoring changes
+- Use `tag: no release notes` for internal/refactoring changes
 - Open as draft first, convert to ready when reviewable
 
 ## Bootstrap constraints (critical)

CONTRIBUTING.md

@@ -118,7 +118,7 @@ Both pull requests and issues should be labelled with at least a component or an
 
 Labels are used to not only categorize but also alter the continuous integration behavior:
 
-* `tag: no release note` to exclude a pull request from the next release changelog. Use it when changes are not relevant to the users like:
+* `tag: no release notes` to exclude a pull request from the next release changelog. Use it when changes are not relevant to the users like:
   * Internal features changes
   * Refactoring pull requests
   * CI and build tools improvements