Add execution instrumentation for Karate v2

.github/CODEOWNERS

@@ -110,7 +110,7 @@
 /dd-java-agent/instrumentation/cucumber-5.4/        @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/jacoco-0.8.9/        @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/junit                @DataDog/ci-app-libraries
-/dd-java-agent/instrumentation/karate-1.0/          @DataDog/ci-app-libraries
+/dd-java-agent/instrumentation/karate/              @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/scalatest-3.0.8/     @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/selenium-3.13/       @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/testng/              @DataDog/ci-app-libraries

dd-java-agent/instrumentation/karate/karate-2.0/build.gradle

@@ -0,0 +1,52 @@
+plugins {
+  id 'dd-trace-java.instrumentation.testing-framework-tests'
+}
+
+apply from: "$rootDir/gradle/java.gradle"
+
+// Karate 2.x is compiled with Java 21 (uses virtual threads for parallelism)
+tracerJava {
+  addSourceSetFor(JavaVersion.VERSION_21)
+}
+
+muzzle {
+  pass {
+    group = 'io.karatelabs'
+    module = 'karate-core'
+    versions = '[2.0.0,)'
+    javaVersion = '21'
+  }
+}
+
+["compileMain_java21Java", "compileTestJava"].each {
+  tasks.named(it, JavaCompile) {
+    configureCompiler(it, 21, JavaVersion.VERSION_21)
+  }
+}
+
+tasks.withType(GroovyCompile).configureEach {
+  configureCompiler(it, 21)
+}
+
+dependencies {
+  main_java21CompileOnly group: 'io.karatelabs', name: 'karate-core', version: '2.0.9'
+
+  testImplementation project(':dd-java-agent:agent-ci-visibility:civisibility-instrumentation-test-fixtures')
+  testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: libs.versions.junit.platform.get()
+  testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: libs.versions.junit5.get()
+
+  testImplementation(group: 'io.karatelabs', name: 'karate-core', version: '2.0.9') {
+    // excluding logback to avoid conflicts with libs.bundles.test.logging
+    exclude group: 'ch.qos.logback', module: 'logback-classic'
+  }
+}
+
+// Using recommended Karate project layout where Karate feature files sit in same /test/java folders as their java counterparts
+sourceSets {
+  test {
+    resources {
+      srcDir file('src/test/java')
+      exclude '**/*.java'
+    }
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java/datadog/trace/instrumentation/karate2/KarateExecutionInstrumentation.java

@@ -0,0 +1,87 @@
+package datadog.trace.instrumentation.karate2;
+
+import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named;
+import static net.bytebuddy.matcher.ElementMatchers.isBridge;
+import static net.bytebuddy.matcher.ElementMatchers.not;
+import static net.bytebuddy.matcher.ElementMatchers.returns;
+import static net.bytebuddy.matcher.ElementMatchers.takesArgument;
+import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
+import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments;
+
+import com.google.auto.service.AutoService;
+import datadog.trace.agent.tooling.Instrumenter;
+import datadog.trace.agent.tooling.InstrumenterModule;
+import datadog.trace.api.Config;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Drives test-retry execution policies (ATR / EFD / attempt-to-fix) and failure suppression for
+ * Karate v2.
+ *
+ * <ul>
+ *   <li>{@code ScenarioRuntime#call()} is advised to re-run the scenario while the execution policy
+ *       is applicable, overriding the returned {@code ScenarioResult} with the final attempt.
+ *   <li>{@code ScenarioResult#addStepResult(StepResult)} is advised to replace a failing step with
+ *       a skipped one when the policy requests failure suppression.
+ * </ul>
+ *
+ * Compiled for Java 8 (see {@link KarateInstrumentation}); the advice lives in the {@code java21}
+ * source set.
+ */
+@AutoService(InstrumenterModule.class)
+public class KarateExecutionInstrumentation extends InstrumenterModule.CiVisibility
+    implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice {
+
+  public KarateExecutionInstrumentation() {
+    super("ci-visibility", "karate", "test-retry");
+  }
+
+  @Override
+  public boolean isEnabled() {
+    return super.isEnabled() && Config.get().isCiVisibilityExecutionPoliciesEnabled();
+  }
+
+  @Override
+  public String[] knownMatchingTypes() {
+    return new String[] {"io.karatelabs.core.ScenarioRuntime", "io.karatelabs.core.ScenarioResult"};
+  }
+
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {
+      packageName + ".KarateUtils",
+      packageName + ".TestEventsHandlerHolder",
+      packageName + ".ExecutionContext",
+      packageName + ".KarateTracingListener",
+      packageName + ".KarateScenarioAdvice",
+      packageName + ".KarateScenarioAdvice$RetryAdvice",
+      packageName + ".KarateScenarioAdvice$SuppressErrorAdvice"
+    };
+  }
+
+  @Override
+  public Map<String, String> contextStore() {
+    return Collections.singletonMap(
+        "io.karatelabs.gherkin.Scenario", packageName + ".ExecutionContext");
+  }
+
+  @Override
+  public void methodAdvice(MethodTransformer transformer) {
+    // ScenarioRuntime#call() is the run()-equivalent; match the concrete ScenarioResult-returning
+    // method, not the synthetic Callable#call() bridge.
+    transformer.applyAdvice(
+        named("call")
+            .and(takesNoArguments())
+            .and(returns(named("io.karatelabs.core.ScenarioResult")))
+            .and(not(isBridge())),
+        packageName + ".KarateScenarioAdvice$RetryAdvice");
+
+    // ScenarioResult#addStepResult(StepResult)
+    transformer.applyAdvice(
+        named("addStepResult")
+            .and(takesArguments(1))
+            .and(takesArgument(0, named("io.karatelabs.core.StepResult"))),
+        packageName + ".KarateScenarioAdvice$SuppressErrorAdvice");
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java/datadog/trace/instrumentation/karate2/KarateInstrumentation.java

@@ -0,0 +1,53 @@
+package datadog.trace.instrumentation.karate2;
+
+import static net.bytebuddy.matcher.ElementMatchers.isConstructor;
+
+import com.google.auto.service.AutoService;
+import datadog.trace.agent.tooling.Instrumenter;
+import datadog.trace.agent.tooling.InstrumenterModule;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Registers a {@code io.karatelabs.core.RunListener} on every {@code
+ * io.karatelabs.core.Runner.Builder}.
+ *
+ * <p>This module is compiled for Java 8 so the agent can enumerate it on any JVM; the advice and
+ * helper classes that reference the Java 21 {@code io.karatelabs} API live in the {@code java21}
+ * source set and are referenced by name.
+ */
+@AutoService(InstrumenterModule.class)
+public class KarateInstrumentation extends InstrumenterModule.CiVisibility
+    implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice {
+
+  public KarateInstrumentation() {
+    super("ci-visibility", "karate");
+  }
+
+  @Override
+  public String instrumentedType() {
+    return "io.karatelabs.core.Runner$Builder";
+  }
+
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {
+      packageName + ".KarateUtils",
+      packageName + ".TestEventsHandlerHolder",
+      packageName + ".ExecutionContext",
+      packageName + ".KarateTracingListener",
+      packageName + ".KarateBuilderAdvice"
+    };
+  }
+
+  @Override
+  public Map<String, String> contextStore() {
+    return Collections.singletonMap(
+        "io.karatelabs.gherkin.Scenario", packageName + ".ExecutionContext");
+  }
+
+  @Override
+  public void methodAdvice(MethodTransformer transformer) {
+    transformer.applyAdvice(isConstructor(), packageName + ".KarateBuilderAdvice");
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/ExecutionContext.java

@@ -0,0 +1,56 @@
+package datadog.trace.instrumentation.karate2;
+
+import datadog.trace.api.civisibility.config.TestIdentifier;
+import datadog.trace.api.civisibility.config.TestSourceData;
+import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
+import io.karatelabs.gherkin.Scenario;
+import java.util.Collection;
+
+public class ExecutionContext {
+
+  private final TestExecutionPolicy executionPolicy;
+  private boolean suppressFailures;
+  private Throwable suppressedError;
+
+  public ExecutionContext(TestExecutionPolicy executionPolicy) {
+    this.executionPolicy = executionPolicy;
+  }
+
+  public void setSuppressFailures(boolean suppressFailures) {
+    this.suppressFailures = suppressFailures;
+  }
+
+  public boolean shouldSuppressFailures() {
+    return suppressFailures;
+  }
+
+  public TestExecutionPolicy getExecutionPolicy() {
+    return executionPolicy;
+  }
+
+  /**
+   * Karate v2 {@code StepResult} is immutable (no {@code setFailedReason}/{@code setErrorIgnored}),
+   * so a failure that was suppressed for retry purposes cannot be carried on the replacement step.
+   * We stash it here instead, so the tracing listener can still report the failure to CI
+   * Visibility.
+   */
+  public void setSuppressedError(Throwable suppressedError) {
+    if (this.suppressedError == null) {
+      this.suppressedError = suppressedError;
+    }
+  }
+
+  public Throwable getAndClearSuppressedError() {
+    Throwable suppressedError = this.suppressedError;
+    this.suppressedError = null;
+    return suppressedError;
+  }
+
+  public static ExecutionContext create(Scenario scenario) {
+    TestIdentifier testIdentifier = KarateUtils.toTestIdentifier(scenario);
+    Collection<String> testTags = KarateUtils.getCategories(scenario.getTagsEffective());
+    return new ExecutionContext(
+        TestEventsHandlerHolder.TEST_EVENTS_HANDLER.executionPolicy(
+            testIdentifier, TestSourceData.UNKNOWN, testTags));
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateBuilderAdvice.java

@@ -0,0 +1,25 @@
+package datadog.trace.instrumentation.karate2;
+
+import datadog.trace.bootstrap.ContextStore;
+import datadog.trace.bootstrap.InstrumentationContext;
+import io.karatelabs.core.RunEvent;
+import io.karatelabs.core.RunListener;
+import io.karatelabs.core.Runner;
+import io.karatelabs.gherkin.Scenario;
+import net.bytebuddy.asm.Advice;
+
+/** Advice for the {@code io.karatelabs.core.Runner.Builder} constructor. */
+public class KarateBuilderAdvice {
+
+  @Advice.OnMethodExit
+  public static void onRunnerBuilderConstructorExit(@Advice.This Runner.Builder builder) {
+    ContextStore<Scenario, ExecutionContext> scenarioContext =
+        InstrumentationContext.get(Scenario.class, ExecutionContext.class);
+    builder.listener(new KarateTracingListener(scenarioContext));
+  }
+
+  // Karate 2.0.0 and above
+  public static void muzzleCheck(RunListener runListener) {
+    runListener.onEvent((RunEvent) null);
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateScenarioAdvice.java

@@ -0,0 +1,100 @@
+package datadog.trace.instrumentation.karate2;
+
+import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
+import datadog.trace.bootstrap.CallDepthThreadLocalMap;
+import datadog.trace.bootstrap.InstrumentationContext;
+import io.karatelabs.core.RunEvent;
+import io.karatelabs.core.RunListener;
+import io.karatelabs.core.ScenarioResult;
+import io.karatelabs.core.ScenarioRuntime;
+import io.karatelabs.core.StepResult;
+import io.karatelabs.gherkin.Scenario;
+import net.bytebuddy.asm.Advice;
+
+/** Advice classes for {@code io.karatelabs.core.ScenarioRuntime}/{@code ScenarioResult}. */
+public class KarateScenarioAdvice {
+
+  public static class RetryAdvice {
+    @Advice.OnMethodEnter
+    public static void beforeExecute(@Advice.This ScenarioRuntime scenarioRuntime) {
+      ExecutionContext executionContext =
+          InstrumentationContext.get(Scenario.class, ExecutionContext.class)
+              .computeIfAbsent(scenarioRuntime.getScenario(), ExecutionContext::create);
+
+      // Indicate beforehand whether failures should be suppressed. This aligns the ordering with
+      // the rest of the frameworks.
+      TestExecutionPolicy executionPolicy = executionContext.getExecutionPolicy();
+      executionContext.setSuppressFailures(executionPolicy.suppressFailures());
+    }
+
+    @Advice.OnMethodExit
+    public static void afterExecute(
+        @Advice.This ScenarioRuntime scenarioRuntime,
+        @Advice.Return(readOnly = false) ScenarioResult result) {
+      if (CallDepthThreadLocalMap.incrementCallDepth(ScenarioRuntime.class) > 0) {
+        // nested call (a retry invoked below, or a called scenario)
+        return;
+      }
+
+      try {
+        Scenario scenario = scenarioRuntime.getScenario();
+        ExecutionContext context =
+            InstrumentationContext.get(Scenario.class, ExecutionContext.class).get(scenario);
+        if (context == null) {
+          return;
+        }
+
+        ScenarioResult finalResult = result;
+        TestExecutionPolicy executionPolicy = context.getExecutionPolicy();
+        while (executionPolicy.applicable()) {
+          ScenarioRuntime retry =
+              new ScenarioRuntime(scenarioRuntime.getFeatureRuntime(), scenario);
+          finalResult = retry.call();
+        }
+
+        // override the return value so the final attempt is the one recorded.
+        result = finalResult;
+      } finally {
+        CallDepthThreadLocalMap.reset(ScenarioRuntime.class);
+      }
+    }
+
+    // Karate 2.0.0 and above
+    public static void muzzleCheck(RunListener runListener) {
+      runListener.onEvent((RunEvent) null);
+    }
+  }
+
+  public static class SuppressErrorAdvice {
+    @Advice.OnMethodEnter
+    public static void onAddingStepResult(
+        @Advice.Argument(value = 0, readOnly = false) StepResult stepResult,
+        @Advice.FieldValue("scenario") Scenario scenario) {
+
+      if (stepResult.isFailed()) {
+        ExecutionContext executionContext =
+            InstrumentationContext.get(Scenario.class, ExecutionContext.class).get(scenario);
+        if (executionContext == null) {
+          return;
+        }
+
+        // Suppress every failing step of a to-be-retried attempt (not just the first): with
+        // continueOnStepFailure a single attempt can add multiple failing steps, and any leak
+        // would mark the retry attempt's result failed.
+        if (executionContext.shouldSuppressFailures()) {
+          // v2 StepResult is immutable: preserve the error out-of-band, then replace the failing
+          // step with a skipped one so the scenario no longer counts as failed.
+          executionContext.setSuppressedError(stepResult.getError());
+          stepResult = StepResult.skipped(stepResult.getStep(), stepResult.getStartTime());
+        }
+      }
+    }
+
+    // Karate 2.0.0 and above
+    public static void muzzleCheck(RunListener runListener) {
+      runListener.onEvent((RunEvent) null);
+    }
+  }
+
+  private KarateScenarioAdvice() {}
+}