From 1d3237d033c8a4693f6ddd953c93e63c261ccbe0 Mon Sep 17 00:00:00 2001 From: Julien Herr Date: Tue, 25 Aug 2026 21:56:01 +0200 Subject: [PATCH 1/2] test(config): pin what configfailurepolicy does at each level `FailurePolicyTest` covers @BeforeClass and @BeforeMethod, but neither @BeforeSuite nor @BeforeTest, under either policy. That is the gap GITHUB-2731 reports through: with `continue`, a failed @BeforeTest is the one and only case where the test method still runs. Record the whole 4x2 matrix so the inconsistency is visible in the test suite rather than only in an issue, and so that whichever way it is eventually settled, the expectations that have to move are named. The samples are top-level classes rather than nested ones on purpose: Gradle's TestNG runner collects the nested classes of a filtered class, and a @BeforeSuite that fails would then poison the whole run. Refs #2731 --- .../issue2731/FailedBeforeClassSample.java | 15 +++++ .../issue2731/FailedBeforeMethodSample.java | 15 +++++ .../issue2731/FailedBeforeSuiteSample.java | 15 +++++ .../issue2731/FailedBeforeTestSample.java | 15 +++++ .../issue2731/IssueTest.java | 66 +++++++++++++++++++ testng-core/src/test/resources/testng.xml | 1 + 6 files changed, 127 insertions(+) create mode 100644 testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeClassSample.java create mode 100644 testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeMethodSample.java create mode 100644 testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeSuiteSample.java create mode 100644 testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeTestSample.java create mode 100644 testng-core/src/test/java/test/configurationfailurepolicy/issue2731/IssueTest.java diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeClassSample.java b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeClassSample.java new file mode 100644 index 0000000000..e9480a1199 --- /dev/null +++ b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeClassSample.java @@ -0,0 +1,15 @@ +package test.configurationfailurepolicy.issue2731; + +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +public class FailedBeforeClassSample { + + @BeforeClass + public void setupClassFails() { + throw new RuntimeException("setup class fail"); + } + + @Test + public void test1() {} +} diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeMethodSample.java b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeMethodSample.java new file mode 100644 index 0000000000..6266fc2fb0 --- /dev/null +++ b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeMethodSample.java @@ -0,0 +1,15 @@ +package test.configurationfailurepolicy.issue2731; + +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class FailedBeforeMethodSample { + + @BeforeMethod + public void setupMethodFails() { + throw new RuntimeException("setup method fail"); + } + + @Test + public void test1() {} +} diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeSuiteSample.java b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeSuiteSample.java new file mode 100644 index 0000000000..e15ea94340 --- /dev/null +++ b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeSuiteSample.java @@ -0,0 +1,15 @@ +package test.configurationfailurepolicy.issue2731; + +import org.testng.annotations.BeforeSuite; +import org.testng.annotations.Test; + +public class FailedBeforeSuiteSample { + + @BeforeSuite + public void setupSuiteFails() { + throw new RuntimeException("setup suite fail"); + } + + @Test + public void test1() {} +} diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeTestSample.java b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeTestSample.java new file mode 100644 index 0000000000..6ea70d3c50 --- /dev/null +++ b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/FailedBeforeTestSample.java @@ -0,0 +1,15 @@ +package test.configurationfailurepolicy.issue2731; + +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +public class FailedBeforeTestSample { + + @BeforeTest + public void setupTestFails() { + throw new RuntimeException("setup test fail"); + } + + @Test + public void test1() {} +} diff --git a/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/IssueTest.java b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/IssueTest.java new file mode 100644 index 0000000000..236df8ff0e --- /dev/null +++ b/testng-core/src/test/java/test/configurationfailurepolicy/issue2731/IssueTest.java @@ -0,0 +1,66 @@ +package test.configurationfailurepolicy.issue2731; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.testng.TestListenerAdapter; +import org.testng.TestNG; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import org.testng.xml.XmlSuite; +import test.SimpleBaseTest; + +/** + * Pins what "configfailurepolicy" does to a test method for each level of failing configuration. + * + *

This is a characterization test: it records the behaviour, it does not argue that the + * behaviour is right. The row that GITHUB-2731 is about is @BeforeTest under CONTINUE, the only + * combination where the test method still runs. @BeforeClass and @BeforeMethod skip it + * under either policy -- CONTINUE only narrows which instance or which invocation the failure + * invalidates, and these samples have a single one of each -- while @BeforeSuite stops the + * suite regardless of the policy. Whichever way that inconsistency is eventually settled, these + * expectations are the ones that have to move. + */ +public class IssueTest extends SimpleBaseTest { + + @DataProvider(name = "dp") + public Object[][] getData() { + // params - sample, policy, passed, skipped, configuration failures + return new Object[][] { + new Object[] {FailedBeforeSuiteSample.class, XmlSuite.FailurePolicy.SKIP, 0, 1, 1}, + new Object[] {FailedBeforeSuiteSample.class, XmlSuite.FailurePolicy.CONTINUE, 0, 1, 1}, + new Object[] {FailedBeforeTestSample.class, XmlSuite.FailurePolicy.SKIP, 0, 1, 1}, + // GITHUB-2731: the odd one out, the test method runs despite its @BeforeTest having failed + new Object[] {FailedBeforeTestSample.class, XmlSuite.FailurePolicy.CONTINUE, 1, 0, 1}, + new Object[] {FailedBeforeClassSample.class, XmlSuite.FailurePolicy.SKIP, 0, 1, 1}, + new Object[] {FailedBeforeClassSample.class, XmlSuite.FailurePolicy.CONTINUE, 0, 1, 1}, + new Object[] {FailedBeforeMethodSample.class, XmlSuite.FailurePolicy.SKIP, 0, 1, 1}, + new Object[] {FailedBeforeMethodSample.class, XmlSuite.FailurePolicy.CONTINUE, 0, 1, 1}, + }; + } + + @Test(dataProvider = "dp", description = "GITHUB-2731") + public void configFailureDecidesTheFateOfTheTestMethod( + Class sample, + XmlSuite.FailurePolicy policy, + int passedTests, + int skippedTests, + int configurationFailures) { + + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG testng = create(sample); + testng.addListener(tla); + testng.setConfigFailurePolicy(policy); + testng.run(); + + String context = sample.getSimpleName() + " with configfailurepolicy=" + policy; + assertThat(tla.getPassedTests()).describedAs("passed tests, " + context).hasSize(passedTests); + assertThat(tla.getSkippedTests()) + .describedAs("skipped tests, " + context) + .hasSize(skippedTests); + assertThat(tla.getFailedTests()).describedAs("failed tests, " + context).isEmpty(); + // The failure is always reported, so "continue" never turns a broken setup into a green run. + assertThat(tla.getConfigurationFailures()) + .describedAs("configuration failures, " + context) + .hasSize(configurationFailures); + } +} diff --git a/testng-core/src/test/resources/testng.xml b/testng-core/src/test/resources/testng.xml index 2f6b051de6..4595732e87 100644 --- a/testng-core/src/test/resources/testng.xml +++ b/testng-core/src/test/resources/testng.xml @@ -802,6 +802,7 @@ + From de5e38c1df53490f67cd553a7c7974eca4831d04 Mon Sep 17 00:00:00 2001 From: Julien Herr Date: Tue, 25 Aug 2026 21:56:07 +0200 Subject: [PATCH 2/2] docs(config): describe what configfailurepolicy actually invalidates `XmlSuite.FailurePolicy` and its accessors carried no semantics at all ("Sets the configuration failure policy."), and the website documents `continue` as "continue to execute the remaining tests in the suite" -- which only @BeforeTest actually does. Write down the matrix the preceding commit pins, rather than either of the two half-truths: `continue` narrows a config failure to the failing instance for @BeforeClass and to the failing invocation for @BeforeMethod, a failed @BeforeTest invalidates no instance at all so those test methods run, and a @BeforeSuite failure stops the suite under either policy. The failure is reported in every case, so `continue` never turns a broken setup into a green run. Refs #2731 --- .../main/java/org/testng/cli/CliOptions.java | 6 +++- .../main/java/org/testng/xml/XmlSuite.java | 36 +++++++++++++++++-- .../src/main/java/org/testng/TestNG.java | 4 +-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/testng-cli/src/main/java/org/testng/cli/CliOptions.java b/testng-cli/src/main/java/org/testng/cli/CliOptions.java index ad645643cd..742f06562b 100644 --- a/testng-cli/src/main/java/org/testng/cli/CliOptions.java +++ b/testng-cli/src/main/java/org/testng/cli/CliOptions.java @@ -101,7 +101,11 @@ public class CliOptions { /** Parallel mode (methods, tests or classes). */ public XmlSuite.@Nullable ParallelMode parallelMode; - /** Configuration failure policy (skip or continue). */ + /** + * Whether TestNG keeps attempting configuration methods after one of them has failed once ({@code + * continue}) or skips the remaining ones ({@code skip}). See {@link + * org.testng.xml.XmlSuite.FailurePolicy}. + */ public @Nullable String configFailurePolicy; /** Number of threads to use when running tests in parallel. */ diff --git a/testng-core-api/src/main/java/org/testng/xml/XmlSuite.java b/testng-core-api/src/main/java/org/testng/xml/XmlSuite.java index ab76c43983..1ff1c26e58 100644 --- a/testng-core-api/src/main/java/org/testng/xml/XmlSuite.java +++ b/testng-core-api/src/main/java/org/testng/xml/XmlSuite.java @@ -77,9 +77,37 @@ public String toString() { } } - /** Configuration failure policy options. */ + /** + * Decides what happens to the configuration methods that would have run after one of them has + * already failed, and hence to the test methods they were setting up. + * + *

What each policy does depends on the level of the configuration method that failed: + * + * + * + * + * + * + * + * + *
Fate of a test method whose setup failed
Failing method{@link #SKIP}{@link #CONTINUE}
@BeforeSuiteskippedskipped
@BeforeTestskippedruns
@BeforeClassskippedskipped
@BeforeMethodskippedskipped
+ * + *

The failure itself is always reported, so neither policy turns a broken setup into a green + * run. A @BeforeSuite failure stops the suite whatever the policy says. + */ public enum FailurePolicy { + /** + * Once a configuration method has failed, stop attempting the remaining ones for the whole + * class (and, for a @BeforeTest failure, for every class of the same {@code }). + */ SKIP("skip"), + /** + * Keep attempting a configuration method that has already failed, invalidating only the + * narrowest scope the failure belongs to: the failing instance for @BeforeClass, the + * failing test-method invocation for @BeforeMethod. Sibling instances and sibling test + * methods still get their configuration methods invoked, and a @BeforeTest failure + * invalidates no instance at all, so the test methods of that {@code } run. + */ CONTINUE("continue"); private final String name; @@ -284,7 +312,8 @@ public void setGuiceStage(String guiceStage) { } /** - * Sets the configuration failure policy. + * Sets whether TestNG keeps attempting configuration methods after one of them has failed once, + * or skips the remaining ones. See {@link FailurePolicy} for what each value invalidates. * * @param configFailurePolicy The config failure policy. */ @@ -293,7 +322,8 @@ public void setConfigFailurePolicy(FailurePolicy configFailurePolicy) { } /** - * Returns the configuration failure policy. + * Returns whether TestNG keeps attempting configuration methods after one of them has failed + * once, or skips the remaining ones. See {@link FailurePolicy} for what each value invalidates. * * @return The configuration failure policy. */ diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java index 2989c251f5..cc5ff0665c 100644 --- a/testng-core/src/main/java/org/testng/TestNG.java +++ b/testng-core/src/main/java/org/testng/TestNG.java @@ -2058,8 +2058,8 @@ public void setDefaultTestName(String defaultTestName) { /** * Sets the policy for whether or not to ever invoke a configuration method again after it has - * failed once. Possible values are defined in {@link XmlSuite}. The default value is {@link - * org.testng.xml.XmlSuite.FailurePolicy#SKIP} + * failed once. See {@link org.testng.xml.XmlSuite.FailurePolicy} for what each value invalidates. + * The default value is {@link org.testng.xml.XmlSuite.FailurePolicy#SKIP} * * @param failurePolicy the configuration failure policy */