diff --git a/smart_tests/commands/subset.py b/smart_tests/commands/subset.py index 9f05b4aa9..a8ac85c09 100644 --- a/smart_tests/commands/subset.py +++ b/smart_tests/commands/subset.py @@ -2,6 +2,7 @@ import json import os import pathlib +import random import re import subprocess import sys @@ -44,6 +45,12 @@ class SubsetUseCase(str, Enum): RECURRING = "recurring" +class FallbackMode(str, Enum): + RUN_ALL = "run-all" + STOP = "stop" + RANDOM_SAMPLE = "random-sample" + + class SubsetResult: def __init__( self, @@ -82,6 +89,14 @@ def from_test_paths(cls, test_paths: List[TestPath]) -> 'SubsetResult': is_observation=False ) + @classmethod + def from_random_sample(cls, test_paths: List[TestPath], target: float) -> 'SubsetResult': + count = max(1, round(len(test_paths) * target)) + sampled = random.sample(test_paths, min(count, len(test_paths))) + sampled_set = {id(t): t for t in sampled} + rest = [t for t in test_paths if id(t) not in sampled_set] + return cls(subset=sampled, rest=rest, subset_id='', summary={}, is_brainless=False, is_observation=False) + # Where we take TestPath, we also accept a path name as a string. TestPathLike = str | TestPath @@ -208,6 +223,14 @@ def __init__( "--use-case", hidden=True )] = None, + fallback_mode: Annotated[FallbackMode, typer.Option( + "--fallback-mode", + hidden=True, + help="Behavior when the subset API is unavailable or the model is untrained. " + "'run-all' (default) runs all tests as usual; 'stop' exits with a non-zero status so CI halts; " + "'random-sample' picks a random subset locally based on the count derived from --target " + "(no duration estimates are available in this path).", + )] = FallbackMode.RUN_ALL, test_runner: Annotated[str | None, typer.Argument()] = None, ): super().__init__(app) @@ -288,6 +311,7 @@ def warn(msg: str): self.same_bin_files = list(same_bin_files) self.is_get_tests_from_guess = is_get_tests_from_guess self.use_case = use_case + self.fallback_mode = fallback_mode self._validate_print_input_snapshot_option() @@ -562,6 +586,22 @@ def _collect_potential_test_files(self): if not found: warn_and_exit_if_fail_fast_mode("Nothing that looks like a test file in the current git repository.") + def _fallback_result(self) -> SubsetResult: + if self.fallback_mode == FallbackMode.STOP: + click.echo( + "Warning: Smart Tests could not retrieve a subset. Stopping build (--fallback-mode=stop).", + err=True, + ) + sys.exit(1) + elif self.fallback_mode == FallbackMode.RANDOM_SAMPLE: + target_fraction = float(self.target) if self.target is not None else 1.0 + click.echo( + f"Warning: Smart Tests could not retrieve a subset. Falling back to local random sample at { + target_fraction:.0%}.", err=True, ) + return SubsetResult.from_random_sample(self.test_paths, target_fraction) + else: + return SubsetResult.from_test_paths(self.test_paths) + def request_subset(self) -> SubsetResult: # temporarily extend the timeout because subset API response has become slow # TODO: remove this line when API response return response @@ -597,7 +637,7 @@ def request_subset(self) -> SubsetResult: ) self.client.print_exception_and_recover( e, "Warning: the service failed to subset. Falling back to running all tests") - return SubsetResult.from_test_paths(self.test_paths) + return self._fallback_result() def _requires_test_input(self) -> bool: return ( @@ -680,7 +720,7 @@ def run(self): if not self.session_id: # Session ID in --session is missing. It might be caused by # Launchable API errors. - subset_result = SubsetResult.from_test_paths(self.test_paths) + subset_result = self._fallback_result() else: subset_result = self.request_subset() @@ -697,6 +737,13 @@ def run(self): # TODO(Konboi): split subset isn't provided for smart-tests initial release # if split: # click.echo("subset/{}".format(subset_result.subset_id)) + if subset_result.is_brainless: + click.echo("Your model is currently in training", err=True) + # brainless mode split tests on servers. so we don't have to run + # client side fallback. + if self.fallback_mode != FallbackMode.RANDOM_SAMPLE: + subset_result = self._fallback_result() + output_subset, output_rests = subset_result.subset, subset_result.rest if subset_result.is_observation: @@ -742,10 +789,6 @@ def run(self): ], ] - if subset_result.is_brainless: - click.echo( - "Your model is currently in training", err=True) - click.echo( "Smart Tests created subset {} for build {} (test session {}) in workspace {}/{}".format( subset_result.subset_id, diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index 86d7b4563..5a9fecb2f 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -420,3 +420,103 @@ def assert_tracking_count(self, tracking, count: int): if attempt > 10: break self.assertEqual(tracking.call_count, count) + + +class FallbackModeTest(CliTestCase): + test_files_dir = Path(__file__).parent.joinpath('../data/minitest/').resolve() + + def _subset_args(self, rest_file_name, extra_args=()): + return ( + "subset", "minitest", + "--target", "50%", + "--session", self.session, + "--rest", rest_file_name, + str(self.test_files_dir) + "/test/**/*.rb", + ) + tuple(extra_args) + + # --- API error cases --- + + @responses.activate + @mock.patch.dict(os.environ, {"SMART_TESTS_TOKEN": CliTestCase.smart_tests_token}) + def test_api_error_fallback_stop(self): + responses.replace( + responses.POST, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/subset", + status=500) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "stop")), mix_stderr=False) + self.assertEqual(result.exit_code, 1) + + @responses.activate + @mock.patch.dict(os.environ, {"SMART_TESTS_TOKEN": CliTestCase.smart_tests_token}) + def test_api_error_fallback_random_sample(self): + responses.replace( + responses.POST, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/subset", + status=500) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + self.assert_success(result) + self.assertIn("example_test.rb", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"SMART_TESTS_TOKEN": CliTestCase.smart_tests_token}) + def test_api_error_fallback_run_all_default(self): + responses.replace( + responses.POST, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/subset", + status=500) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name), mix_stderr=False) + self.assert_success(result) + self.assertIn("example_test.rb", result.stdout) + + # --- Brainless mode cases --- + + @responses.activate + @mock.patch.dict(os.environ, {"SMART_TESTS_TOKEN": CliTestCase.smart_tests_token}) + def test_brainless_fallback_stop(self): + responses.replace( + responses.POST, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/subset", + json={"testPaths": [[{"type": "file", "name": "example_test.rb"}]], + "rest": [], "subsettingId": 1, "isBrainless": True, "summary": {}}, + status=200) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "stop")), mix_stderr=False) + self.assertEqual(result.exit_code, 1) + + @responses.activate + @mock.patch.dict(os.environ, {"SMART_TESTS_TOKEN": CliTestCase.smart_tests_token}) + def test_brainless_fallback_random_sample(self): + # In brainless mode the server already split the tests, so random-sample keeps the server's result as-is. + responses.replace( + responses.POST, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/subset", + json={"testPaths": [[{"type": "file", "name": "example_test.rb"}]], + "rest": [], "subsettingId": 1, "isBrainless": True, "summary": {}}, + status=200) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + self.assert_success(result) + self.assertIn("example_test.rb", result.stdout) + + @responses.activate + @mock.patch.dict(os.environ, {"SMART_TESTS_TOKEN": CliTestCase.smart_tests_token}) + def test_brainless_fallback_run_all_default(self): + responses.replace( + responses.POST, + f"{get_base_url()}/intake/organizations/{self.organization}/workspaces/{self.workspace}/subset", + json={"testPaths": [[{"type": "file", "name": "example_test.rb"}]], + "rest": [], "subsettingId": 1, "isBrainless": True, "summary": {}}, + status=200) + + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name), mix_stderr=False) + self.assert_success(result) + self.assertIn("example_test.rb", result.stdout)