From 6808585c423673d9bc1722561f7e5c267224bd92 Mon Sep 17 00:00:00 2001 From: Quinn Date: Wed, 29 Apr 2026 19:51:26 +0000 Subject: [PATCH 01/19] Add compression infra. --- .../tests/package_tests/utils/compress.py | 168 ++++++++++++++++++ .../tests/package_tests/utils/decompress.py | 69 +++++++ integration-tests/tests/utils/classes.py | 39 +++- 3 files changed, 271 insertions(+), 5 deletions(-) create mode 100644 integration-tests/tests/package_tests/utils/compress.py create mode 100644 integration-tests/tests/package_tests/utils/decompress.py diff --git a/integration-tests/tests/package_tests/utils/compress.py b/integration-tests/tests/package_tests/utils/compress.py new file mode 100644 index 0000000000..53fe5319c7 --- /dev/null +++ b/integration-tests/tests/package_tests/utils/compress.py @@ -0,0 +1,168 @@ +"""Functions to facilitate CLP package compression testing.""" + +import logging +from pathlib import Path + +import pytest +from clp_py_utils.clp_config import StorageEngine + +from tests.package_tests.classes import ClpPackage +from tests.package_tests.utils.decompress import decompress_clp_package +from tests.utils.classes import ( + CmdArgs, + ExternalAction, + IntegrationTestDataset, + VerificationResult, +) +from tests.utils.logging_utils import format_action_failure_msg +from tests.utils.utils import clear_directory, is_dir_tree_content_equal + +logger = logging.getLogger(__name__) + + +class CompressArgs(CmdArgs): + """Docstring.""" + + script_path: Path + config: Path + dataset: str | None = None + timestamp_key: str | None = None + unstructured: bool = False + paths: list[Path] + + def to_cmd(self) -> list[str]: + """Docstring.""" + cmd: list[str] = [ + str(self.script_path), + "--config", + str(self.config), + ] + + if self.dataset: + cmd.append("--dataset") + cmd.append(self.dataset) + if self.timestamp_key: + cmd.append("--timestamp-key") + cmd.append(self.timestamp_key) + if self.unstructured: + cmd.append("--unstructured") + + cmd.extend([str(path) for path in self.paths]) + + return cmd + + +def compress_clp_package( + clp_package: ClpPackage, + dataset: IntegrationTestDataset, +) -> ExternalAction: + """Docstring.""" + log_msg = f"Compressing the '{dataset.dataset_name}' dataset." + logger.info(log_msg) + + args: CompressArgs = _construct_compress_args(clp_package, dataset) + return ExternalAction(cmd=args.to_cmd(), args=args) + + +def _construct_compress_args( + clp_package: ClpPackage, dataset: IntegrationTestDataset +) -> CompressArgs: + """Docstring.""" + path_config = clp_package.path_config + args = CompressArgs( + script_path=path_config.compress_path, + config=clp_package.temp_config_file_path, + paths=[dataset.logs_path], + ) + + if clp_package.clp_config.package.storage_engine == StorageEngine.CLP_S: + args.dataset = dataset.metadata.dataset_name + args.timestamp_key = dataset.metadata.timestamp_key + args.unstructured = dataset.metadata.unstructured + + return args + + +def verify_compress_action( + compress_action: ExternalAction, + clp_package: ClpPackage, + original_dataset: IntegrationTestDataset, +) -> VerificationResult: + """Docstring.""" + logger.info("Verifying %s package compression.", clp_package.mode_name) + if compress_action.completed_proc.returncode != 0: + return VerificationResult.fail( + format_action_failure_msg( + "The compress.sh subprocess returned a non-zero exit code.", + compress_action, + ) + ) + + if original_dataset.metadata.unstructured: + return _verify_compress_action_unstructured_logs( + compress_action, clp_package, original_dataset + ) + return _verify_compress_action_structured_logs(compress_action, clp_package) + + +def _verify_compress_action_structured_logs( + compress_action: ExternalAction, clp_package: ClpPackage +) -> VerificationResult: + """Docstring.""" + logger.info("Verifying %s package compression of structured logs.", clp_package.mode_name) + if compress_action.completed_proc.returncode != 0: + return VerificationResult.fail( + format_action_failure_msg( + "The compress.sh subprocess returned a non-zero exit code.", + compress_action, + ) + ) + + # TODO: Waiting for PR 1299 (clp-json decompression) to be merged. + return VerificationResult.ok() + + +def _verify_compress_action_unstructured_logs( + compress_action: ExternalAction, + clp_package: ClpPackage, + original_dataset: IntegrationTestDataset, +) -> VerificationResult: + """Docstring.""" + logger.info("Verifying %s package compression of unstructured logs.", clp_package.mode_name) + if compress_action.completed_proc.returncode != 0: + return VerificationResult.fail( + format_action_failure_msg( + "The 'compress.sh' subprocess returned a non-zero exit code.", + compress_action, + ) + ) + + # Decompress the contents of `clp-package/var/data/archives`. + path_config = clp_package.path_config + clear_directory(path_config.package_decompression_dir) + + decompress_action = decompress_clp_package(clp_package, path_config.package_decompression_dir) + if decompress_action.completed_proc.returncode != 0: + pytest.fail( + "During compress action verification, supporting call to 'decompress.sh' returned a" + f" non-zero exit code. Subprocess log: {decompress_action.log_file_path}" + ) + + # Verify equality between original logs and decompressed logs. + original_logs_path = original_dataset.logs_path + decompressed_logs_path = path_config.package_decompression_dir / original_logs_path.relative_to( + original_logs_path.anchor + ) + + equal = is_dir_tree_content_equal(original_logs_path, decompressed_logs_path) + clear_directory(path_config.package_decompression_dir) + if equal: + return VerificationResult.ok() + + return VerificationResult.fail( + format_action_failure_msg( + f"Compress verification failure: mismatch between original logs at" + f" '{original_logs_path}' and decompressed logs at '{decompressed_logs_path}'.", + compress_action, + ) + ) diff --git a/integration-tests/tests/package_tests/utils/decompress.py b/integration-tests/tests/package_tests/utils/decompress.py new file mode 100644 index 0000000000..1d224a48a8 --- /dev/null +++ b/integration-tests/tests/package_tests/utils/decompress.py @@ -0,0 +1,69 @@ +"""Functions to facilitate CLP package decompression testing.""" + +import logging +from pathlib import Path +from typing import Any + +from clp_package_utils.general import EXTRACT_FILE_CMD + +from tests.package_tests.classes import ClpPackage +from tests.utils.classes import CmdArgs, ExternalAction + +logger = logging.getLogger(__name__) + + +class DecompressArgs(CmdArgs): + """Docstring.""" + + script_path: Path + config: Path + extraction_dir: Path + paths: list[Path] | None = None + + def to_cmd(self) -> list[str]: + """Docstring.""" + cmd: list[str] = [ + str(self.script_path), + "--config", + str(self.config), + EXTRACT_FILE_CMD, + "--extraction-dir", + str(self.extraction_dir), + ] + + if self.paths: + cmd.extend([str(path) for path in self.paths]) + + return cmd + + +# TODO: note that decompress can only be used in conjunction with compress. +def decompress_clp_package( + clp_package: ClpPackage, + extraction_dir: Any, + paths: list[Path] | None = None, +) -> ExternalAction: + """Docstring.""" + logger.info("Decompressing '%s' package.", clp_package.mode_name) + + args: DecompressArgs = _construct_decompress_args(clp_package, extraction_dir, paths) + return ExternalAction(cmd=args.to_cmd(), args=args) + + +def _construct_decompress_args( + clp_package: ClpPackage, + extraction_dir: Any, + paths: list[Path] | None = None, +) -> DecompressArgs: + """Docstring.""" + path_config = clp_package.path_config + args = DecompressArgs( + script_path=path_config.decompress_path, + config=clp_package.temp_config_file_path, + extraction_dir=extraction_dir, + ) + + if paths: + args.paths = paths + + return args diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index 4de0916d0a..96a2107471 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -9,6 +9,7 @@ import pytest from pydantic import BaseModel +from typing_extensions import Self from tests.conftest import get_test_log_dir from tests.utils.utils import validate_dir_exists, validate_file_exists @@ -131,6 +132,14 @@ def logs_path(self) -> Path: return self.dataset_root_dir / self.metadata.logs_subdir +class CmdArgs(BaseModel, ABC): + """Abstract base class for all CLP command argument models.""" + + @abstractmethod + def to_cmd(self) -> list[str]: + """:return: list of command arguments constructed from this instance's data members.""" + + @dataclass class ExternalAction: """Metadata for an external action executed during an integration test.""" @@ -138,6 +147,9 @@ class ExternalAction: #: Command to pass to `subprocess.run()`. cmd: list[str] + #: Optional structured arguments for verification purposes. Not used by `ExternalAction` itself. + args: CmdArgs | None = None + #: The completed process returned from `subprocess.run()`. completed_proc: subprocess.CompletedProcess[str] = field(init=False) @@ -222,9 +234,26 @@ def _log_action_summary_to_file(self) -> None: logger.info(log_msg) -class CmdArgs(BaseModel, ABC): - """Abstract base class for all CLP command argument models.""" +@dataclass(frozen=True) +class VerificationResult: + """Outcome from a verification function.""" - @abstractmethod - def to_cmd(self) -> list[str]: - """:return: list of command arguments constructed from this instance's data members.""" + #: Whether or not the verification was successful. + success: bool + + #: Message describing the failure, if the verification failed. + failure_message: str = "" + + def __bool__(self) -> bool: + """Makes class truthy.""" + return self.success + + @classmethod + def ok(cls) -> Self: + """:return: A successful `VerificationResult`.""" + return cls(success=True) + + @classmethod + def fail(cls, failure_message: str) -> Self: + """:return: A failed `VerificationResult` carrying `failure_message`.""" + return cls(success=False, failure_message=failure_message) From 2151c1579f9014a2eb6b4fc7af3c9ec93d27f1f4 Mon Sep 17 00:00:00 2001 From: Quinn Date: Wed, 29 Apr 2026 21:06:19 +0000 Subject: [PATCH 02/19] Add search infra. --- .../tests/package_tests/utils/search.py | 260 ++++++++++++++++++ integration-tests/tests/utils/classes.py | 39 ++- 2 files changed, 294 insertions(+), 5 deletions(-) create mode 100644 integration-tests/tests/package_tests/utils/search.py diff --git a/integration-tests/tests/package_tests/utils/search.py b/integration-tests/tests/package_tests/utils/search.py new file mode 100644 index 0000000000..72bcff13cb --- /dev/null +++ b/integration-tests/tests/package_tests/utils/search.py @@ -0,0 +1,260 @@ +"""Functions and classes to facilitate CLP package search.""" + +import logging +import re +from enum import auto, Enum +from pathlib import Path + +import pytest +from clp_py_utils.clp_config import StorageEngine + +from tests.package_tests.classes import ClpPackage +from tests.utils.classes import CmdArgs, ExternalAction, IntegrationTestDataset, VerificationResult +from tests.utils.logging_utils import format_action_failure_msg +from tests.utils.utils import get_binary_path + +logger = logging.getLogger(__name__) + + +DEFAULT_COUNT_BY_TIME_INTERVAL = 10 + + +class SearchArgs(CmdArgs): + """Docstring.""" + + script_path: Path + config: Path + wildcard_query: str + raw: bool = True + dataset: str | None = None + file_path: Path | None = None + ignore_case: bool = False + count: bool = False + count_by_time: int | None = None + begin_ts: int | None = None + end_ts: int | None = None + + def to_cmd(self) -> list[str]: + """Docstring.""" + cmd: list[str] = [ + str(self.script_path), + "--config", + str(self.config), + ] + + if self.dataset: + cmd.append("--dataset") + cmd.append(self.dataset) + if self.file_path: + cmd.append("--file-path") + cmd.append(str(self.file_path)) + if self.ignore_case: + cmd.append("--ignore-case") + if self.count: + cmd.append("--count") + if self.count_by_time is not None: + cmd.append("--count-by-time") + cmd.append(str(self.count_by_time)) + if self.begin_ts is not None: + cmd.append("--begin-time") + cmd.append(str(self.begin_ts)) + if self.end_ts is not None: + cmd.append("--end-time") + cmd.append(str(self.end_ts)) + if self.raw: + cmd.append("--raw") + + cmd.append(self.wildcard_query) + + return cmd + + +class ClpPackageSearchType(Enum): + """An enumeration of the types of search we can perform with the CLP package.""" + + BASIC = auto() + FILE_PATH = auto() + IGNORE_CASE = auto() + COUNT_RESULTS = auto() + COUNT_BY_TIME = auto() + TIME_RANGE = auto() + + +def search_clp_package( + clp_package: ClpPackage, + dataset: IntegrationTestDataset, + search_type: ClpPackageSearchType, + wildcard_query: str, +) -> ExternalAction: + """Docstring.""" + logger.info( + "Performing '%s' search on the '%s' dataset.", search_type.name, dataset.dataset_name + ) + + args: SearchArgs = _construct_args(clp_package, dataset, search_type, wildcard_query) + return ExternalAction(cmd=args.to_cmd(), args=args) + + +def _construct_args( + clp_package: ClpPackage, + dataset: IntegrationTestDataset, + search_type: ClpPackageSearchType, + wildcard_query: str, +) -> SearchArgs: + """Docstring.""" + path_config = clp_package.path_config + args = SearchArgs( + script_path=path_config.search_path, + config=clp_package.temp_config_file_path, + wildcard_query=wildcard_query, + ) + + if clp_package.clp_config.package.storage_engine == StorageEngine.CLP_S: + args.dataset = dataset.metadata.dataset_name + + match search_type: + case ClpPackageSearchType.BASIC: + pass + case ClpPackageSearchType.FILE_PATH: + args.file_path = dataset.logs_path / dataset.metadata.file_names[0] + case ClpPackageSearchType.IGNORE_CASE: + args.ignore_case = True + case ClpPackageSearchType.COUNT_RESULTS: + args.count = True + case ClpPackageSearchType.COUNT_BY_TIME: + args.count_by_time = DEFAULT_COUNT_BY_TIME_INTERVAL + case ClpPackageSearchType.TIME_RANGE: + args.begin_ts = dataset.metadata.begin_ts + args.end_ts = dataset.metadata.end_ts + case _: + pytest.fail(f"Unsupported search type for CLP package: '{search_type}'") + + return args + + +def verify_search_action( + action: ExternalAction, + search_type: ClpPackageSearchType, + original_dataset: IntegrationTestDataset, +) -> VerificationResult: + """Docstring.""" + logger.info("Verifying search.") + if action.completed_proc.returncode != 0: + return VerificationResult.fail( + format_action_failure_msg( + "The 'search.sh' subprocess returned a non-zero exit code.", action + ) + ) + + args = action.args + assert isinstance(args, SearchArgs) + + # Construct and run grep command. + grep_action = ExternalAction( + cmd=_construct_grep_verification_cmd(args, search_type, original_dataset) + ) + + if grep_action.completed_proc.returncode != 0: + pytest.fail( + "During search action verification, internal grep command returned a non-zero exit" + f" code. Subprocess log: {grep_action.log_file_path}" + ) + + # Compare grep result with search result. + formatted_grep_result = _format_grep_result_for_search_type( + grep_action.completed_proc.stdout, search_type + ) + formatted_search_result = _format_search_result_for_search_type( + action.completed_proc.stdout, search_type + ) + if formatted_grep_result == formatted_search_result: + return VerificationResult.ok() + + return VerificationResult.fail( + format_action_failure_msg( + f"Search verification failure: mismatch between formatted search result" + f" '{formatted_search_result}' and formatted grep result" + f" '{formatted_grep_result}'.", + action, + grep_action, + ) + ) + + +def _construct_grep_verification_cmd( + args: SearchArgs, + search_type: ClpPackageSearchType, + original_dataset: IntegrationTestDataset, +) -> list[str]: + grep_cmd_options = _get_grep_options_from_search_type(search_type) + path_for_grep = args.file_path or original_dataset.logs_path + return [ + get_binary_path("grep"), + *grep_cmd_options, + args.wildcard_query, + str(path_for_grep), + ] + + +def _get_grep_options_from_search_type(search_type: ClpPackageSearchType) -> list[str]: + grep_cmd_options: list[str] = [ + "--recursive", + "--no-filename", + "--color=never", + ] + + match search_type: + case ( + ClpPackageSearchType.BASIC + | ClpPackageSearchType.FILE_PATH + | ClpPackageSearchType.COUNT_RESULTS + | ClpPackageSearchType.COUNT_BY_TIME + | ClpPackageSearchType.TIME_RANGE + ): + return grep_cmd_options + case ClpPackageSearchType.IGNORE_CASE: + grep_cmd_options.append("--ignore-case") + return grep_cmd_options + case _: + pytest.fail( + f"Search type '{search_type.name}' not configured for grep command construction." + ) + + +def _format_grep_result_for_search_type(grep_result: str, search_type: ClpPackageSearchType) -> str: + match search_type: + case ( + ClpPackageSearchType.BASIC + | ClpPackageSearchType.FILE_PATH + | ClpPackageSearchType.IGNORE_CASE + | ClpPackageSearchType.TIME_RANGE + ): + return grep_result + case ClpPackageSearchType.COUNT_RESULTS | ClpPackageSearchType.COUNT_BY_TIME: + return str(len(grep_result.splitlines())) + "\n" + case _: + pytest.fail( + f"Search type '{search_type.name}' not configured for grep result formatting." + ) + + +def _format_search_result_for_search_type( + search_result: str, search_type: ClpPackageSearchType +) -> str: + match search_type: + case ( + ClpPackageSearchType.BASIC + | ClpPackageSearchType.FILE_PATH + | ClpPackageSearchType.IGNORE_CASE + | ClpPackageSearchType.TIME_RANGE + ): + return search_result + case ClpPackageSearchType.COUNT_RESULTS | ClpPackageSearchType.COUNT_BY_TIME: + match = re.search(r"count: (\d+)", search_result) + if match: + return match.group(1) + "\n" + pytest.fail(f"The search result '{search_result}' wasn't in the correct format.") + case _: + pytest.fail( + f"Search type '{search_type.name}' not configured for search result formatting." + ) diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index 4de0916d0a..96a2107471 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -9,6 +9,7 @@ import pytest from pydantic import BaseModel +from typing_extensions import Self from tests.conftest import get_test_log_dir from tests.utils.utils import validate_dir_exists, validate_file_exists @@ -131,6 +132,14 @@ def logs_path(self) -> Path: return self.dataset_root_dir / self.metadata.logs_subdir +class CmdArgs(BaseModel, ABC): + """Abstract base class for all CLP command argument models.""" + + @abstractmethod + def to_cmd(self) -> list[str]: + """:return: list of command arguments constructed from this instance's data members.""" + + @dataclass class ExternalAction: """Metadata for an external action executed during an integration test.""" @@ -138,6 +147,9 @@ class ExternalAction: #: Command to pass to `subprocess.run()`. cmd: list[str] + #: Optional structured arguments for verification purposes. Not used by `ExternalAction` itself. + args: CmdArgs | None = None + #: The completed process returned from `subprocess.run()`. completed_proc: subprocess.CompletedProcess[str] = field(init=False) @@ -222,9 +234,26 @@ def _log_action_summary_to_file(self) -> None: logger.info(log_msg) -class CmdArgs(BaseModel, ABC): - """Abstract base class for all CLP command argument models.""" +@dataclass(frozen=True) +class VerificationResult: + """Outcome from a verification function.""" - @abstractmethod - def to_cmd(self) -> list[str]: - """:return: list of command arguments constructed from this instance's data members.""" + #: Whether or not the verification was successful. + success: bool + + #: Message describing the failure, if the verification failed. + failure_message: str = "" + + def __bool__(self) -> bool: + """Makes class truthy.""" + return self.success + + @classmethod + def ok(cls) -> Self: + """:return: A successful `VerificationResult`.""" + return cls(success=True) + + @classmethod + def fail(cls, failure_message: str) -> Self: + """:return: A failed `VerificationResult` carrying `failure_message`.""" + return cls(success=False, failure_message=failure_message) From d82b7930b6539f4565038caf6fd4306c7b0a4caf Mon Sep 17 00:00:00 2001 From: Quinn Date: Wed, 29 Apr 2026 21:42:56 +0000 Subject: [PATCH 03/19] Docstrings. --- .../tests/package_tests/utils/compress.py | 34 ++++++++++++++----- .../tests/package_tests/utils/decompress.py | 19 +++++++---- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/integration-tests/tests/package_tests/utils/compress.py b/integration-tests/tests/package_tests/utils/compress.py index 53fe5319c7..f60b693255 100644 --- a/integration-tests/tests/package_tests/utils/compress.py +++ b/integration-tests/tests/package_tests/utils/compress.py @@ -21,7 +21,7 @@ class CompressArgs(CmdArgs): - """Docstring.""" + """Command argument model for compressing with the CLP package.""" script_path: Path config: Path @@ -31,7 +31,7 @@ class CompressArgs(CmdArgs): paths: list[Path] def to_cmd(self) -> list[str]: - """Docstring.""" + """Converts the model attributes to a command list.""" cmd: list[str] = [ str(self.script_path), "--config", @@ -56,9 +56,18 @@ def compress_clp_package( clp_package: ClpPackage, dataset: IntegrationTestDataset, ) -> ExternalAction: - """Docstring.""" - log_msg = f"Compressing the '{dataset.dataset_name}' dataset." - logger.info(log_msg) + """ + Compresses the specified dataset into a CLP package. + + :param clp_package: + :param dataset: + :return: The `ExternalAction` instance that runs the compression. + """ + logger.info( + "Compressing the '%s' dataset with the '%s' package.", + dataset.dataset_name, + clp_package.mode_name, + ) args: CompressArgs = _construct_compress_args(clp_package, dataset) return ExternalAction(cmd=args.to_cmd(), args=args) @@ -67,7 +76,7 @@ def compress_clp_package( def _construct_compress_args( clp_package: ClpPackage, dataset: IntegrationTestDataset ) -> CompressArgs: - """Docstring.""" + """Construct the `CompressArgs` object for compressing the specified dataset.""" path_config = clp_package.path_config args = CompressArgs( script_path=path_config.compress_path, @@ -88,7 +97,14 @@ def verify_compress_action( clp_package: ClpPackage, original_dataset: IntegrationTestDataset, ) -> VerificationResult: - """Docstring.""" + """ + Verifies the compression action. + + :param compress_action: + :param clp_package: + :param original_dataset: + :return: A `VerificationResult` indicating the success or failure of the verification. + """ logger.info("Verifying %s package compression.", clp_package.mode_name) if compress_action.completed_proc.returncode != 0: return VerificationResult.fail( @@ -108,7 +124,7 @@ def verify_compress_action( def _verify_compress_action_structured_logs( compress_action: ExternalAction, clp_package: ClpPackage ) -> VerificationResult: - """Docstring.""" + """Verifies the compression of structured logs.""" logger.info("Verifying %s package compression of structured logs.", clp_package.mode_name) if compress_action.completed_proc.returncode != 0: return VerificationResult.fail( @@ -127,7 +143,7 @@ def _verify_compress_action_unstructured_logs( clp_package: ClpPackage, original_dataset: IntegrationTestDataset, ) -> VerificationResult: - """Docstring.""" + """Verifies the compression of unstructured logs.""" logger.info("Verifying %s package compression of unstructured logs.", clp_package.mode_name) if compress_action.completed_proc.returncode != 0: return VerificationResult.fail( diff --git a/integration-tests/tests/package_tests/utils/decompress.py b/integration-tests/tests/package_tests/utils/decompress.py index 1d224a48a8..04a88f0358 100644 --- a/integration-tests/tests/package_tests/utils/decompress.py +++ b/integration-tests/tests/package_tests/utils/decompress.py @@ -13,7 +13,7 @@ class DecompressArgs(CmdArgs): - """Docstring.""" + """Command argument model for decompressing with the CLP package.""" script_path: Path config: Path @@ -21,7 +21,7 @@ class DecompressArgs(CmdArgs): paths: list[Path] | None = None def to_cmd(self) -> list[str]: - """Docstring.""" + """Converts the model attributes to a command list.""" cmd: list[str] = [ str(self.script_path), "--config", @@ -37,14 +37,21 @@ def to_cmd(self) -> list[str]: return cmd -# TODO: note that decompress can only be used in conjunction with compress. def decompress_clp_package( clp_package: ClpPackage, extraction_dir: Any, paths: list[Path] | None = None, ) -> ExternalAction: - """Docstring.""" - logger.info("Decompressing '%s' package.", clp_package.mode_name) + """ + Decompresses the specified CLP package archives. Note that decompression can only be used in + conjunction with compression. + + :param clp_package: + :param extraction_dir: + :param paths: + :return: The `ExternalAction` instance that runs the decompression. + """ + logger.info("Decompressing the '%s' package archives.", clp_package.mode_name) args: DecompressArgs = _construct_decompress_args(clp_package, extraction_dir, paths) return ExternalAction(cmd=args.to_cmd(), args=args) @@ -55,7 +62,7 @@ def _construct_decompress_args( extraction_dir: Any, paths: list[Path] | None = None, ) -> DecompressArgs: - """Docstring.""" + """Constructs the `DecompressArgs` object for decompressing the specified CLP package.""" path_config = clp_package.path_config args = DecompressArgs( script_path=path_config.decompress_path, From 3b62e1ff67b7b23320d85c418b0bf444458e0e06 Mon Sep 17 00:00:00 2001 From: Quinn Date: Wed, 29 Apr 2026 23:22:56 +0000 Subject: [PATCH 04/19] Docstrings. --- .../tests/package_tests/utils/search.py | 45 +++++++++++++++---- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/integration-tests/tests/package_tests/utils/search.py b/integration-tests/tests/package_tests/utils/search.py index 72bcff13cb..3abca14a14 100644 --- a/integration-tests/tests/package_tests/utils/search.py +++ b/integration-tests/tests/package_tests/utils/search.py @@ -9,7 +9,12 @@ from clp_py_utils.clp_config import StorageEngine from tests.package_tests.classes import ClpPackage -from tests.utils.classes import CmdArgs, ExternalAction, IntegrationTestDataset, VerificationResult +from tests.utils.classes import ( + CmdArgs, + ExternalAction, + IntegrationTestDataset, + VerificationResult, +) from tests.utils.logging_utils import format_action_failure_msg from tests.utils.utils import get_binary_path @@ -20,7 +25,7 @@ class SearchArgs(CmdArgs): - """Docstring.""" + """Command argument model for searching with the CLP package.""" script_path: Path config: Path @@ -35,7 +40,7 @@ class SearchArgs(CmdArgs): end_ts: int | None = None def to_cmd(self) -> list[str]: - """Docstring.""" + """Converts the model attributes to a command list.""" cmd: list[str] = [ str(self.script_path), "--config", @@ -70,7 +75,7 @@ def to_cmd(self) -> list[str]: class ClpPackageSearchType(Enum): - """An enumeration of the types of search we can perform with the CLP package.""" + """Possible search types.""" BASIC = auto() FILE_PATH = auto() @@ -86,9 +91,19 @@ def search_clp_package( search_type: ClpPackageSearchType, wildcard_query: str, ) -> ExternalAction: - """Docstring.""" + """ + Performs the specified search on the dataset using the CLP package. + + :param clp_package: + :param dataset: + :param search_type: + :param wildcard_query: + :return: The `ExternalAction` instance that runs the search. + """ logger.info( - "Performing '%s' search on the '%s' dataset.", search_type.name, dataset.dataset_name + "Performing '%s' search on the '%s' dataset.", + search_type.name, + dataset.dataset_name, ) args: SearchArgs = _construct_args(clp_package, dataset, search_type, wildcard_query) @@ -101,7 +116,7 @@ def _construct_args( search_type: ClpPackageSearchType, wildcard_query: str, ) -> SearchArgs: - """Docstring.""" + """Construct the `SearchArgs` object for the specified search on the dataset.""" path_config = clp_package.path_config args = SearchArgs( script_path=path_config.search_path, @@ -137,8 +152,20 @@ def verify_search_action( search_type: ClpPackageSearchType, original_dataset: IntegrationTestDataset, ) -> VerificationResult: - """Docstring.""" - logger.info("Verifying search.") + """ + Verifies the search action. + + :param action: + :param search_type: + :param original_dataset: + :return: A `VerificationResult` indicating the success or failure of the verification. + """ + logger.info( + "Verifying '%s' search on the '%s' dataset.", + search_type.name, + original_dataset.dataset_name, + ) + if action.completed_proc.returncode != 0: return VerificationResult.fail( format_action_failure_msg( From 4979cb7d550051c38ade8c2684519bbba6ba1714 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 30 Apr 2026 14:45:31 +0000 Subject: [PATCH 05/19] Rabbit. --- .../tests/package_tests/utils/compress.py | 31 ++++--------------- .../tests/package_tests/utils/decompress.py | 13 +++----- 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/integration-tests/tests/package_tests/utils/compress.py b/integration-tests/tests/package_tests/utils/compress.py index f60b693255..0f14a7d012 100644 --- a/integration-tests/tests/package_tests/utils/compress.py +++ b/integration-tests/tests/package_tests/utils/compress.py @@ -76,7 +76,7 @@ def compress_clp_package( def _construct_compress_args( clp_package: ClpPackage, dataset: IntegrationTestDataset ) -> CompressArgs: - """Construct the `CompressArgs` object for compressing the specified dataset.""" + """Constructs the `CompressArgs` object for compressing the specified dataset.""" path_config = clp_package.path_config args = CompressArgs( script_path=path_config.compress_path, @@ -105,7 +105,8 @@ def verify_compress_action( :param original_dataset: :return: A `VerificationResult` indicating the success or failure of the verification. """ - logger.info("Verifying %s package compression.", clp_package.mode_name) + logger.info("Verifying '%s' package compression.", clp_package.mode_name) + if compress_action.completed_proc.returncode != 0: return VerificationResult.fail( format_action_failure_msg( @@ -118,22 +119,11 @@ def verify_compress_action( return _verify_compress_action_unstructured_logs( compress_action, clp_package, original_dataset ) - return _verify_compress_action_structured_logs(compress_action, clp_package) + return _verify_compress_action_structured_logs() -def _verify_compress_action_structured_logs( - compress_action: ExternalAction, clp_package: ClpPackage -) -> VerificationResult: +def _verify_compress_action_structured_logs() -> VerificationResult: """Verifies the compression of structured logs.""" - logger.info("Verifying %s package compression of structured logs.", clp_package.mode_name) - if compress_action.completed_proc.returncode != 0: - return VerificationResult.fail( - format_action_failure_msg( - "The compress.sh subprocess returned a non-zero exit code.", - compress_action, - ) - ) - # TODO: Waiting for PR 1299 (clp-json decompression) to be merged. return VerificationResult.ok() @@ -144,19 +134,10 @@ def _verify_compress_action_unstructured_logs( original_dataset: IntegrationTestDataset, ) -> VerificationResult: """Verifies the compression of unstructured logs.""" - logger.info("Verifying %s package compression of unstructured logs.", clp_package.mode_name) - if compress_action.completed_proc.returncode != 0: - return VerificationResult.fail( - format_action_failure_msg( - "The 'compress.sh' subprocess returned a non-zero exit code.", - compress_action, - ) - ) + path_config = clp_package.path_config # Decompress the contents of `clp-package/var/data/archives`. - path_config = clp_package.path_config clear_directory(path_config.package_decompression_dir) - decompress_action = decompress_clp_package(clp_package, path_config.package_decompression_dir) if decompress_action.completed_proc.returncode != 0: pytest.fail( diff --git a/integration-tests/tests/package_tests/utils/decompress.py b/integration-tests/tests/package_tests/utils/decompress.py index 04a88f0358..8dae387a8f 100644 --- a/integration-tests/tests/package_tests/utils/decompress.py +++ b/integration-tests/tests/package_tests/utils/decompress.py @@ -2,7 +2,6 @@ import logging from pathlib import Path -from typing import Any from clp_package_utils.general import EXTRACT_FILE_CMD @@ -39,7 +38,7 @@ def to_cmd(self) -> list[str]: def decompress_clp_package( clp_package: ClpPackage, - extraction_dir: Any, + extraction_dir: Path, paths: list[Path] | None = None, ) -> ExternalAction: """ @@ -59,18 +58,14 @@ def decompress_clp_package( def _construct_decompress_args( clp_package: ClpPackage, - extraction_dir: Any, + extraction_dir: Path, paths: list[Path] | None = None, ) -> DecompressArgs: """Constructs the `DecompressArgs` object for decompressing the specified CLP package.""" path_config = clp_package.path_config - args = DecompressArgs( + return DecompressArgs( script_path=path_config.decompress_path, config=clp_package.temp_config_file_path, extraction_dir=extraction_dir, + paths=paths, ) - - if paths: - args.paths = paths - - return args From f9d3189dea99baf40b498914c516ae9fedcd6881 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 30 Apr 2026 16:31:26 +0000 Subject: [PATCH 06/19] Rabbit. --- integration-tests/tests/package_tests/utils/search.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/package_tests/utils/search.py b/integration-tests/tests/package_tests/utils/search.py index 3abca14a14..d2b9d365b8 100644 --- a/integration-tests/tests/package_tests/utils/search.py +++ b/integration-tests/tests/package_tests/utils/search.py @@ -174,14 +174,17 @@ def verify_search_action( ) args = action.args - assert isinstance(args, SearchArgs) + if not isinstance(args, SearchArgs): + pytest.fail( + "Search verification requires `ExternalAction.args` to be a SearchArgs instance." + ) # Construct and run grep command. grep_action = ExternalAction( cmd=_construct_grep_verification_cmd(args, search_type, original_dataset) ) - if grep_action.completed_proc.returncode != 0: + if grep_action.completed_proc.returncode not in (0, 1): pytest.fail( "During search action verification, internal grep command returned a non-zero exit" f" code. Subprocess log: {grep_action.log_file_path}" From 5d1cae709edb200364d6fb5c050ff1e4c068aa2f Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 30 Apr 2026 16:56:56 +0000 Subject: [PATCH 07/19] Rabbit. --- integration-tests/tests/package_tests/utils/compress.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/package_tests/utils/compress.py b/integration-tests/tests/package_tests/utils/compress.py index 0f14a7d012..f33e20edb7 100644 --- a/integration-tests/tests/package_tests/utils/compress.py +++ b/integration-tests/tests/package_tests/utils/compress.py @@ -141,8 +141,12 @@ def _verify_compress_action_unstructured_logs( decompress_action = decompress_clp_package(clp_package, path_config.package_decompression_dir) if decompress_action.completed_proc.returncode != 0: pytest.fail( - "During compress action verification, supporting call to 'decompress.sh' returned a" - f" non-zero exit code. Subprocess log: {decompress_action.log_file_path}" + format_action_failure_msg( + "During compress action verification, supporting call to 'decompress.sh' returned a" + " non-zero exit code.", + compress_action, + decompress_action, + ) ) # Verify equality between original logs and decompressed logs. @@ -161,5 +165,6 @@ def _verify_compress_action_unstructured_logs( f"Compress verification failure: mismatch between original logs at" f" '{original_logs_path}' and decompressed logs at '{decompressed_logs_path}'.", compress_action, + decompress_action, ) ) From 89677f8424b290e9c31d940d5b2492092575dd07 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 30 Apr 2026 18:00:18 +0000 Subject: [PATCH 08/19] Make FILE_PATH search less flimsy. --- integration-tests/tests/data/json_multifile/metadata.json | 3 ++- integration-tests/tests/data/text_multifile/metadata.json | 3 ++- integration-tests/tests/package_tests/utils/search.py | 2 +- integration-tests/tests/utils/classes.py | 8 ++++++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/integration-tests/tests/data/json_multifile/metadata.json b/integration-tests/tests/data/json_multifile/metadata.json index 39cbe4b33d..0dd22a8158 100644 --- a/integration-tests/tests/data/json_multifile/metadata.json +++ b/integration-tests/tests/data/json_multifile/metadata.json @@ -12,5 +12,6 @@ "sts-135-2011-07-19.jsonl", "sts-135-2011-07-21.jsonl" ], - "single_match_wildcard_query": "\"detail\":\"Roll program complete, heads down attitude achieved for ascent\"" + "single_match_wildcard_query": "\"detail\":\"Roll program complete, heads down attitude achieved for ascent\"", + "single_match_file": "sts-135-2011-07-08.jsonl" } diff --git a/integration-tests/tests/data/text_multifile/metadata.json b/integration-tests/tests/data/text_multifile/metadata.json index 0f075c969a..1e569e8199 100644 --- a/integration-tests/tests/data/text_multifile/metadata.json +++ b/integration-tests/tests/data/text_multifile/metadata.json @@ -12,5 +12,6 @@ "apollo-17_day10.txt", "apollo-17_day13.txt" ], - "single_match_wildcard_query": "Saturn" + "single_match_wildcard_query": "Saturn", + "single_match_file": "apollo-17_day01.txt" } diff --git a/integration-tests/tests/package_tests/utils/search.py b/integration-tests/tests/package_tests/utils/search.py index d2b9d365b8..698bb97872 100644 --- a/integration-tests/tests/package_tests/utils/search.py +++ b/integration-tests/tests/package_tests/utils/search.py @@ -131,7 +131,7 @@ def _construct_args( case ClpPackageSearchType.BASIC: pass case ClpPackageSearchType.FILE_PATH: - args.file_path = dataset.logs_path / dataset.metadata.file_names[0] + args.file_path = dataset.logs_path / dataset.metadata.single_match_file case ClpPackageSearchType.IGNORE_CASE: args.ignore_case = True case ClpPackageSearchType.COUNT_RESULTS: diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index 96a2107471..d4ddcb4a0b 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -80,6 +80,7 @@ class IntegrationTestDatasetMetadata(BaseModel): logs_subdir: str file_names: list[str] single_match_wildcard_query: str + single_match_file: str @dataclass @@ -121,6 +122,13 @@ def __post_init__(self) -> None: file_path_abs = self.logs_path / file_path validate_file_exists(file_path_abs) + if self.metadata.single_match_file not in self.metadata.file_names: + err_msg = ( + f"`single_match_file` '{self.metadata.single_match_file}' is not listed in" + " `file_names`." + ) + raise ValueError(err_msg) + @property def metadata_file_path(self) -> Path: """:return: The absolute path to the file containing metadata for the dataset.""" From d2d2ef0b9e31e7188341ca8f108e8850c37e010d Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 12 May 2026 17:02:10 +0000 Subject: [PATCH 09/19] Deprecate PackageCompressionJob and related flows. --- .../package_tests/clp_json/test_clp_json.py | 22 +------ .../package_tests/clp_text/test_clp_text.py | 17 +---- .../tests/utils/asserting_utils.py | 65 +------------------ integration-tests/tests/utils/config.py | 14 ---- .../tests/utils/package_utils.py | 37 +---------- 5 files changed, 6 insertions(+), 149 deletions(-) diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index 67f7105d97..8b1f9ecd8f 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -7,10 +7,8 @@ from tests.package_tests.clp_json.utils.mode import CLP_JSON_MODE from tests.utils.asserting_utils import ( validate_package_running, - verify_package_compression, ) -from tests.utils.config import PackageCompressionJob, PackageInstance -from tests.utils.package_utils import run_package_compression_script +from tests.utils.config import PackageInstance logger = logging.getLogger(__name__) @@ -53,23 +51,7 @@ def test_clp_json_compression_json_multifile(fixt_package_instance: PackageInsta package_path_config = package_test_config.path_config package_path_config.clear_package_archives() - # Compress a dataset. - compression_job = PackageCompressionJob( - path_to_original_dataset=( - package_path_config.clp_json_test_data_path / "json-multifile" / "logs" - ), - options=[ - "--timestamp-key", - "timestamp", - "--dataset", - "json_multifile", - ], - positional_args=None, - ) - run_package_compression_script(compression_job, package_test_config) - - # Check the correctness of compression. - verify_package_compression(compression_job.path_to_original_dataset, package_test_config) + # TODO: Compress a dataset. # Clear archives. package_path_config.clear_package_archives() diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index 1fe4dea731..4da3f44972 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -7,10 +7,8 @@ from tests.package_tests.clp_text.utils.mode import CLP_TEXT_MODE from tests.utils.asserting_utils import ( validate_package_running, - verify_package_compression, ) -from tests.utils.config import PackageCompressionJob, PackageInstance -from tests.utils.package_utils import run_package_compression_script +from tests.utils.config import PackageInstance logger = logging.getLogger(__name__) @@ -49,18 +47,7 @@ def test_clp_text_compression_text_multifile(fixt_package_instance: PackageInsta package_path_config = package_test_config.path_config package_path_config.clear_package_archives() - # Compress a dataset. - compression_job = PackageCompressionJob( - path_to_original_dataset=( - package_path_config.clp_text_test_data_path / "text-multifile" / "logs" - ), - options=None, - positional_args=None, - ) - run_package_compression_script(compression_job, package_test_config) - - # Check the correctness of compression. - verify_package_compression(compression_job.path_to_original_dataset, package_test_config) + # TODO: Compress a dataset. # Clear archives. package_path_config.clear_package_archives() diff --git a/integration-tests/tests/utils/asserting_utils.py b/integration-tests/tests/utils/asserting_utils.py index 6515e3b913..b854cc23eb 100644 --- a/integration-tests/tests/utils/asserting_utils.py +++ b/integration-tests/tests/utils/asserting_utils.py @@ -1,18 +1,11 @@ """Utilities that raise pytest assertions on failure.""" import logging -from pathlib import Path import pytest -from clp_package_utils.general import EXTRACT_FILE_CMD -from tests.utils.config import PackageInstance, PackageTestConfig +from tests.utils.config import PackageInstance from tests.utils.docker_utils import list_running_services_in_compose_project -from tests.utils.subprocess_utils import run_and_log_subprocess -from tests.utils.utils import ( - clear_directory, - is_dir_tree_content_equal, -) logger = logging.getLogger(__name__) @@ -51,59 +44,3 @@ def validate_package_running(package_instance: PackageInstance) -> None: fail_msg += f"\nUnexpected services: {unexpected_components}." pytest.fail(fail_msg) - - -def verify_package_compression( - path_to_original_dataset: Path, - package_test_config: PackageTestConfig, -) -> None: - """ - Verify that compression has been executed correctly by decompressing the contents of - `clp-package/var/data/archives` and comparing the decompressed logs to the originals stored at - `path_to_original_dataset`. - - :param path_to_original_dataset: - :param package_test_config: - """ - mode = package_test_config.mode_config.mode_name - log_msg = f"Verifying {mode} package compression." - logger.info(log_msg) - - if mode == "clp-json": - # TODO: Waiting for PR 1299 to be merged. - assert True - elif mode == "clp-text": - # Decompress the contents of `clp-package/var/data/archives`. - path_config = package_test_config.path_config - decompress_script_path = path_config.decompress_script_path - decompression_dir = path_config.package_decompression_dir - temp_config_file_path = package_test_config.temp_config_file_path - - clear_directory(decompression_dir) - - decompress_cmd = [ - str(decompress_script_path), - "--config", - str(temp_config_file_path), - EXTRACT_FILE_CMD, - "--extraction-dir", - str(decompression_dir), - ] - - # Run decompression command and assert that it succeeds. - run_and_log_subprocess(decompress_cmd) - - # Verify content equality. - output_path = decompression_dir / path_to_original_dataset.relative_to( - path_to_original_dataset.anchor - ) - - try: - if not is_dir_tree_content_equal(path_to_original_dataset, output_path): - err_msg = ( - f"Mismatch between clp input {path_to_original_dataset} and output" - f" {output_path}." - ) - pytest.fail(err_msg) - finally: - clear_directory(decompression_dir) diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 314b1387ae..7987973d71 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -161,20 +161,6 @@ def clear_package_archives(self) -> None: clear_directory(archives_dir) -@dataclass(frozen=True) -class PackageCompressionJob: - """A compression job for a package test.""" - - #: The absolute path to the dataset (either a file or directory). - path_to_original_dataset: Path - - #: Options to specify in the compression command. - options: list[str] | None - - #: Positional arguments to specify in the compression command (do not put paths to compress) - positional_args: list[str] | None - - @dataclass(frozen=True) class PackageModeConfig: """Mode configuration for the CLP package.""" diff --git a/integration-tests/tests/utils/package_utils.py b/integration-tests/tests/utils/package_utils.py index b578676ed4..df5304ae05 100644 --- a/integration-tests/tests/utils/package_utils.py +++ b/integration-tests/tests/utils/package_utils.py @@ -1,9 +1,6 @@ """Provides utility functions related to the CLP package used across `integration-tests`.""" -from tests.utils.config import ( - PackageCompressionJob, - PackageTestConfig, -) +from tests.utils.config import PackageTestConfig from tests.utils.subprocess_utils import run_and_log_subprocess @@ -45,35 +42,3 @@ def stop_clp_package(package_test_config: PackageTestConfig) -> None: ] # fmt: on run_and_log_subprocess(stop_cmd) - - -def run_package_compression_script( - compression_job: PackageCompressionJob, - package_test_config: PackageTestConfig, -) -> None: - """ - Constructs and runs a compression command on the CLP package. - - :param compression_job: - :param package_test_config: - """ - path_config = package_test_config.path_config - compress_script_path = path_config.compress_script_path - temp_config_file_path = package_test_config.temp_config_file_path - - compress_cmd = [ - str(compress_script_path), - "--config", - str(temp_config_file_path), - ] - - if compression_job.options is not None: - compress_cmd.extend(compression_job.options) - - if compression_job.positional_args is not None: - compress_cmd.extend(compression_job.positional_args) - - compress_cmd.append(str(compression_job.path_to_original_dataset)) - - # Run compression command for this job and assert that it succeeds. - run_and_log_subprocess(compress_cmd) From 8e02f435a975dc120073102bc27222a64ba0d4bb Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 21 May 2026 14:52:17 +0000 Subject: [PATCH 10/19] Clear package archives after spindown. --- integration-tests/tests/package_tests/fixtures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/tests/package_tests/fixtures.py b/integration-tests/tests/package_tests/fixtures.py index 2008811188..bc1a448211 100644 --- a/integration-tests/tests/package_tests/fixtures.py +++ b/integration-tests/tests/package_tests/fixtures.py @@ -91,4 +91,5 @@ def clp_package( stop_result = verify_stop_clp_action(stop_clp_action, clp_package) assert stop_result, stop_result.failure_message + clp_package_test_path_config.clear_package_archives() clp_package.temp_config_file_path.unlink(missing_ok=True) From e3866ecb1a7e697457b3a20e8fdfca12adb43a81 Mon Sep 17 00:00:00 2001 From: Quinn Date: Tue, 26 May 2026 15:45:10 +0000 Subject: [PATCH 11/19] Cleanup --- integration-tests/tests/package_tests/classes.py | 2 +- .../tests/package_tests/clp_json/test_clp_json.py | 13 +++++-------- .../tests/package_tests/clp_text/test_clp_text.py | 10 ++-------- integration-tests/tests/package_tests/fixtures.py | 15 +++++++++++++++ 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/integration-tests/tests/package_tests/classes.py b/integration-tests/tests/package_tests/classes.py index caefab1ebd..19c01ad170 100644 --- a/integration-tests/tests/package_tests/classes.py +++ b/integration-tests/tests/package_tests/classes.py @@ -118,7 +118,7 @@ def _static_paths(self) -> list[Path]: self.stop_clp_path, ] - def clear_package_archives(self) -> None: + def clear_archives(self) -> None: """Removes the contents of the package archives directory.""" # TODO: this method will be replaced with a more robust version that uses `archive-manager` # or `dataset-manager` (as appropriate) to clear archives correctly. diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index a3e4643ccf..86e4f95f02 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -19,7 +19,9 @@ pytestmark = [ pytest.mark.package, pytest.mark.clp_json, - pytest.mark.parametrize("clp_package", [CLP_JSON_MODE], indirect=True), + pytest.mark.parametrize( + "clp_package", [CLP_JSON_MODE], indirect=True, ids=[CLP_JSON_MODE.mode_name] + ), ] @@ -38,6 +40,7 @@ def test_clp_json_startup(clp_package: ClpPackage) -> None: @pytest.mark.compression +@pytest.mark.usefixtures("clear_package_archives") def test_clp_json_compression_json_multifile( clp_package: ClpPackage, json_multifile: SampleDataset, @@ -50,19 +53,15 @@ def test_clp_json_compression_json_multifile( """ logger.info("Starting test: 'test_clp_json_compression_json_multifile'") - package_path_config = clp_package.path_config - package_path_config.clear_package_archives() - compress_action = compress_clp_package(clp_package, json_multifile) result = verify_compress_action(compress_action, clp_package, json_multifile) assert result, result.failure_message - package_path_config.clear_package_archives() - logger.info("Test complete: 'test_clp_json_compression_json_multifile'") @pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") def test_clp_json_search(clp_package: ClpPackage) -> None: """ Validate that the `clp-json` package successfully searches some dataset. @@ -80,5 +79,3 @@ def test_clp_json_search(clp_package: ClpPackage) -> None: assert clp_package logger.info("Test complete: 'test_clp_json_search'") - - # TODO: clean up clp-package/var/data, clp-package/var/log, and clp-package/var/tmp diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index 26adcafdd0..5f4ecbad27 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -10,9 +10,7 @@ compress_clp_package, verify_compress_action, ) -from tests.utils.classes import ( - SampleDataset, -) +from tests.utils.classes import SampleDataset logger = logging.getLogger(__name__) @@ -42,6 +40,7 @@ def test_clp_text_startup(clp_package: ClpPackage) -> None: @pytest.mark.compression +@pytest.mark.usefixtures("clear_package_archives") def test_clp_text_compression_text_multifile( clp_package: ClpPackage, text_multifile: SampleDataset, @@ -54,13 +53,8 @@ def test_clp_text_compression_text_multifile( """ logger.info("Starting test: 'test_clp_text_compression_text_multifile'") - package_path_config = clp_package.path_config - package_path_config.clear_package_archives() - compress_action = compress_clp_package(clp_package, text_multifile) result = verify_compress_action(compress_action, clp_package, text_multifile) assert result, result.failure_message - package_path_config.clear_package_archives() - logger.info("Test complete: 'test_clp_text_compression_text_multifile'") diff --git a/integration-tests/tests/package_tests/fixtures.py b/integration-tests/tests/package_tests/fixtures.py index 9a2e0ee1b4..121a7f1462 100644 --- a/integration-tests/tests/package_tests/fixtures.py +++ b/integration-tests/tests/package_tests/fixtures.py @@ -96,3 +96,18 @@ def clp_package( stop_clp_action: ClpAction = stop_clp_package(clp_package) stop_result = verify_stop_clp_action(stop_clp_action, clp_package) assert stop_result, stop_result.failure_message + + +@pytest.fixture +def clear_package_archives(clp_package: ClpPackage) -> Iterator[None]: + """ + Clears all archives from the `clp_package` at the beginning and end of each test that uses the + fixture. + + :param clp_package: + """ + # TODO: fixture will be updated as archive-manager and dataset-manager capabilities are merged. + package_path_config = clp_package.path_config + package_path_config.clear_archives() + yield + package_path_config.clear_archives() From a6b775b2b0614039d695b0b2da9b3ec4d74cb81f Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 28 May 2026 13:32:09 +0000 Subject: [PATCH 12/19] Address Bill's comments. --- integration-tests/.pytest.ini | 2 +- integration-tests/tests/conftest.py | 1 + integration-tests/tests/fixtures/logging.py | 21 +++++++++++++++++++ .../tests/package_tests/classes.py | 1 + .../package_tests/clp_json/test_clp_json.py | 21 ++----------------- .../package_tests/clp_text/test_clp_text.py | 17 ++------------- 6 files changed, 28 insertions(+), 35 deletions(-) create mode 100644 integration-tests/tests/fixtures/logging.py diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index d18f328ccb..0cf24cf095 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -30,4 +30,4 @@ markers = core: mark tests that test the CLP core binaries package: mark tests that use the CLP package search: mark tests that test search - startup: mark tests that test startup + startstop: mark tests that test start-stop diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index 56189e6d5d..da54b18c91 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -10,6 +10,7 @@ # Make the fixtures defined in `tests/fixtures/` globally available without imports. pytest_plugins = [ + "tests.fixtures.logging", "tests.fixtures.sample_datasets", "tests.fixtures.path_configs", "tests.package_tests.fixtures", diff --git a/integration-tests/tests/fixtures/logging.py b/integration-tests/tests/fixtures/logging.py new file mode 100644 index 0000000000..470570a874 --- /dev/null +++ b/integration-tests/tests/fixtures/logging.py @@ -0,0 +1,21 @@ +"""Fixtures for logging test lifecycle events.""" + +import logging +from collections.abc import Iterator + +import pytest + +logger = logging.getLogger(__name__) + + +@pytest.fixture(autouse=True) +def log_test_lifecycle(request: pytest.FixtureRequest) -> Iterator[None]: + """ + Logs a message that identifies a test by name before it starts and after it finishes. + + :param request: + """ + test_name = request.node.originalname + logger.info("Starting test: '%s'", test_name) + yield + logger.info("Test complete: '%s'", test_name) diff --git a/integration-tests/tests/package_tests/classes.py b/integration-tests/tests/package_tests/classes.py index 19c01ad170..7250cdd469 100644 --- a/integration-tests/tests/package_tests/classes.py +++ b/integration-tests/tests/package_tests/classes.py @@ -122,6 +122,7 @@ def clear_archives(self) -> None: """Removes the contents of the package archives directory.""" # TODO: this method will be replaced with a more robust version that uses `archive-manager` # or `dataset-manager` (as appropriate) to clear archives correctly. + logger.info("Clearing package archives.") clear_directory(self.package_archives_path) diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index 86e4f95f02..f9116c2608 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -1,7 +1,5 @@ """Tests for the clp-json package.""" -import logging - import pytest from tests.package_tests.classes import ClpPackage @@ -12,9 +10,6 @@ ) from tests.utils.classes import SampleDataset -logger = logging.getLogger(__name__) - - # Pytest markers for this module. pytestmark = [ pytest.mark.package, @@ -25,19 +20,15 @@ ] -@pytest.mark.startup -def test_clp_json_startup(clp_package: ClpPackage) -> None: +@pytest.mark.startstop +def test_clp_json_startstop(clp_package: ClpPackage) -> None: """ Validate that the `clp-json` package starts up successfully. :param clp_package: """ - logger.info("Starting test: 'test_clp_json_startup'") - assert clp_package - logger.info("Test complete: 'test_clp_json_startup'") - @pytest.mark.compression @pytest.mark.usefixtures("clear_package_archives") @@ -51,14 +42,10 @@ def test_clp_json_compression_json_multifile( :param clp_package: :param json_multifile: """ - logger.info("Starting test: 'test_clp_json_compression_json_multifile'") - compress_action = compress_clp_package(clp_package, json_multifile) result = verify_compress_action(compress_action, clp_package, json_multifile) assert result, result.failure_message - logger.info("Test complete: 'test_clp_json_compression_json_multifile'") - @pytest.mark.search @pytest.mark.usefixtures("clear_package_archives") @@ -68,8 +55,6 @@ def test_clp_json_search(clp_package: ClpPackage) -> None: :param clp_package: """ - logger.info("Starting test: 'test_clp_json_search'") - # TODO: compress a dataset # TODO: check the correctness of the compression @@ -77,5 +62,3 @@ def test_clp_json_search(clp_package: ClpPackage) -> None: # TODO: search through that dataset and check the correctness of the search results. assert clp_package - - logger.info("Test complete: 'test_clp_json_search'") diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index 5f4ecbad27..ab8e9b815b 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -1,7 +1,5 @@ """Tests for the clp-text package.""" -import logging - import pytest from tests.package_tests.classes import ClpPackage @@ -12,9 +10,6 @@ ) from tests.utils.classes import SampleDataset -logger = logging.getLogger(__name__) - - # Pytest markers for this module. pytestmark = [ pytest.mark.package, @@ -25,19 +20,15 @@ ] -@pytest.mark.startup -def test_clp_text_startup(clp_package: ClpPackage) -> None: +@pytest.mark.startstop +def test_clp_text_startstop(clp_package: ClpPackage) -> None: """ Validate that the `clp-text` package successfully starts up. :param clp_package: """ - logger.info("Starting test: 'test_clp_text_startup'") - assert clp_package - logger.info("Test complete: 'test_clp_text_startup'") - @pytest.mark.compression @pytest.mark.usefixtures("clear_package_archives") @@ -51,10 +42,6 @@ def test_clp_text_compression_text_multifile( :param clp_package: :param text_multifile: """ - logger.info("Starting test: 'test_clp_text_compression_text_multifile'") - compress_action = compress_clp_package(clp_package, text_multifile) result = verify_compress_action(compress_action, clp_package, text_multifile) assert result, result.failure_message - - logger.info("Test complete: 'test_clp_text_compression_text_multifile'") From 4e928780533d8af2007c4e404218b90905a9ae22 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 1 Jun 2026 18:22:30 +0000 Subject: [PATCH 13/19] Describe differences between operating modes in test functions instead of shared helpers. --- .../package_tests/clp_json/test_clp_json.py | 32 ++++- .../clp_json/verification/__init__.py | 1 + .../clp_json/verification/compress.py | 33 +++++ .../package_tests/clp_text/test_clp_text.py | 25 +++- .../clp_text/verification/__init__.py | 1 + .../clp_text/verification/compress.py | 62 +++++++++ .../tests/package_tests/utils/compress.py | 120 +----------------- .../tests/package_tests/utils/decompress.py | 35 +---- 8 files changed, 146 insertions(+), 163 deletions(-) create mode 100644 integration-tests/tests/package_tests/clp_json/verification/__init__.py create mode 100644 integration-tests/tests/package_tests/clp_json/verification/compress.py create mode 100644 integration-tests/tests/package_tests/clp_text/verification/__init__.py create mode 100644 integration-tests/tests/package_tests/clp_text/verification/compress.py diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index f9116c2608..3e14e9a315 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -1,14 +1,23 @@ """Tests for the clp-json package.""" +import logging + import pytest from tests.package_tests.classes import ClpPackage from tests.package_tests.clp_json.utils.mode import CLP_JSON_MODE +from tests.package_tests.clp_json.verification.compress import verify_compress_structured_clp_json from tests.package_tests.utils.compress import ( - compress_clp_package, - verify_compress_action, + CompressArgs, +) +from tests.utils.classes import ( + ClpAction, + SampleDataset, + SampleDatasetMetadata, ) -from tests.utils.classes import SampleDataset + +logger = logging.getLogger(__name__) + # Pytest markers for this module. pytestmark = [ @@ -42,8 +51,21 @@ def test_clp_json_compression_json_multifile( :param clp_package: :param json_multifile: """ - compress_action = compress_clp_package(clp_package, json_multifile) - result = verify_compress_action(compress_action, clp_package, json_multifile) + metadata: SampleDatasetMetadata = json_multifile.metadata + args: CompressArgs = CompressArgs( + script_path=clp_package.path_config.compress_path, + config=clp_package.temp_config_file_path, + dataset=metadata.dataset_name, + timestamp_key=metadata.timestamp_key, + unstructured=metadata.unstructured, + paths=[json_multifile.logs_path], + ) + + logger.info("Compressing the 'json_multifile' dataset with the 'clp-json' package.") + action = ClpAction.from_args(args) + + logger.info("Verifying the compression of the 'json_multifile' dataset.") + result = verify_compress_structured_clp_json(action, clp_package, json_multifile) assert result, result.failure_message diff --git a/integration-tests/tests/package_tests/clp_json/verification/__init__.py b/integration-tests/tests/package_tests/clp_json/verification/__init__.py new file mode 100644 index 0000000000..31d6ab5cea --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/verification/__init__.py @@ -0,0 +1 @@ +"""Verification functions for the clp-json package.""" diff --git a/integration-tests/tests/package_tests/clp_json/verification/compress.py b/integration-tests/tests/package_tests/clp_json/verification/compress.py new file mode 100644 index 0000000000..7bd14b5f1c --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/verification/compress.py @@ -0,0 +1,33 @@ +"""Compression verification helpers specific to the clp-json package.""" + +from tests.package_tests.classes import ClpPackage +from tests.package_tests.utils.compress import CompressArgs +from tests.utils.classes import ClpAction, ClpVerificationResult, SampleDataset + + +def verify_compress_structured_clp_json( + action: ClpAction, + clp_package: ClpPackage, # noqa: ARG001 + original_dataset: SampleDataset, +) -> ClpVerificationResult: + """ + Verifies that the clp-json `clp_package` has compressed `original_dataset` correctly by + decompressing the compressed logs and comparing the decompressed logs to the original dataset. + + :param action: + :param clp_package: + :param original_dataset: + :return: A `ClpVerificationResult` indicating whether the round-trip matched the original logs. + """ + if not isinstance(action.args, CompressArgs): + err_msg = "'verify_compress_structured_clp_json' requires a 'CompressArgs' action." + raise TypeError(err_msg) + if action.args.unstructured or original_dataset.metadata.unstructured: + err_msg = "'verify_compress_structured_clp_json' cannot verify unstructured datasets." + raise ValueError(err_msg) + + # TODO: Waiting for PR 1299 (clp-json decompression) to be merged. + return action.verify_returncode() + + +# TODO: add verify_compress_unstructured_clp_json() and verify_compress_ir_clp_json(). diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index ab8e9b815b..05b23ebd81 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -1,14 +1,16 @@ """Tests for the clp-text package.""" +import logging + import pytest from tests.package_tests.classes import ClpPackage from tests.package_tests.clp_text.utils.mode import CLP_TEXT_MODE -from tests.package_tests.utils.compress import ( - compress_clp_package, - verify_compress_action, -) -from tests.utils.classes import SampleDataset +from tests.package_tests.clp_text.verification.compress import verify_compress_clp_text +from tests.package_tests.utils.compress import CompressArgs +from tests.utils.classes import ClpAction, SampleDataset + +logger = logging.getLogger(__name__) # Pytest markers for this module. pytestmark = [ @@ -42,6 +44,15 @@ def test_clp_text_compression_text_multifile( :param clp_package: :param text_multifile: """ - compress_action = compress_clp_package(clp_package, text_multifile) - result = verify_compress_action(compress_action, clp_package, text_multifile) + args = CompressArgs( + script_path=clp_package.path_config.compress_path, + config=clp_package.temp_config_file_path, + paths=[text_multifile.logs_path], + ) + + logger.info("Compressing the 'text_multifile' dataset with the 'clp-text' package.") + action = ClpAction.from_args(args) + + logger.info("Verifying the compression of the 'text_multifile' dataset.") + result = verify_compress_clp_text(action, clp_package, text_multifile) assert result, result.failure_message diff --git a/integration-tests/tests/package_tests/clp_text/verification/__init__.py b/integration-tests/tests/package_tests/clp_text/verification/__init__.py new file mode 100644 index 0000000000..bbb4a5f6e4 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/verification/__init__.py @@ -0,0 +1 @@ +"""Verification functions for the clp-text package.""" diff --git a/integration-tests/tests/package_tests/clp_text/verification/compress.py b/integration-tests/tests/package_tests/clp_text/verification/compress.py new file mode 100644 index 0000000000..f3eb6b71d7 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/verification/compress.py @@ -0,0 +1,62 @@ +"""Compression verification helpers specific to the clp-text package.""" + +from tests.package_tests.classes import ClpPackage +from tests.package_tests.utils.decompress import DecompressArgs +from tests.utils.classes import ClpAction, ClpVerificationResult, SampleDataset +from tests.utils.fs_validation import is_dir_tree_content_equal +from tests.utils.utils import clear_directory + + +def verify_compress_clp_text( + action: ClpAction, + clp_package: ClpPackage, + original_dataset: SampleDataset, +) -> ClpVerificationResult: + """ + Verifies that the clp-text `clp_package` has compressed `original_dataset` correctly by + decompressing the compressed logs and comparing the decompressed logs to the original dataset. + + :param action: + :param clp_package: + :param original_dataset: + :return: A `ClpVerificationResult` indicating whether the round-trip matched the original logs. + """ + result = action.verify_returncode() + if not result: + return result + + path_config = clp_package.path_config + clear_directory(path_config.package_decompression_dir) + + args: DecompressArgs = DecompressArgs( + script_path=path_config.decompress_path, + config=clp_package.temp_config_file_path, + extraction_dir=path_config.package_decompression_dir, + ) + + decompress_action = ClpAction.from_args(args) + + result = decompress_action.verify_returncode() + if not result: + return action.fail_verification( + "During compress action verification, supporting call to 'decompress.sh' returned a" + " non-zero exit code.", + supporting_action=decompress_action, + ) + + # Verify equality between original logs and decompressed logs. + original_logs_path = original_dataset.logs_path + decompressed_logs_path = path_config.package_decompression_dir / original_logs_path.relative_to( + original_logs_path.anchor + ) + equal = is_dir_tree_content_equal(original_logs_path, decompressed_logs_path) + clear_directory(path_config.package_decompression_dir) + + if equal: + return action.pass_verification() + + return action.fail_verification( + f"Compress verification failure: mismatch between original logs at" + f" '{original_logs_path}' and decompressed logs at '{decompressed_logs_path}'.", + supporting_action=decompress_action, + ) diff --git a/integration-tests/tests/package_tests/utils/compress.py b/integration-tests/tests/package_tests/utils/compress.py index 2a01ad79b9..2438d4f397 100644 --- a/integration-tests/tests/package_tests/utils/compress.py +++ b/integration-tests/tests/package_tests/utils/compress.py @@ -1,17 +1,8 @@ -"""Functions to facilitate CLP package compression testing.""" +"""Classes to facilitate CLP package compression testing.""" -import logging from pathlib import Path -from clp_py_utils.clp_config import StorageEngine - -from tests.package_tests.classes import ClpPackage -from tests.package_tests.utils.decompress import decompress_clp_package -from tests.utils.classes import ClpAction, ClpVerificationResult, CmdArgs, SampleDataset -from tests.utils.fs_validation import is_dir_tree_content_equal -from tests.utils.utils import clear_directory - -logger = logging.getLogger(__name__) +from tests.utils.classes import CmdArgs class CompressArgs(CmdArgs): @@ -44,110 +35,3 @@ def to_cmd(self) -> list[str]: cmd.extend([str(path) for path in self.paths]) return cmd - - -def compress_clp_package( - clp_package: ClpPackage, - dataset: SampleDataset, -) -> ClpAction: - """ - Compresses the specified dataset into a CLP package. - - :param clp_package: - :param dataset: - :return: The `ClpAction` instance that runs the compression. - """ - logger.info( - "Compressing the '%s' dataset with the '%s' package.", - dataset.dataset_name, - clp_package.mode_name, - ) - - args: CompressArgs = _construct_compress_args(clp_package, dataset) - return ClpAction.from_args(args) - - -def _construct_compress_args(clp_package: ClpPackage, dataset: SampleDataset) -> CompressArgs: - """Constructs the `CompressArgs` object for compressing the specified dataset.""" - path_config = clp_package.path_config - args = CompressArgs( - script_path=path_config.compress_path, - config=clp_package.temp_config_file_path, - paths=[dataset.logs_path], - ) - - if clp_package.clp_config.package.storage_engine == StorageEngine.CLP_S: - args.dataset = dataset.metadata.dataset_name - args.timestamp_key = dataset.metadata.timestamp_key - args.unstructured = dataset.metadata.unstructured - - return args - - -def verify_compress_action( - compress_action: ClpAction, - clp_package: ClpPackage, - original_dataset: SampleDataset, -) -> ClpVerificationResult: - """ - Verifies the compression action. - - :param compress_action: - :param clp_package: - :param original_dataset: - :return: A `ClpVerificationResult` indicating the success or failure of the verification. - """ - logger.info("Verifying '%s' package compression.", clp_package.mode_name) - - returncode_result = compress_action.verify_returncode() - if not returncode_result: - return returncode_result - - if original_dataset.metadata.unstructured: - return _verify_compress_action_unstructured_logs( - compress_action, clp_package, original_dataset - ) - return _verify_compress_action_structured_logs(compress_action) - - -def _verify_compress_action_structured_logs(compress_action: ClpAction) -> ClpVerificationResult: - """Verifies the compression of structured logs.""" - # TODO: Waiting for PR 1299 (clp-json decompression) to be merged. - return compress_action.pass_verification() - - -def _verify_compress_action_unstructured_logs( - compress_action: ClpAction, - clp_package: ClpPackage, - original_dataset: SampleDataset, -) -> ClpVerificationResult: - """Verifies the compression of unstructured logs.""" - path_config = clp_package.path_config - - # Decompress the contents of `clp-package/var/data/archives`. - clear_directory(path_config.package_decompression_dir) - decompress_action = decompress_clp_package(clp_package, path_config.package_decompression_dir) - decompress_returncode_result = decompress_action.verify_returncode() - if not decompress_returncode_result: - return compress_action.fail_verification( - "During compress action verification, supporting call to 'decompress.sh' returned a" - " non-zero exit code.", - supporting_action=decompress_action, - ) - - # Verify equality between original logs and decompressed logs. - original_logs_path = original_dataset.logs_path - decompressed_logs_path = path_config.package_decompression_dir / original_logs_path.relative_to( - original_logs_path.anchor - ) - - equal = is_dir_tree_content_equal(original_logs_path, decompressed_logs_path) - clear_directory(path_config.package_decompression_dir) - if equal: - return compress_action.pass_verification() - - return compress_action.fail_verification( - f"Compress verification failure: mismatch between original logs at" - f" '{original_logs_path}' and decompressed logs at '{decompressed_logs_path}'.", - supporting_action=decompress_action, - ) diff --git a/integration-tests/tests/package_tests/utils/decompress.py b/integration-tests/tests/package_tests/utils/decompress.py index e556ad837a..c36e8f4075 100644 --- a/integration-tests/tests/package_tests/utils/decompress.py +++ b/integration-tests/tests/package_tests/utils/decompress.py @@ -1,15 +1,10 @@ -"""Functions to facilitate CLP package decompression testing.""" +"""Classes to facilitate CLP package decompression testing.""" -import logging from pathlib import Path -from typing import Any from clp_package_utils.general import EXTRACT_FILE_CMD -from tests.package_tests.classes import ClpPackage -from tests.utils.classes import ClpAction, CmdArgs - -logger = logging.getLogger(__name__) +from tests.utils.classes import CmdArgs class DecompressArgs(CmdArgs): @@ -35,29 +30,3 @@ def to_cmd(self) -> list[str]: cmd.extend([str(path) for path in self.paths]) return cmd - - -def decompress_clp_package( - clp_package: ClpPackage, - extraction_dir: Any, - paths: list[Path] | None = None, -) -> ClpAction: - """ - Decompresses the specified CLP package archives. Note that decompression can only be used in - conjunction with compression. - - :param clp_package: - :param extraction_dir: - :param paths: - :return: The `ClpAction` instance that runs the decompression. - """ - logger.info("Decompressing '%s' package.", clp_package.mode_name) - - path_config = clp_package.path_config - args: DecompressArgs = DecompressArgs( - script_path=path_config.decompress_path, - config=clp_package.temp_config_file_path, - extraction_dir=extraction_dir, - paths=paths, - ) - return ClpAction.from_args(args) From a41411f6b9fa0ab5f22762675c3e6dce439d06d7 Mon Sep 17 00:00:00 2001 From: Quinn Date: Mon, 1 Jun 2026 18:34:30 +0000 Subject: [PATCH 14/19] Add placeholder for clp-json unstructured compression behaviour. --- .../package_tests/clp_json/test_clp_json.py | 36 ++++++++++++++++++- .../clp_json/verification/compress.py | 28 ++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index 3e14e9a315..eed9402b16 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -6,7 +6,10 @@ from tests.package_tests.classes import ClpPackage from tests.package_tests.clp_json.utils.mode import CLP_JSON_MODE -from tests.package_tests.clp_json.verification.compress import verify_compress_structured_clp_json +from tests.package_tests.clp_json.verification.compress import ( + verify_compress_structured_clp_json, + verify_compress_unstructured_clp_json, +) from tests.package_tests.utils.compress import ( CompressArgs, ) @@ -69,6 +72,37 @@ def test_clp_json_compression_json_multifile( assert result, result.failure_message +@pytest.mark.compression +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_compression_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package successfully compresses the `text-multifile` dataset as + unstructured text. + + :param clp_package: + :param text_multifile: + """ + metadata: SampleDatasetMetadata = text_multifile.metadata + args: CompressArgs = CompressArgs( + script_path=clp_package.path_config.compress_path, + config=clp_package.temp_config_file_path, + dataset=metadata.dataset_name, + timestamp_key=metadata.timestamp_key, + unstructured=metadata.unstructured, + paths=[text_multifile.logs_path], + ) + + logger.info("Compressing the 'text_multifile' dataset with the 'clp-json' package.") + action = ClpAction.from_args(args) + + logger.info("Verifying the compression of the 'text_multifile' dataset.") + result = verify_compress_unstructured_clp_json(action, clp_package, text_multifile) + assert result, result.failure_message + + @pytest.mark.search @pytest.mark.usefixtures("clear_package_archives") def test_clp_json_search(clp_package: ClpPackage) -> None: diff --git a/integration-tests/tests/package_tests/clp_json/verification/compress.py b/integration-tests/tests/package_tests/clp_json/verification/compress.py index 7bd14b5f1c..3f60fd4512 100644 --- a/integration-tests/tests/package_tests/clp_json/verification/compress.py +++ b/integration-tests/tests/package_tests/clp_json/verification/compress.py @@ -30,4 +30,30 @@ def verify_compress_structured_clp_json( return action.verify_returncode() -# TODO: add verify_compress_unstructured_clp_json() and verify_compress_ir_clp_json(). +def verify_compress_unstructured_clp_json( + action: ClpAction, + clp_package: ClpPackage, # noqa: ARG001 + original_dataset: SampleDataset, +) -> ClpVerificationResult: + """ + Verifies that the clp-json `clp_package` has compressed the unstructured `original_dataset` + correctly by decompressing the compressed logs and comparing the decompressed logs to the + original dataset. + + :param action: + :param clp_package: + :param original_dataset: + :return: A `ClpVerificationResult` indicating whether the round-trip matched the original logs. + """ + if not isinstance(action.args, CompressArgs): + err_msg = "'verify_compress_unstructured_clp_json' requires a 'CompressArgs' action." + raise TypeError(err_msg) + if not action.args.unstructured or not original_dataset.metadata.unstructured: + err_msg = "'verify_compress_unstructured_clp_json' can only verify unstructured datasets." + raise ValueError(err_msg) + + # TODO: Waiting for PR 1299 (clp-json decompression) to be merged. + return action.verify_returncode() + + +# TODO: add verify_compress_ir_clp_json(). From 6c49f4b2b206d09ab0651b77fd330bbd0e0af875 Mon Sep 17 00:00:00 2001 From: Quinn Date: Thu, 2 Jul 2026 18:11:22 +0000 Subject: [PATCH 15/19] Write kv-centric search verification --- integration-tests/tests/data/README.md | 4 +- .../tests/data/json_multifile/metadata.json | 4 +- .../tests/data/text_multifile/metadata.json | 4 +- .../tests/data/text_singlefile/metadata.json | 3 +- .../package_tests/clp_json/test_clp_json.py | 99 ++++++++++- .../clp_json/verification/search.py | 161 ++++++++++++++++++ .../clp_text/verification/search.py | 1 + .../tests/package_tests/utils/search.py | 28 +-- integration-tests/tests/utils/classes.py | 9 - 9 files changed, 270 insertions(+), 43 deletions(-) create mode 100644 integration-tests/tests/package_tests/clp_json/verification/search.py create mode 100644 integration-tests/tests/package_tests/clp_text/verification/search.py diff --git a/integration-tests/tests/data/README.md b/integration-tests/tests/data/README.md index f0720a71d7..96283034d2 100644 --- a/integration-tests/tests/data/README.md +++ b/integration-tests/tests/data/README.md @@ -35,8 +35,7 @@ following rules must be observed: "str", "str", ... - ], - "single_match_wildcard_query": "str" + ] } ``` @@ -49,7 +48,6 @@ following rules must be observed: | `end_ts` | The latest timestamp present in the dataset (ms). | | `logs_subdir` | The name of the subdirectory containing logs. | | `file_names` | A list of the files within `logs_subdir`. | - | `single_match_wildcard_query` | A wildcard query that matches exactly one log message in the dataset. | ## Accessing sample datasets within the testing system diff --git a/integration-tests/tests/data/json_multifile/metadata.json b/integration-tests/tests/data/json_multifile/metadata.json index 0dd22a8158..da9bbebcf3 100644 --- a/integration-tests/tests/data/json_multifile/metadata.json +++ b/integration-tests/tests/data/json_multifile/metadata.json @@ -11,7 +11,5 @@ "sts-135-2011-07-11.jsonl", "sts-135-2011-07-19.jsonl", "sts-135-2011-07-21.jsonl" - ], - "single_match_wildcard_query": "\"detail\":\"Roll program complete, heads down attitude achieved for ascent\"", - "single_match_file": "sts-135-2011-07-08.jsonl" + ] } diff --git a/integration-tests/tests/data/text_multifile/metadata.json b/integration-tests/tests/data/text_multifile/metadata.json index 1e569e8199..c37c8a693f 100644 --- a/integration-tests/tests/data/text_multifile/metadata.json +++ b/integration-tests/tests/data/text_multifile/metadata.json @@ -11,7 +11,5 @@ "apollo-17_day07.txt", "apollo-17_day10.txt", "apollo-17_day13.txt" - ], - "single_match_wildcard_query": "Saturn", - "single_match_file": "apollo-17_day01.txt" + ] } diff --git a/integration-tests/tests/data/text_singlefile/metadata.json b/integration-tests/tests/data/text_singlefile/metadata.json index 1ef6c24929..6ae08de1cd 100644 --- a/integration-tests/tests/data/text_singlefile/metadata.json +++ b/integration-tests/tests/data/text_singlefile/metadata.json @@ -7,6 +7,5 @@ "logs_subdir": "logs", "file_names": [ "simple.txt" - ], - "single_match_wildcard_query": "TEST1" + ] } diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index 75a7b87112..58a9115898 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -10,9 +10,9 @@ verify_compress_structured_clp_json, verify_compress_unstructured_clp_json, ) -from tests.package_tests.utils.compress import ( - CompressArgs, -) +from tests.package_tests.clp_json.verification.search import verify_search_clp_json +from tests.package_tests.utils.compress import CompressArgs +from tests.package_tests.utils.search import SearchArgs from tests.utils.classes import ( ClpAction, SampleDataset, @@ -109,16 +109,97 @@ def test_clp_json_compression_text_multifile( @pytest.mark.search @pytest.mark.usefixtures("clear_package_archives") -def test_clp_json_search(clp_package: ClpPackage) -> None: +def test_clp_json_search_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: """ - Validate that the `clp-json` package successfully searches some dataset. + Validate that the `clp-json` package successfully searches the `json-multifile` dataset. :param clp_package: + :param json_multifile: """ - # TODO: compress a dataset + metadata: SampleDatasetMetadata = json_multifile.metadata + compress_args: CompressArgs = CompressArgs( + script_path=clp_package.path_config.compress_path, + config=clp_package.temp_config_file_path, + dataset=metadata.dataset_name, + timestamp_key=metadata.timestamp_key, + unstructured=metadata.unstructured, + paths=[json_multifile.logs_path], + ) - # TODO: check the correctness of the compression + logger.info("Compressing the 'json_multifile' dataset with the 'clp-json' package.") + compress_action = ClpAction.from_args(compress_args) + compress_result = compress_action.verify_returncode() + assert compress_result, compress_result.failure_message - # TODO: search through that dataset and check the correctness of the search results. + logger.info("Verifying the compression of the 'json_multifile' dataset.") + compress_result = verify_compress_structured_clp_json( + compress_action, clp_package, json_multifile + ) + assert compress_result, compress_result.failure_message - assert clp_package + search_args: SearchArgs = SearchArgs( + script_path=clp_package.path_config.search_path, + config=clp_package.temp_config_file_path, + query='detail: "*maximum dynamic*"', + dataset=metadata.dataset_name, + ) + + logger.info("Searching the 'json_multifile' dataset with the 'clp-json' package.") + search_action = ClpAction.from_args(search_args) + search_result = search_action.verify_returncode() + assert search_result, search_result.failure_message + + logger.info("Verifying the search results for the 'json_multifile' dataset.") + search_result = verify_search_clp_json(search_action, clp_package, json_multifile) + assert search_result, search_result.failure_message + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package successfully searches the `text-multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + metadata: SampleDatasetMetadata = text_multifile.metadata + args: CompressArgs = CompressArgs( + script_path=clp_package.path_config.compress_path, + config=clp_package.temp_config_file_path, + dataset=metadata.dataset_name, + timestamp_key=metadata.timestamp_key, + unstructured=metadata.unstructured, + paths=[text_multifile.logs_path], + ) + + logger.info("Compressing the 'text_multifile' dataset with the 'clp-json' package.") + action = ClpAction.from_args(args) + result = action.verify_returncode() + assert result, result.failure_message + + logger.info("Verifying the compression of the 'text_multifile' dataset.") + result = verify_compress_unstructured_clp_json(action, clp_package, text_multifile) + assert result, result.failure_message + + search_args: SearchArgs = SearchArgs( + script_path=clp_package.path_config.search_path, + config=clp_package.temp_config_file_path, + query="*Saturn*", + dataset=metadata.dataset_name, + ) + + logger.info("Searching the 'text_multifile' dataset with the 'clp-json' package.") + search_action = ClpAction.from_args(search_args) + search_result = search_action.verify_returncode() + assert search_result, search_result.failure_message + + logger.info("Verifying the search results for the 'text_multifile' dataset.") + search_result = verify_search_clp_json(search_action, clp_package, text_multifile) + assert search_result, search_result.failure_message diff --git a/integration-tests/tests/package_tests/clp_json/verification/search.py b/integration-tests/tests/package_tests/clp_json/verification/search.py new file mode 100644 index 0000000000..edac7d932c --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/verification/search.py @@ -0,0 +1,161 @@ +"""Search verification helpers specific to the clp-json package.""" + +import json +import logging +import re +from typing import Any + +from tests.package_tests.classes import ClpPackage +from tests.package_tests.utils.search import SearchArgs +from tests.utils.classes import ( + ClpAction, + ClpVerificationResult, + SampleDataset, + SampleDatasetMetadata, +) + +logger = logging.getLogger(__name__) + + +WILDCARD_MULTIMATCH_CHAR = "*" +KV_DELIMITER_COLON = ":" +MIN_QUOTED_LENGTH = 2 + + +def verify_search_clp_json( + action: ClpAction, clp_package: ClpPackage, dataset: SampleDataset +) -> ClpVerificationResult: + """Docstring.""" + args = action.args + if not isinstance(args, SearchArgs): + err_msg = "Verification expects a 'SearchArgs' action." + raise TypeError(err_msg) + + # Construct KV-pair from query. + kv_pair = _construct_kv_pair_from_query(args.query) + + # Convert original logs to list[dict] objects for processing. + metadata: SampleDatasetMetadata = dataset.metadata + if metadata.unstructured: + # Get the structured version of the unstructured data. Could use log-converter directly. + supporting_search_args: SearchArgs = SearchArgs( + script_path=clp_package.path_config.search_path, + config=clp_package.temp_config_file_path, + query="*", + dataset=metadata.dataset_name, + ) + supporting_search_action = ClpAction.from_args(supporting_search_args) + search_result = supporting_search_action.verify_returncode() + if not search_result: + return action.fail_verification( + "During search action verification, supporting call to 'search.sh' returned a" + " non-zero exit code.", + supporting_action=supporting_search_action, + ) + supporting_output = supporting_search_action.completed_proc.stdout + log_list = [json.loads(line) for line in supporting_output.splitlines() if line.strip()] + else: + log_list = _load_all_jsonl_logs_from_dataset(dataset) + + # Find entries that match the kv_pair from the original query. + found_entries: list[dict[str, Any]] = _search_log_list_for_kv_pair(log_list, kv_pair) + + # Compare the found entries with the structurized search action output. + search_output = action.completed_proc.stdout + search_output_list = [json.loads(line) for line in search_output.splitlines() if line.strip()] + if _normalize_entries(found_entries) == _normalize_entries(search_output_list): + return action.pass_verification() + + return action.fail_verification( + "Search verification failure: mismatch between search output and expected output." + f" Expected output: '{found_entries}', actual output: '{search_output_list}'", + ) + + +def _normalize_entries(entries: list[dict[str, Any]]) -> list[str]: + """ + Serializes each entry into a canonical JSON string and returns the sorted list, so that two + result sets can be compared independent of ordering while tolerating unhashable dict entries. + """ + return sorted(json.dumps(entry, sort_keys=True) for entry in entries) + + +def _construct_kv_pair_from_query(query: str) -> tuple[str, Any]: + """ + Constructs a key-value pair from a query string. The query is split on the first + `KV_DELIMITER_COLON` that is not inside a quoted group (`"..."` or `'...'`). + + :param query: The query string to parse. + :return: `(KEY, VALUE)` if `KV_DELIMITER_COLON` is present, else + `(WILDCARD_MULTIMATCH_CHAR, VALUE)`. + """ + colon_index = _find_unquoted_colon(query) + if colon_index is None: + return WILDCARD_MULTIMATCH_CHAR, _strip_surrounding_quotes(query.strip()) + key = _strip_surrounding_quotes(query[:colon_index].strip()) + value = _strip_surrounding_quotes(query[colon_index + 1 :].strip()) + return key, value + + +def _find_unquoted_colon(query: str) -> int | None: + """Returns the index of the first colon not in a quoted group, or None if there is none.""" + open_quote: str | None = None + for index, char in enumerate(query): + if open_quote is not None: + if char == open_quote: + open_quote = None + elif char in ('"', "'"): + open_quote = char + elif char == KV_DELIMITER_COLON: + return index + return None + + +def _strip_surrounding_quotes(text: str) -> str: + """Removes a single pair of matching surrounding quotes (`"` or `'`) from `text`, if present.""" + if len(text) >= MIN_QUOTED_LENGTH and text[0] == text[-1] and text[0] in ('"', "'"): + return text[1:-1] + return text + + +def _load_all_jsonl_logs_from_dataset(dataset: SampleDataset) -> list[dict[str, Any]]: + """Load every JSONL record from a dataset into a single list of dicts.""" + records: list[dict[str, Any]] = [] + for path in dataset.metadata.file_names: + absolute_path = dataset.logs_path / path + with absolute_path.open("r", encoding="utf-8") as f: + for line in f: + stripped = line.strip() + if not stripped: + continue + records.append(json.loads(stripped)) + return records + + +def _search_log_list_for_kv_pair( + log_list: list[dict[str, Any]], kv_pair: tuple[str, Any] +) -> list[dict[str, Any]]: + """ + Searches a list of log entries for those that match a given key-value pair. Both the key + and value may contain `WILDCARD_MULTIMATCH_CHAR`, which matches zero or more characters. + """ + key, value = kv_pair + key_regex = _convert_wildcard_to_regex(str(key)) + value_regex = _convert_wildcard_to_regex(str(value)) + return [ + entry + for entry in log_list + if any( + key_regex.fullmatch(str(entry_key)) and value_regex.fullmatch(str(entry_value)) + for entry_key, entry_value in entry.items() + ) + ] + + +def _convert_wildcard_to_regex(pattern: str) -> re.Pattern[str]: + """ + Compiles a wildcard `pattern` into a regex in which each `WILDCARD_MULTIMATCH_CHAR` matches + zero or more characters and all other characters are matched literally. + """ + regex = ".*".join(re.escape(segment) for segment in pattern.split(WILDCARD_MULTIMATCH_CHAR)) + return re.compile(regex, re.DOTALL) diff --git a/integration-tests/tests/package_tests/clp_text/verification/search.py b/integration-tests/tests/package_tests/clp_text/verification/search.py new file mode 100644 index 0000000000..276bb942de --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/verification/search.py @@ -0,0 +1 @@ +"""Search verification helpers specific to the clp-text package.""" diff --git a/integration-tests/tests/package_tests/utils/search.py b/integration-tests/tests/package_tests/utils/search.py index 740779a20e..c28d41cf8c 100644 --- a/integration-tests/tests/package_tests/utils/search.py +++ b/integration-tests/tests/package_tests/utils/search.py @@ -1,4 +1,4 @@ -"""Functions and classes to facilitate CLP package search.""" +"""Classes to facilitate CLP package search testing.""" import logging import re @@ -31,7 +31,7 @@ class SearchArgs(CmdArgs): script_path: Path config: Path - wildcard_query: str + query: str raw: bool = True dataset: str | None = None file_path: Path | None = None @@ -71,7 +71,7 @@ def to_cmd(self) -> list[str]: if self.raw: cmd.append("--raw") - cmd.append(self.wildcard_query) + cmd.append(self.query) return cmd @@ -91,7 +91,7 @@ def search_clp_package( clp_package: ClpPackage, dataset: SampleDataset, search_type: ClpPackageSearchType, - wildcard_query: str, + query: str, ) -> ClpAction: """ Performs the specified search on the dataset using the CLP package. @@ -99,7 +99,7 @@ def search_clp_package( :param clp_package: :param dataset: :param search_type: - :param wildcard_query: + :param query: :return: The `ClpAction` instance that runs the search. """ logger.info( @@ -108,22 +108,22 @@ def search_clp_package( dataset.dataset_name, ) - args: SearchArgs = _construct_args(clp_package, dataset, search_type, wildcard_query) - return ClpAction(cmd=args.to_cmd(), args=args) + args: SearchArgs = _construct_args(clp_package, dataset, search_type, query) + return ClpAction.from_args(args) def _construct_args( clp_package: ClpPackage, dataset: SampleDataset, search_type: ClpPackageSearchType, - wildcard_query: str, + query: str, ) -> SearchArgs: """Construct the `SearchArgs` object for the specified search on the dataset.""" path_config = clp_package.path_config args = SearchArgs( script_path=path_config.search_path, config=clp_package.temp_config_file_path, - wildcard_query=wildcard_query, + query=query, ) if clp_package.clp_config.package.storage_engine == StorageEngine.CLP_S: @@ -133,7 +133,7 @@ def _construct_args( case ClpPackageSearchType.BASIC: pass case ClpPackageSearchType.FILE_PATH: - args.file_path = dataset.logs_path / dataset.metadata.single_match_file + pytest.fail("FILE_PATH search not yet implemented.") case ClpPackageSearchType.IGNORE_CASE: args.ignore_case = True case ClpPackageSearchType.COUNT_RESULTS: @@ -168,9 +168,9 @@ def verify_search_action( original_dataset.dataset_name, ) - returncode_result = action.verify_returncode() - if not returncode_result: - return returncode_result + result = action.verify_returncode() + if not result: + return result args = action.args assert isinstance(args, SearchArgs) @@ -208,7 +208,7 @@ def _construct_grep_verification_cmd( return [ get_binary_path("grep"), *grep_cmd_options, - args.wildcard_query, + args.query, str(path_for_grep), ] diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index 18230a9875..5209847066 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -89,8 +89,6 @@ class SampleDatasetMetadata(BaseModel): end_ts: int logs_subdir: str file_names: list[str] - single_match_wildcard_query: str - single_match_file: str @dataclass @@ -132,13 +130,6 @@ def __post_init__(self) -> None: file_path_abs = self.logs_path / file_path validate_file_exists(file_path_abs) - if self.metadata.single_match_file not in self.metadata.file_names: - err_msg = ( - f"`single_match_file` '{self.metadata.single_match_file}' is not listed in" - " `file_names`." - ) - raise ValueError(err_msg) - @property def metadata_file_path(self) -> Path: """:return: The absolute path to the file containing metadata for the dataset.""" From f523b512620474c4b3c1d17184e4b2ca31fb8c8b Mon Sep 17 00:00:00 2001 From: Quinn Date: Fri, 3 Jul 2026 18:03:55 +0000 Subject: [PATCH 16/19] Modify search verif. w.r.t. search flags. --- integration-tests/tests/data/README.md | 34 +++ .../tests/data/json_multifile/metadata.json | 3 + .../tests/data/text_multifile/metadata.json | 4 + .../tests/data/text_singlefile/metadata.json | 1 + .../package_tests/clp_json/test_clp_json.py | 56 ++++ .../clp_json/verification/search.py | 244 +++++++++++++++--- .../package_tests/clp_text/test_clp_text.py | 55 +++- .../clp_text/verification/search.py | 186 +++++++++++++ .../tests/package_tests/utils/search.py | 239 ++--------------- integration-tests/tests/utils/classes.py | 18 ++ 10 files changed, 593 insertions(+), 247 deletions(-) diff --git a/integration-tests/tests/data/README.md b/integration-tests/tests/data/README.md index 96283034d2..e12471f380 100644 --- a/integration-tests/tests/data/README.md +++ b/integration-tests/tests/data/README.md @@ -28,6 +28,7 @@ following rules must be observed: "dataset_name": "str", "unstructured": bool, "timestamp_key": "str" | null, + "timestamp_format": | null, "begin_ts": int, "end_ts": int, "logs_subdir": "str", @@ -44,11 +45,42 @@ following rules must be observed: | `dataset_name` | The name of the sample dataset directory. | | `unstructured` | `True` if logs are unstructured, else `False`. | | `timestamp_key` | The authoritative timestamp key, or `null` if there is no such key. | + | `timestamp_format` | Description of authoritative timestamp encoding (see below). | | `begin_ts` | The earliest timestamp present in the dataset (ms). | | `end_ts` | The latest timestamp present in the dataset (ms). | | `logs_subdir` | The name of the subdirectory containing logs. | | `file_names` | A list of the files within `logs_subdir`. | + `timestamp_format` is a field tagged by by `kind` and containing `pattern` if applicable: + + ```json + {"kind": "epoch_ms"} + {"kind": "strptime", "pattern": "str"} + ``` + + | Field | Description | + | --- | --- | + | `kind` | `"epoch_ms"` for epoch ms timestamps, or `"strptime"` for formatted strings. | + | `pattern` | (`strptime` only) The Python [`strptime`][strptime] pattern for the timestamp. | + +## Time-range search verification + +Search verification reproduces the package's `--begin-time`/`--end-time` filter by independently +determining each log event's timestamp. For a dataset to be usable in a time-range search test, the +following restrictions must be observed: + +1. `begin_ts` must equal the earliest timestamp present in the dataset, and `end_ts` the latest. +2. `begin_ts` must be less than or equal to `end_ts`. +3. For **structured** datasets: + * `timestamp_key` must not be `null`, and must be a *top-level* key. Nested/dotted keys are + not supported. + * `timestamp_format` must not be `null`. + * Each log in the dataset must contain `timestamp_key`. +4. For **unstructured** datasets: + * `timestamp_format` must not be `null`. + * Every log line must begin with a single whitespace-delimited timestamp token written in the + format described by `timestamp_format`. + ## Accessing sample datasets within the testing system To access a sample dataset from within the test system, the following rules should be observed: @@ -72,3 +104,5 @@ To access a sample dataset from within the test system, the following rules shou Tests should use sample dataset fixtures instead of reading the logs directly, because many verification flows rely on dataset metadata. + +[strptime]: https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes diff --git a/integration-tests/tests/data/json_multifile/metadata.json b/integration-tests/tests/data/json_multifile/metadata.json index da9bbebcf3..c6a12e5782 100644 --- a/integration-tests/tests/data/json_multifile/metadata.json +++ b/integration-tests/tests/data/json_multifile/metadata.json @@ -2,6 +2,9 @@ "dataset_name": "json_multifile", "unstructured": false, "timestamp_key": "timestamp", + "timestamp_format": { + "kind": "epoch_ms" + }, "begin_ts": 1310138944000, "end_ts": 1311208074120, "logs_subdir": "logs", diff --git a/integration-tests/tests/data/text_multifile/metadata.json b/integration-tests/tests/data/text_multifile/metadata.json index c37c8a693f..38e86ff427 100644 --- a/integration-tests/tests/data/text_multifile/metadata.json +++ b/integration-tests/tests/data/text_multifile/metadata.json @@ -2,6 +2,10 @@ "dataset_name": "text_multifile", "unstructured": true, "timestamp_key": null, + "timestamp_format": { + "kind": "strptime", + "pattern": "%Y-%m-%dT%H:%M:%S.%f" + }, "begin_ts": 92554380000, "end_ts": 93596007004, "logs_subdir": "logs", diff --git a/integration-tests/tests/data/text_singlefile/metadata.json b/integration-tests/tests/data/text_singlefile/metadata.json index 6ae08de1cd..cc159aede8 100644 --- a/integration-tests/tests/data/text_singlefile/metadata.json +++ b/integration-tests/tests/data/text_singlefile/metadata.json @@ -2,6 +2,7 @@ "dataset_name": "text_singlefile", "unstructured": true, "timestamp_key": null, + "timestamp_format": null, "begin_ts": 1427089710122, "end_ts": 1427089710122, "logs_subdir": "logs", diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index 58a9115898..2bebe54943 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -157,6 +157,62 @@ def test_clp_json_search_json_multifile( assert search_result, search_result.failure_message +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_time_range_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package honours `--begin-time`/`--end-time` when searching the + `json-multifile` dataset. The time range is a strict sub-range of the dataset's span + (`[begin_ts + 1, end_ts - 1]`), which is guaranteed to exclude the earliest and latest records. + + :param clp_package: + :param json_multifile: + """ + metadata: SampleDatasetMetadata = json_multifile.metadata + compress_args: CompressArgs = CompressArgs( + script_path=clp_package.path_config.compress_path, + config=clp_package.temp_config_file_path, + dataset=metadata.dataset_name, + timestamp_key=metadata.timestamp_key, + unstructured=metadata.unstructured, + paths=[json_multifile.logs_path], + ) + + logger.info("Compressing the 'json_multifile' dataset with the 'clp-json' package.") + compress_action = ClpAction.from_args(compress_args) + compress_result = compress_action.verify_returncode() + assert compress_result, compress_result.failure_message + + logger.info("Verifying the compression of the 'json_multifile' dataset.") + compress_result = verify_compress_structured_clp_json( + compress_action, clp_package, json_multifile + ) + assert compress_result, compress_result.failure_message + + search_args: SearchArgs = SearchArgs( + script_path=clp_package.path_config.search_path, + config=clp_package.temp_config_file_path, + query='mission: "STS-135"', + dataset=metadata.dataset_name, + begin_ts=metadata.begin_ts + 1, + end_ts=metadata.end_ts - 1, + ) + + logger.info( + "Searching the 'json_multifile' dataset over a time range with the 'clp-json' package." + ) + search_action = ClpAction.from_args(search_args) + search_result = search_action.verify_returncode() + assert search_result, search_result.failure_message + + logger.info("Verifying the time-range search results for the 'json_multifile' dataset.") + search_result = verify_search_clp_json(search_action, clp_package, json_multifile) + assert search_result, search_result.failure_message + + @pytest.mark.search @pytest.mark.usefixtures("clear_package_archives") def test_clp_json_search_text_multifile( diff --git a/integration-tests/tests/package_tests/clp_json/verification/search.py b/integration-tests/tests/package_tests/clp_json/verification/search.py index edac7d932c..a0a7c24b8f 100644 --- a/integration-tests/tests/package_tests/clp_json/verification/search.py +++ b/integration-tests/tests/package_tests/clp_json/verification/search.py @@ -5,13 +5,17 @@ import re from typing import Any +import pytest + from tests.package_tests.classes import ClpPackage -from tests.package_tests.utils.search import SearchArgs +from tests.package_tests.utils.search import parse_timestamp_to_epoch_ms, SearchArgs from tests.utils.classes import ( ClpAction, ClpVerificationResult, + EpochMsTimestampFormat, SampleDataset, SampleDatasetMetadata, + TimestampFormat, ) logger = logging.getLogger(__name__) @@ -23,9 +27,29 @@ def verify_search_clp_json( - action: ClpAction, clp_package: ClpPackage, dataset: SampleDataset + action: ClpAction, + clp_package: ClpPackage, + dataset: SampleDataset, ) -> ClpVerificationResult: - """Docstring.""" + """ + Verifies that the search action performed on the `clp-json` package returns the expected + results. Verification is performed as follows: + + 1. The query string from the search action is parsed into a key-value pair. + 2. The dataset's log entries are gathered into a list of dicts: read directly from the original + JSONL files for a structured dataset, or recovered by re-running the search with a match-all + query for an unstructured dataset (whose original logs aren't JSON). + 3. The list of dicts is searched for entries that match the key-value pair. + 4. The found entries are compared against the original search action's output. For a count-style + search (`--count`), the number of found entries is compared against the reported count; + otherwise the set of found entries is compared against the set of output logs. + + :param action: + :param clp_package: + :param dataset: + :return: A `ClpVerificationResult` indicating the success or failure of the verification. + """ + # Verify that the action's arguments are of the expected type. args = action.args if not isinstance(args, SearchArgs): err_msg = "Verification expects a 'SearchArgs' action." @@ -52,40 +76,47 @@ def verify_search_clp_json( " non-zero exit code.", supporting_action=supporting_search_action, ) + supporting_output = supporting_search_action.completed_proc.stdout - log_list = [json.loads(line) for line in supporting_output.splitlines() if line.strip()] + all_logs = [json.loads(line) for line in supporting_output.splitlines() if line.strip()] else: - log_list = _load_all_jsonl_logs_from_dataset(dataset) + all_logs = _load_all_jsonl_logs_from_dataset(dataset) # Find entries that match the kv_pair from the original query. - found_entries: list[dict[str, Any]] = _search_log_list_for_kv_pair(log_list, kv_pair) + found_entries: list[dict[str, Any]] = _search_all_logs_for_kv_pair( + all_logs, kv_pair, args, metadata + ) - # Compare the found entries with the structurized search action output. + # Compare the found entries with the search action output. search_output = action.completed_proc.stdout + if args.is_count_query: + expected_count = str(len(found_entries)) + "\n" + actual_count = _extract_count_from_search_output(search_output) + + if expected_count == actual_count: + return action.pass_verification() + + return action.fail_verification( + "Search verification failure: mismatch between search count and expected count.\n" + f"Expected count: '{expected_count}'\nActual count: '{actual_count}'", + ) + search_output_list = [json.loads(line) for line in search_output.splitlines() if line.strip()] if _normalize_entries(found_entries) == _normalize_entries(search_output_list): return action.pass_verification() return action.fail_verification( - "Search verification failure: mismatch between search output and expected output." - f" Expected output: '{found_entries}', actual output: '{search_output_list}'", + "Search verification failure: mismatch between search output and expected output.\n" + f"Expected output: '{found_entries}'\nActual output: '{search_output_list}'", ) -def _normalize_entries(entries: list[dict[str, Any]]) -> list[str]: - """ - Serializes each entry into a canonical JSON string and returns the sorted list, so that two - result sets can be compared independent of ordering while tolerating unhashable dict entries. - """ - return sorted(json.dumps(entry, sort_keys=True) for entry in entries) - - def _construct_kv_pair_from_query(query: str) -> tuple[str, Any]: """ Constructs a key-value pair from a query string. The query is split on the first `KV_DELIMITER_COLON` that is not inside a quoted group (`"..."` or `'...'`). - :param query: The query string to parse. + :param query: :return: `(KEY, VALUE)` if `KV_DELIMITER_COLON` is present, else `(WILDCARD_MULTIMATCH_CHAR, VALUE)`. """ @@ -98,7 +129,12 @@ def _construct_kv_pair_from_query(query: str) -> tuple[str, Any]: def _find_unquoted_colon(query: str) -> int | None: - """Returns the index of the first colon not in a quoted group, or None if there is none.""" + """ + Finds the first colon in `query` that is not inside a quoted group (`"..."` or `'...'`). + + :param query: + :return: The index of the first unquoted colon, or `None` if there is none. + """ open_quote: str | None = None for index, char in enumerate(query): if open_quote is not None: @@ -112,14 +148,24 @@ def _find_unquoted_colon(query: str) -> int | None: def _strip_surrounding_quotes(text: str) -> str: - """Removes a single pair of matching surrounding quotes (`"` or `'`) from `text`, if present.""" + """ + Removes a single pair of matching surrounding quotes (`"` or `'`) from `text`, if present. + + :param text: + :return: The modified text string. + """ if len(text) >= MIN_QUOTED_LENGTH and text[0] == text[-1] and text[0] in ('"', "'"): return text[1:-1] return text def _load_all_jsonl_logs_from_dataset(dataset: SampleDataset) -> list[dict[str, Any]]: - """Load every JSONL record from a dataset into a single list of dicts.""" + """ + Loads every JSONL log from a dataset into a single list of dicts. + + :param dataset: + :return: A list of dicts, each of which represents a JSONL record. + """ records: list[dict[str, Any]] = [] for path in dataset.metadata.file_names: absolute_path = dataset.logs_path / path @@ -132,30 +178,164 @@ def _load_all_jsonl_logs_from_dataset(dataset: SampleDataset) -> list[dict[str, return records -def _search_log_list_for_kv_pair( - log_list: list[dict[str, Any]], kv_pair: tuple[str, Any] +def _search_all_logs_for_kv_pair( + all_logs: list[dict[str, Any]], + kv_pair: tuple[str, Any], + args: SearchArgs, + metadata: SampleDatasetMetadata, ) -> list[dict[str, Any]]: """ - Searches a list of log entries for those that match a given key-value pair. Both the key - and value may contain `WILDCARD_MULTIMATCH_CHAR`, which matches zero or more characters. + Searches `all_logs` for entries that contain `kv_pair`, then applies the same time-range filter + the search applied (`--begin-time`/`--end-time`). Both the key and the value in `kv_pair` may + contain `WILDCARD_MULTIMATCH_CHAR`, which matches zero or more characters. Following the + semantics of the package's `--ignore-case` flag, `args.ignore_case` makes value matching (but + not key matching) case-insensitive. When `args` specifies a time range, `metadata`'s timestamp + key and format are used to read and interpret each entry's timestamp. + + :param all_logs: + :param kv_pair: + :param args: + :param metadata: + :return: A list of dicts that contain the `kv_pair` and fall within the search's time range. """ key, value = kv_pair - key_regex = _convert_wildcard_to_regex(str(key)) - value_regex = _convert_wildcard_to_regex(str(value)) - return [ + key_regex = _convert_wildcard_to_regex(str(key), ignore_case=False) + value_regex = _convert_wildcard_to_regex(str(value), ignore_case=args.ignore_case) + found_entries = [ entry - for entry in log_list + for entry in all_logs if any( key_regex.fullmatch(str(entry_key)) and value_regex.fullmatch(str(entry_value)) for entry_key, entry_value in entry.items() ) ] + # Filter the found entries if a time-range filter was used in the original query. + if args.begin_ts is not None or args.end_ts is not None: + timestamp_key = metadata.timestamp_key + timestamp_format = metadata.timestamp_format + if timestamp_key is None or timestamp_format is None: + pytest.fail( + "Time-range search verification requires a structured dataset whose metadata" + " defines a top-level 'timestamp_key' and a 'timestamp_format'." + ) + + found_entries = _filter_entries_by_time_range( + found_entries, + timestamp_key, + timestamp_format, + args.begin_ts, + args.end_ts, + ) + + return found_entries -def _convert_wildcard_to_regex(pattern: str) -> re.Pattern[str]: + +def _convert_wildcard_to_regex(pattern: str, ignore_case: bool) -> re.Pattern[str]: """ Compiles a wildcard `pattern` into a regex in which each `WILDCARD_MULTIMATCH_CHAR` matches - zero or more characters and all other characters are matched literally. + zero or more characters and all other characters are matched literally. When `ignore_case` is + set, the compiled regex matches case-insensitively. + + :param pattern: + :param ignore_case: + :return: A compiled regex pattern. """ regex = ".*".join(re.escape(segment) for segment in pattern.split(WILDCARD_MULTIMATCH_CHAR)) - return re.compile(regex, re.DOTALL) + flags = re.DOTALL + if ignore_case: + flags |= re.IGNORECASE + return re.compile(regex, flags) + + +def _filter_entries_by_time_range( + entries: list[dict[str, Any]], + timestamp_key: str, + timestamp_format: TimestampFormat, + begin_ts: int | None, + end_ts: int | None, +) -> list[dict[str, Any]]: + """ + Filters `entries` down to those whose timestamp falls within the inclusive `[begin_ts, end_ts]` + range, mirroring the package's `--begin-time`/`--end-time` semantics. `begin_ts` and `end_ts` + are epoch-millisecond bounds, either of which may be `None` to leave that end unbounded. Each + entry's timestamp is read from the top-level `timestamp_key` and interpreted according to + `timestamp_format`. + + :param entries: + :param timestamp_key: + :param timestamp_format: + :param begin_ts: + :param end_ts: + :return: The entries that fall within the time range. + """ + filtered_entries: list[dict[str, Any]] = [] + for entry in entries: + if timestamp_key not in entry: + pytest.fail(f"Log entry '{entry}' is missing the timestamp key '{timestamp_key}'.") + timestamp = _resolve_timestamp_to_ms(entry[timestamp_key], timestamp_key, timestamp_format) + if (begin_ts is None or begin_ts <= timestamp) and (end_ts is None or timestamp <= end_ts): + filtered_entries.append(entry) + return filtered_entries + + +def _resolve_timestamp_to_ms( + raw_timestamp: Any, + timestamp_key: str, + timestamp_format: TimestampFormat, +) -> int: + """ + Resolves a log entry's raw timestamp value into epoch milliseconds according to + `timestamp_format`. For an `epoch_ms` format, `raw_timestamp` must be an integer number of + milliseconds; for a `strptime` format, it must be a string parseable by the format's pattern. + `timestamp_key` is only used to contextualize failure messages. + + :param raw_timestamp: + :param timestamp_key: + :param timestamp_format: + :return: The timestamp as an integer number of milliseconds since the UNIX epoch. + """ + if isinstance(timestamp_format, EpochMsTimestampFormat): + if not isinstance(raw_timestamp, int): + pytest.fail( + f"Log entry timestamp '{raw_timestamp}' under key '{timestamp_key}' is not an" + " integer, which the 'epoch_ms' timestamp kind requires." + ) + return raw_timestamp + + if not isinstance(raw_timestamp, str): + pytest.fail( + f"Log entry timestamp '{raw_timestamp}' under key '{timestamp_key}' is not a string," + " which the 'strptime' timestamp kind requires." + ) + + try: + return parse_timestamp_to_epoch_ms(raw_timestamp, timestamp_format.pattern) + except ValueError: + pytest.fail( + f"Failed to parse log entry timestamp '{raw_timestamp}' under key '{timestamp_key}'" + f" using pattern '{timestamp_format.pattern}'." + ) + + +def _extract_count_from_search_output(search_output: str) -> str: + """ + Extracts the count reported by a count-style search. + + :param search_output: + :return: The reported count, followed by a newline. + """ + match = re.search(r"count: (\d+)", search_output) + if match: + return match.group(1) + "\n" + pytest.fail(f"The search result '{search_output}' wasn't in the correct format.") + + +def _normalize_entries(entries: list[dict[str, Any]]) -> list[str]: + """ + Serializes each entry into a canonical JSON string and returns the sorted list. + + :param entries: + :return: A sorted list of canonical JSON strings. + """ + return sorted(json.dumps(entry, sort_keys=True) for entry in entries) diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index b802cbe59a..304a958e63 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -7,8 +7,10 @@ from tests.package_tests.classes import ClpPackage from tests.package_tests.clp_text.utils.mode import CLP_TEXT_MODE from tests.package_tests.clp_text.verification.compress import verify_compress_clp_text +from tests.package_tests.clp_text.verification.search import verify_search_clp_text from tests.package_tests.utils.compress import CompressArgs -from tests.utils.classes import ClpAction, SampleDataset +from tests.package_tests.utils.search import SearchArgs +from tests.utils.classes import ClpAction, SampleDataset, SampleDatasetMetadata logger = logging.getLogger(__name__) @@ -58,3 +60,54 @@ def test_clp_text_compression_text_multifile( logger.info("Verifying the compression of the 'text_multifile' dataset.") result = verify_compress_clp_text(action, clp_package, text_multifile) assert result, result.failure_message + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_time_range_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package honours `--begin-time`/`--end-time` when searching the + `text-multifile` dataset. The time range is a strict sub-range of the dataset's span + (`[begin_ts + 1, end_ts - 1]`), which is guaranteed to exclude the earliest and latest log + events. + + :param clp_package: + :param text_multifile: + """ + metadata: SampleDatasetMetadata = text_multifile.metadata + compress_args = CompressArgs( + script_path=clp_package.path_config.compress_path, + config=clp_package.temp_config_file_path, + paths=[text_multifile.logs_path], + ) + + logger.info("Compressing the 'text_multifile' dataset with the 'clp-text' package.") + compress_action = ClpAction.from_args(compress_args) + compress_result = compress_action.verify_returncode() + assert compress_result, compress_result.failure_message + + logger.info("Verifying the compression of the 'text_multifile' dataset.") + compress_result = verify_compress_clp_text(compress_action, clp_package, text_multifile) + assert compress_result, compress_result.failure_message + + search_args = SearchArgs( + script_path=clp_package.path_config.search_path, + config=clp_package.temp_config_file_path, + query="nominal", + begin_ts=metadata.begin_ts + 1, + end_ts=metadata.end_ts - 1, + ) + + logger.info( + "Searching the 'text_multifile' dataset over a time range with the 'clp-text' package." + ) + search_action = ClpAction.from_args(search_args) + search_result = search_action.verify_returncode() + assert search_result, search_result.failure_message + + logger.info("Verifying the time-range search results for the 'text_multifile' dataset.") + search_result = verify_search_clp_text(search_action, text_multifile) + assert search_result, search_result.failure_message diff --git a/integration-tests/tests/package_tests/clp_text/verification/search.py b/integration-tests/tests/package_tests/clp_text/verification/search.py index 276bb942de..76c7432708 100644 --- a/integration-tests/tests/package_tests/clp_text/verification/search.py +++ b/integration-tests/tests/package_tests/clp_text/verification/search.py @@ -1 +1,187 @@ """Search verification helpers specific to the clp-text package.""" + +import logging +import re + +import pytest + +from tests.package_tests.utils.search import parse_timestamp_to_epoch_ms, SearchArgs +from tests.utils.classes import ( + ClpAction, + ClpVerificationResult, + EpochMsTimestampFormat, + NonClpAction, + SampleDataset, + TimestampFormat, +) +from tests.utils.utils import ( + get_binary_path, +) + +logger = logging.getLogger(__name__) + + +def verify_search_clp_text( + action: ClpAction, + dataset: SampleDataset, +) -> ClpVerificationResult: + """ + Verifies that the search action performed on the `clp-text` package returns the expected + results. Verification is performed as follows: + + 1. A grep command written with respect to the original search arguments is run on the dataset. + 2. The grep output lines are filtered by time range when the search specifies one + (`--begin-time`/`--end-time`). + 3. The filtered grep lines are compared against the original search action's output. For a + count-style search (`--count`), the number of matched lines is compared against the reported + count; otherwise the set of matched lines is compared against the set of output lines. + + :param action: + :param dataset: + :return: A `ClpVerificationResult` indicating the success or failure of the verification. + """ + # Verify that the action's arguments are of the expected type. + args = action.args + if not isinstance(args, SearchArgs): + err_msg = "Verification expects a 'SearchArgs' action." + raise TypeError(err_msg) + + # Construct and run grep command. + grep_action = NonClpAction(cmd=_construct_grep_verification_cmd(args, dataset)) + grep_action.check_returncode(dependent_action=action) + grep_lines = grep_action.completed_proc.stdout.splitlines() + + # Filter the found entries if a time-range filter was used in the original query. + if args.begin_ts is not None or args.end_ts is not None: + grep_lines = _filter_lines_by_time_range( + grep_lines, dataset.metadata.timestamp_format, args.begin_ts, args.end_ts + ) + + # Compare the matched grep lines with the search action output. + search_output = action.completed_proc.stdout + if args.is_count_query: + expected_count = str(len(grep_lines)) + "\n" + actual_count = _extract_count_from_search_output(search_output) + + if expected_count == actual_count: + return action.pass_verification() + + return action.fail_verification( + "Search verification failure: mismatch between search count and expected count.\n" + f"Expected count: '{expected_count}'\n" + f"Actual count: '{actual_count}'", + supporting_action=grep_action, + ) + + formatted_grep_result = "\n".join(sorted(grep_lines)) + formatted_search_result = "\n".join(sorted(search_output.splitlines())) + if formatted_grep_result == formatted_search_result: + return action.pass_verification() + + return action.fail_verification( + f"Search verification failure: mismatch between formatted search result and formatted grep" + f" result.\n" + f"Formatted search result: '{formatted_search_result}'\n" + f"Formatted grep result: '{formatted_grep_result}'", + supporting_action=grep_action, + ) + + +def _construct_grep_verification_cmd( + args: SearchArgs, + dataset: SampleDataset, +) -> list[str]: + """ + Constructs a grep command that reproduces the search described by `args` on `dataset`'s original + logs. + + :param args: + :param dataset: + :return: The grep command as a list of arguments. + """ + grep_cmd_options: list[str] = [ + "--recursive", + "--no-filename", + "--color=never", + ] + + if args.ignore_case: + grep_cmd_options.append("--ignore-case") + + path_for_grep = args.file_path or dataset.logs_path + return [ + get_binary_path("grep"), + *grep_cmd_options, + args.query, + str(path_for_grep), + ] + + +def _filter_lines_by_time_range( + lines: list[str], + timestamp_format: TimestampFormat | None, + begin_ts: int | None, + end_ts: int | None, +) -> list[str]: + """ + Filters `lines` down to those whose leading timestamp falls within the inclusive + `[begin_ts, end_ts]` range, mirroring the package's `--begin-time`/`--end-time` semantics. + `begin_ts` and `end_ts` are epoch-millisecond bounds, either of which may be `None` to leave + that end unbounded. Each line must begin with a whitespace-delimited timestamp token interpreted + according to `timestamp_format`. + + :param lines: + :param timestamp_format: + :param begin_ts: + :param end_ts: + :return: The lines that fall within the time range. + """ + if timestamp_format is None: + pytest.fail( + "clp-text time-range search verification requires the dataset's metadata to define a" + " 'timestamp_format'." + ) + + filtered_entries: list[str] = [] + for line in lines: + if not line.strip(): + continue + timestamp = _parse_leading_timestamp_ms(line, timestamp_format) + if (begin_ts is None or begin_ts <= timestamp) and (end_ts is None or timestamp <= end_ts): + filtered_entries.append(line) + return filtered_entries + + +def _parse_leading_timestamp_ms(line: str, timestamp_format: TimestampFormat) -> int: + """ + Parses the leading whitespace-delimited timestamp token of `line` according to + `timestamp_format`. + + :param line: + :param timestamp_format: + :return: The parsed timestamp as an integer number of milliseconds since the UNIX epoch. + """ + token = line.split(maxsplit=1)[0] + if isinstance(timestamp_format, EpochMsTimestampFormat): + return int(token) + + try: + return parse_timestamp_to_epoch_ms(token, timestamp_format.pattern) + except ValueError: + pytest.fail( + f"Failed to parse timestamp token '{token}' from line '{line}' using pattern" + f" '{timestamp_format.pattern}'." + ) + + +def _extract_count_from_search_output(search_output: str) -> str: + """ + Extracts the count reported by a count-style search. + + :param search_output: + :return: The reported count, followed by a newline. + """ + match = re.search(r"count: (\d+)", search_output) + if match: + return match.group(1) + "\n" + pytest.fail(f"The search result '{search_output}' wasn't in the correct format.") diff --git a/integration-tests/tests/package_tests/utils/search.py b/integration-tests/tests/package_tests/utils/search.py index c28d41cf8c..0315f93912 100644 --- a/integration-tests/tests/package_tests/utils/search.py +++ b/integration-tests/tests/package_tests/utils/search.py @@ -1,23 +1,11 @@ """Classes to facilitate CLP package search testing.""" import logging -import re -from enum import auto, Enum +from datetime import datetime, timezone from pathlib import Path -import pytest -from clp_py_utils.clp_config import StorageEngine - -from tests.package_tests.classes import ClpPackage from tests.utils.classes import ( - ClpAction, - ClpVerificationResult, CmdArgs, - NonClpAction, - SampleDataset, -) -from tests.utils.utils import ( - get_binary_path, ) logger = logging.getLogger(__name__) @@ -26,6 +14,25 @@ DEFAULT_COUNT_BY_TIME_INTERVAL = 10 +def parse_timestamp_to_epoch_ms(timestamp: str, strptime_pattern: str) -> int: + """ + Parses `timestamp` using the `datetime.strptime` pattern `strptime_pattern` into an integer + number of milliseconds since the UNIX epoch. A timestamp without an offset in the pattern is + interpreted as UTC, while a pattern that captures an offset (e.g. `%z`) is honoured. This is the + single strptime-parsing mechanism shared by the clp-json and clp-text time-range verifiers so + that both agree on epoch conversion. + + :param timestamp: + :param strptime_pattern: + :raises ValueError: If `timestamp` doesn't match `strptime_pattern`. + :return: The parsed timestamp as an integer number of milliseconds since the UNIX epoch. + """ + parsed = datetime.strptime(timestamp, strptime_pattern) # noqa: DTZ007 + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return round(parsed.timestamp() * 1000) + + class SearchArgs(CmdArgs): """Command argument model for searching with the CLP package.""" @@ -41,6 +48,11 @@ class SearchArgs(CmdArgs): begin_ts: int | None = None end_ts: int | None = None + @property + def is_count_query(self) -> bool: + """:return: `True` if this object represents a count-style search; else `False`.""" + return self.count or self.count_by_time is not None + def to_cmd(self) -> list[str]: """Converts the model attributes to a command list.""" cmd: list[str] = [ @@ -74,204 +86,3 @@ def to_cmd(self) -> list[str]: cmd.append(self.query) return cmd - - -class ClpPackageSearchType(Enum): - """Possible search types.""" - - BASIC = auto() - FILE_PATH = auto() - IGNORE_CASE = auto() - COUNT_RESULTS = auto() - COUNT_BY_TIME = auto() - TIME_RANGE = auto() - - -def search_clp_package( - clp_package: ClpPackage, - dataset: SampleDataset, - search_type: ClpPackageSearchType, - query: str, -) -> ClpAction: - """ - Performs the specified search on the dataset using the CLP package. - - :param clp_package: - :param dataset: - :param search_type: - :param query: - :return: The `ClpAction` instance that runs the search. - """ - logger.info( - "Performing '%s' search on the '%s' dataset.", - search_type.name, - dataset.dataset_name, - ) - - args: SearchArgs = _construct_args(clp_package, dataset, search_type, query) - return ClpAction.from_args(args) - - -def _construct_args( - clp_package: ClpPackage, - dataset: SampleDataset, - search_type: ClpPackageSearchType, - query: str, -) -> SearchArgs: - """Construct the `SearchArgs` object for the specified search on the dataset.""" - path_config = clp_package.path_config - args = SearchArgs( - script_path=path_config.search_path, - config=clp_package.temp_config_file_path, - query=query, - ) - - if clp_package.clp_config.package.storage_engine == StorageEngine.CLP_S: - args.dataset = dataset.metadata.dataset_name - - match search_type: - case ClpPackageSearchType.BASIC: - pass - case ClpPackageSearchType.FILE_PATH: - pytest.fail("FILE_PATH search not yet implemented.") - case ClpPackageSearchType.IGNORE_CASE: - args.ignore_case = True - case ClpPackageSearchType.COUNT_RESULTS: - args.count = True - case ClpPackageSearchType.COUNT_BY_TIME: - args.count_by_time = DEFAULT_COUNT_BY_TIME_INTERVAL - case ClpPackageSearchType.TIME_RANGE: - args.begin_ts = dataset.metadata.begin_ts - args.end_ts = dataset.metadata.end_ts - case _: - pytest.fail(f"Unsupported search type for CLP package: '{search_type}'") - - return args - - -def verify_search_action( - action: ClpAction, - search_type: ClpPackageSearchType, - original_dataset: SampleDataset, -) -> ClpVerificationResult: - """ - Verifies the search action. - - :param action: - :param search_type: - :param original_dataset: - :return: A `ClpVerificationResult` indicating the success or failure of the verification. - """ - logger.info( - "Verifying '%s' search on the '%s' dataset.", - search_type.name, - original_dataset.dataset_name, - ) - - result = action.verify_returncode() - if not result: - return result - - args = action.args - assert isinstance(args, SearchArgs) - - # Construct and run grep command. - grep_action = NonClpAction( - cmd=_construct_grep_verification_cmd(args, search_type, original_dataset) - ) - grep_action.check_returncode(dependent_action=action) - - # Compare grep result with search result. - formatted_grep_result = _format_grep_result_for_search_type( - grep_action.completed_proc.stdout, search_type - ) - formatted_search_result = _format_search_result_for_search_type( - action.completed_proc.stdout, search_type - ) - if formatted_grep_result == formatted_search_result: - return action.pass_verification() - - return action.fail_verification( - f"Search verification failure: mismatch between formatted search result" - f" '{formatted_search_result}' and formatted grep result '{formatted_grep_result}'.", - supporting_action=grep_action, - ) - - -def _construct_grep_verification_cmd( - args: SearchArgs, - search_type: ClpPackageSearchType, - original_dataset: SampleDataset, -) -> list[str]: - grep_cmd_options = _get_grep_options_from_search_type(search_type) - path_for_grep = args.file_path or original_dataset.logs_path - return [ - get_binary_path("grep"), - *grep_cmd_options, - args.query, - str(path_for_grep), - ] - - -def _get_grep_options_from_search_type(search_type: ClpPackageSearchType) -> list[str]: - grep_cmd_options: list[str] = [ - "--recursive", - "--no-filename", - "--color=never", - ] - - match search_type: - case ( - ClpPackageSearchType.BASIC - | ClpPackageSearchType.FILE_PATH - | ClpPackageSearchType.COUNT_RESULTS - | ClpPackageSearchType.COUNT_BY_TIME - | ClpPackageSearchType.TIME_RANGE - ): - return grep_cmd_options - case ClpPackageSearchType.IGNORE_CASE: - grep_cmd_options.append("--ignore-case") - return grep_cmd_options - case _: - pytest.fail( - f"Search type '{search_type.name}' not configured for grep command construction." - ) - - -def _format_grep_result_for_search_type(grep_result: str, search_type: ClpPackageSearchType) -> str: - match search_type: - case ( - ClpPackageSearchType.BASIC - | ClpPackageSearchType.FILE_PATH - | ClpPackageSearchType.IGNORE_CASE - | ClpPackageSearchType.TIME_RANGE - ): - return grep_result - case ClpPackageSearchType.COUNT_RESULTS | ClpPackageSearchType.COUNT_BY_TIME: - return str(len(grep_result.splitlines())) + "\n" - case _: - pytest.fail( - f"Search type '{search_type.name}' not configured for grep result formatting." - ) - - -def _format_search_result_for_search_type( - search_result: str, search_type: ClpPackageSearchType -) -> str: - match search_type: - case ( - ClpPackageSearchType.BASIC - | ClpPackageSearchType.FILE_PATH - | ClpPackageSearchType.IGNORE_CASE - | ClpPackageSearchType.TIME_RANGE - ): - return search_result - case ClpPackageSearchType.COUNT_RESULTS | ClpPackageSearchType.COUNT_BY_TIME: - match = re.search(r"count: (\d+)", search_result) - if match: - return match.group(1) + "\n" - pytest.fail(f"The search result '{search_result}' wasn't in the correct format.") - case _: - pytest.fail( - f"Search type '{search_type.name}' not configured for search result formatting." - ) diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index 5209847066..f2e80d8688 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -8,6 +8,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path +from typing import Literal import pytest from pydantic import BaseModel @@ -76,6 +77,22 @@ def _static_paths(self) -> list[Path]: return [self.test_data_dir] +class EpochMsTimestampFormat(BaseModel): + """A timestamp encoded as an integer number of milliseconds since the UNIX epoch.""" + + kind: Literal["epoch_ms"] + + +class StrptimeTimestampFormat(BaseModel): + """A timestamp encoded as a string parseable by a `datetime.strptime` pattern.""" + + kind: Literal["strptime"] + pattern: str + + +TimestampFormat = EpochMsTimestampFormat | StrptimeTimestampFormat + + class SampleDatasetMetadata(BaseModel): """ Metadata for a sample dataset. All `/metadata.json` files must conform to this @@ -85,6 +102,7 @@ class SampleDatasetMetadata(BaseModel): dataset_name: str unstructured: bool timestamp_key: str | None + timestamp_format: TimestampFormat | None begin_ts: int end_ts: int logs_subdir: str From 6c678f71b2c92454795761a8863c99f5eab6b028 Mon Sep 17 00:00:00 2001 From: Quinn Date: Fri, 3 Jul 2026 19:30:31 +0000 Subject: [PATCH 17/19] Fix docstrings, README, and imports. --- integration-tests/tests/data/README.md | 12 +++-- .../clp_json/verification/search.py | 49 +++++++------------ .../clp_text/verification/search.py | 21 +++----- .../tests/package_tests/utils/search.py | 11 ++--- integration-tests/tests/utils/classes.py | 4 +- 5 files changed, 35 insertions(+), 62 deletions(-) diff --git a/integration-tests/tests/data/README.md b/integration-tests/tests/data/README.md index e12471f380..fa9c1cd16b 100644 --- a/integration-tests/tests/data/README.md +++ b/integration-tests/tests/data/README.md @@ -63,6 +63,9 @@ following rules must be observed: | `kind` | `"epoch_ms"` for epoch ms timestamps, or `"strptime"` for formatted strings. | | `pattern` | (`strptime` only) The Python [`strptime`][strptime] pattern for the timestamp. | +3. For **structured** datasets: files in the dataset must be `.jsonl`, with only one log record per + line. + ## Time-range search verification Search verification reproduces the package's `--begin-time`/`--end-time` filter by independently @@ -71,15 +74,14 @@ following restrictions must be observed: 1. `begin_ts` must equal the earliest timestamp present in the dataset, and `end_ts` the latest. 2. `begin_ts` must be less than or equal to `end_ts`. -3. For **structured** datasets: +3. `timestamp_format` must not be `null`. +4. For **structured** datasets: * `timestamp_key` must not be `null`, and must be a *top-level* key. Nested/dotted keys are not supported. - * `timestamp_format` must not be `null`. * Each log in the dataset must contain `timestamp_key`. -4. For **unstructured** datasets: - * `timestamp_format` must not be `null`. +5. For **unstructured** datasets: * Every log line must begin with a single whitespace-delimited timestamp token written in the - format described by `timestamp_format`. + format described by `timestamp_format.pattern`. ## Accessing sample datasets within the testing system diff --git a/integration-tests/tests/package_tests/clp_json/verification/search.py b/integration-tests/tests/package_tests/clp_json/verification/search.py index a0a7c24b8f..44c430695a 100644 --- a/integration-tests/tests/package_tests/clp_json/verification/search.py +++ b/integration-tests/tests/package_tests/clp_json/verification/search.py @@ -36,10 +36,8 @@ def verify_search_clp_json( results. Verification is performed as follows: 1. The query string from the search action is parsed into a key-value pair. - 2. The dataset's log entries are gathered into a list of dicts: read directly from the original - JSONL files for a structured dataset, or recovered by re-running the search with a match-all - query for an unstructured dataset (whose original logs aren't JSON). - 3. The list of dicts is searched for entries that match the key-value pair. + 2. The dataset's log entries are loaded into a list of dicts for processing. + 3. The list of dicts is searched for entries that match or contain the key-value pair. 4. The found entries are compared against the original search action's output. For a count-style search (`--count`), the number of found entries is compared against the reported count; otherwise the set of found entries is compared against the set of output logs. @@ -49,19 +47,16 @@ def verify_search_clp_json( :param dataset: :return: A `ClpVerificationResult` indicating the success or failure of the verification. """ - # Verify that the action's arguments are of the expected type. args = action.args if not isinstance(args, SearchArgs): err_msg = "Verification expects a 'SearchArgs' action." raise TypeError(err_msg) - # Construct KV-pair from query. kv_pair = _construct_kv_pair_from_query(args.query) - # Convert original logs to list[dict] objects for processing. metadata: SampleDatasetMetadata = dataset.metadata if metadata.unstructured: - # Get the structured version of the unstructured data. Could use log-converter directly. + # Use search.sh to get the structured version of the unstructured data. supporting_search_args: SearchArgs = SearchArgs( script_path=clp_package.path_config.search_path, config=clp_package.temp_config_file_path, @@ -82,12 +77,10 @@ def verify_search_clp_json( else: all_logs = _load_all_jsonl_logs_from_dataset(dataset) - # Find entries that match the kv_pair from the original query. found_entries: list[dict[str, Any]] = _search_all_logs_for_kv_pair( all_logs, kv_pair, args, metadata ) - # Compare the found entries with the search action output. search_output = action.completed_proc.stdout if args.is_count_query: expected_count = str(len(found_entries)) + "\n" @@ -152,7 +145,7 @@ def _strip_surrounding_quotes(text: str) -> str: Removes a single pair of matching surrounding quotes (`"` or `'`) from `text`, if present. :param text: - :return: The modified text string. + :return: The processed text string. """ if len(text) >= MIN_QUOTED_LENGTH and text[0] == text[-1] and text[0] in ('"', "'"): return text[1:-1] @@ -185,18 +178,14 @@ def _search_all_logs_for_kv_pair( metadata: SampleDatasetMetadata, ) -> list[dict[str, Any]]: """ - Searches `all_logs` for entries that contain `kv_pair`, then applies the same time-range filter - the search applied (`--begin-time`/`--end-time`). Both the key and the value in `kv_pair` may - contain `WILDCARD_MULTIMATCH_CHAR`, which matches zero or more characters. Following the - semantics of the package's `--ignore-case` flag, `args.ignore_case` makes value matching (but - not key matching) case-insensitive. When `args` specifies a time range, `metadata`'s timestamp - key and format are used to read and interpret each entry's timestamp. + Searches `all_logs` for entries that contain `kv_pair`. The search respects the various + constraints posed by the content of `kv_pair`, `args` and `metadata`. :param all_logs: :param kv_pair: :param args: :param metadata: - :return: A list of dicts that contain the `kv_pair` and fall within the search's time range. + :return: A list of dicts that contain the `kv_pair` when searched w.r.t. the constraints. """ key, value = kv_pair key_regex = _convert_wildcard_to_regex(str(key), ignore_case=False) @@ -216,8 +205,8 @@ def _search_all_logs_for_kv_pair( timestamp_format = metadata.timestamp_format if timestamp_key is None or timestamp_format is None: pytest.fail( - "Time-range search verification requires a structured dataset whose metadata" - " defines a top-level 'timestamp_key' and a 'timestamp_format'." + "clp-json time-range search verification requires the original dataset's metadata" + " to define a top-level 'timestamp_key' and a 'timestamp_format'." ) found_entries = _filter_entries_by_time_range( @@ -257,10 +246,8 @@ def _filter_entries_by_time_range( ) -> list[dict[str, Any]]: """ Filters `entries` down to those whose timestamp falls within the inclusive `[begin_ts, end_ts]` - range, mirroring the package's `--begin-time`/`--end-time` semantics. `begin_ts` and `end_ts` - are epoch-millisecond bounds, either of which may be `None` to leave that end unbounded. Each - entry's timestamp is read from the top-level `timestamp_key` and interpreted according to - `timestamp_format`. + range. Each entry's timestamp is read from the top-level `timestamp_key` and interpreted + according to `timestamp_format`. :param entries: :param timestamp_key: @@ -285,10 +272,8 @@ def _resolve_timestamp_to_ms( timestamp_format: TimestampFormat, ) -> int: """ - Resolves a log entry's raw timestamp value into epoch milliseconds according to - `timestamp_format`. For an `epoch_ms` format, `raw_timestamp` must be an integer number of - milliseconds; for a `strptime` format, it must be a string parseable by the format's pattern. - `timestamp_key` is only used to contextualize failure messages. + Converts a log entry's raw timestamp value into UNIX epoch ms according to `timestamp_format`, + if not already given in UNIX epoch ms. :param raw_timestamp: :param timestamp_key: @@ -298,15 +283,15 @@ def _resolve_timestamp_to_ms( if isinstance(timestamp_format, EpochMsTimestampFormat): if not isinstance(raw_timestamp, int): pytest.fail( - f"Log entry timestamp '{raw_timestamp}' under key '{timestamp_key}' is not an" - " integer, which the 'epoch_ms' timestamp kind requires." + f"Log entry timestamp '{raw_timestamp}' under key '{timestamp_key}' should be an" + " integer, but it is not." ) return raw_timestamp if not isinstance(raw_timestamp, str): pytest.fail( - f"Log entry timestamp '{raw_timestamp}' under key '{timestamp_key}' is not a string," - " which the 'strptime' timestamp kind requires." + f"Log entry timestamp '{raw_timestamp}' under key '{timestamp_key}' should be a string," + " but it is not." ) try: diff --git a/integration-tests/tests/package_tests/clp_text/verification/search.py b/integration-tests/tests/package_tests/clp_text/verification/search.py index 76c7432708..bc837d6366 100644 --- a/integration-tests/tests/package_tests/clp_text/verification/search.py +++ b/integration-tests/tests/package_tests/clp_text/verification/search.py @@ -14,9 +14,7 @@ SampleDataset, TimestampFormat, ) -from tests.utils.utils import ( - get_binary_path, -) +from tests.utils.utils import get_binary_path logger = logging.getLogger(__name__) @@ -30,9 +28,7 @@ def verify_search_clp_text( results. Verification is performed as follows: 1. A grep command written with respect to the original search arguments is run on the dataset. - 2. The grep output lines are filtered by time range when the search specifies one - (`--begin-time`/`--end-time`). - 3. The filtered grep lines are compared against the original search action's output. For a + 2. The filtered grep lines are compared against the original search action's output. For a count-style search (`--count`), the number of matched lines is compared against the reported count; otherwise the set of matched lines is compared against the set of output lines. @@ -40,13 +36,11 @@ def verify_search_clp_text( :param dataset: :return: A `ClpVerificationResult` indicating the success or failure of the verification. """ - # Verify that the action's arguments are of the expected type. args = action.args if not isinstance(args, SearchArgs): err_msg = "Verification expects a 'SearchArgs' action." raise TypeError(err_msg) - # Construct and run grep command. grep_action = NonClpAction(cmd=_construct_grep_verification_cmd(args, dataset)) grep_action.check_returncode(dependent_action=action) grep_lines = grep_action.completed_proc.stdout.splitlines() @@ -57,7 +51,6 @@ def verify_search_clp_text( grep_lines, dataset.metadata.timestamp_format, args.begin_ts, args.end_ts ) - # Compare the matched grep lines with the search action output. search_output = action.completed_proc.stdout if args.is_count_query: expected_count = str(len(grep_lines)) + "\n" @@ -125,10 +118,8 @@ def _filter_lines_by_time_range( ) -> list[str]: """ Filters `lines` down to those whose leading timestamp falls within the inclusive - `[begin_ts, end_ts]` range, mirroring the package's `--begin-time`/`--end-time` semantics. - `begin_ts` and `end_ts` are epoch-millisecond bounds, either of which may be `None` to leave - that end unbounded. Each line must begin with a whitespace-delimited timestamp token interpreted - according to `timestamp_format`. + `[begin_ts, end_ts]` range. Each line must begin with a whitespace-delimited timestamp token + interpreted according to `timestamp_format`. :param lines: :param timestamp_format: @@ -138,8 +129,8 @@ def _filter_lines_by_time_range( """ if timestamp_format is None: pytest.fail( - "clp-text time-range search verification requires the dataset's metadata to define a" - " 'timestamp_format'." + "clp-text time-range search verification requires the original dataset's metadata to" + " define a 'timestamp_format'." ) filtered_entries: list[str] = [] diff --git a/integration-tests/tests/package_tests/utils/search.py b/integration-tests/tests/package_tests/utils/search.py index 0315f93912..43c6db6c54 100644 --- a/integration-tests/tests/package_tests/utils/search.py +++ b/integration-tests/tests/package_tests/utils/search.py @@ -4,9 +4,7 @@ from datetime import datetime, timezone from pathlib import Path -from tests.utils.classes import ( - CmdArgs, -) +from tests.utils.classes import CmdArgs logger = logging.getLogger(__name__) @@ -16,11 +14,8 @@ def parse_timestamp_to_epoch_ms(timestamp: str, strptime_pattern: str) -> int: """ - Parses `timestamp` using the `datetime.strptime` pattern `strptime_pattern` into an integer - number of milliseconds since the UNIX epoch. A timestamp without an offset in the pattern is - interpreted as UTC, while a pattern that captures an offset (e.g. `%z`) is honoured. This is the - single strptime-parsing mechanism shared by the clp-json and clp-text time-range verifiers so - that both agree on epoch conversion. + Parses `timestamp` using `strptime_pattern` into an integer number of milliseconds since the + UNIX epoch. A timestamp without an offset in the pattern is interpreted as UTC. :param timestamp: :param strptime_pattern: diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index f2e80d8688..80249c5de9 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -78,13 +78,13 @@ def _static_paths(self) -> list[Path]: class EpochMsTimestampFormat(BaseModel): - """A timestamp encoded as an integer number of milliseconds since the UNIX epoch.""" + """Indicates a timestamp encoded as an integer number of milliseconds since the UNIX epoch.""" kind: Literal["epoch_ms"] class StrptimeTimestampFormat(BaseModel): - """A timestamp encoded as a string parseable by a `datetime.strptime` pattern.""" + """Indicates a timestamp encoded as a string parseable by a `datetime.strptime` pattern.""" kind: Literal["strptime"] pattern: str From 8fb7480a75659ec0f129518c57466abccc38e8c7 Mon Sep 17 00:00:00 2001 From: Quinn Date: Sat, 4 Jul 2026 04:00:58 +0000 Subject: [PATCH 18/19] Fix count-by-time search and start adding test functions. --- .../package_tests/clp_json/test_clp_json.py | 355 ++++++++++++------ .../clp_json/verification/search.py | 40 +- .../clp_text/verification/search.py | 11 +- 3 files changed, 272 insertions(+), 134 deletions(-) diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index 2bebe54943..71fa56ea00 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -49,29 +49,12 @@ def test_clp_json_compression_json_multifile( json_multifile: SampleDataset, ) -> None: """ - Validate that the `clp-json` package successfully compresses the `json-multifile` dataset. + Validate that the `clp-json` package successfully compresses the `json_multifile` dataset. :param clp_package: :param json_multifile: """ - metadata: SampleDatasetMetadata = json_multifile.metadata - args: CompressArgs = CompressArgs( - script_path=clp_package.path_config.compress_path, - config=clp_package.temp_config_file_path, - dataset=metadata.dataset_name, - timestamp_key=metadata.timestamp_key, - unstructured=metadata.unstructured, - paths=[json_multifile.logs_path], - ) - - logger.info("Compressing the 'json_multifile' dataset with the 'clp-json' package.") - action = ClpAction.from_args(args) - result = action.verify_returncode() - assert result, result.failure_message - - logger.info("Verifying the compression of the 'json_multifile' dataset.") - result = verify_compress_structured_clp_json(action, clp_package, json_multifile) - assert result, result.failure_message + _compress_structured_dataset(clp_package, json_multifile) @pytest.mark.compression @@ -81,80 +64,79 @@ def test_clp_json_compression_text_multifile( text_multifile: SampleDataset, ) -> None: """ - Validate that the `clp-json` package successfully compresses the `text-multifile` dataset as + Validate that the `clp-json` package successfully compresses the `text_multifile` dataset as unstructured text. :param clp_package: :param text_multifile: """ - metadata: SampleDatasetMetadata = text_multifile.metadata - args: CompressArgs = CompressArgs( - script_path=clp_package.path_config.compress_path, - config=clp_package.temp_config_file_path, - dataset=metadata.dataset_name, - timestamp_key=metadata.timestamp_key, - unstructured=metadata.unstructured, - paths=[text_multifile.logs_path], - ) + _compress_unstructured_dataset(clp_package, text_multifile) - logger.info("Compressing the 'text_multifile' dataset with the 'clp-json' package.") - action = ClpAction.from_args(args) - result = action.verify_returncode() - assert result, result.failure_message - logger.info("Verifying the compression of the 'text_multifile' dataset.") - result = verify_compress_unstructured_clp_json(action, clp_package, text_multifile) - assert result, result.failure_message +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_basic_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a basic search on the `json_multifile` dataset. + + :param clp_package: + :param json_multifile: + """ + _compress_structured_dataset(clp_package, json_multifile) + _search_basic(clp_package, json_multifile, 'detail: "*maximum dynamic*"') @pytest.mark.search @pytest.mark.usefixtures("clear_package_archives") -def test_clp_json_search_json_multifile( +def test_clp_json_search_ignore_case_json_multifile( clp_package: ClpPackage, json_multifile: SampleDataset, ) -> None: """ - Validate that the `clp-json` package successfully searches the `json-multifile` dataset. + Validate that the `clp-json` package performs a case-insensitive search on the `json_multifile` + dataset. :param clp_package: :param json_multifile: """ - metadata: SampleDatasetMetadata = json_multifile.metadata - compress_args: CompressArgs = CompressArgs( - script_path=clp_package.path_config.compress_path, - config=clp_package.temp_config_file_path, - dataset=metadata.dataset_name, - timestamp_key=metadata.timestamp_key, - unstructured=metadata.unstructured, - paths=[json_multifile.logs_path], - ) + _compress_structured_dataset(clp_package, json_multifile) + _search_ignore_case(clp_package, json_multifile, 'detail: "*mAxImUm DyNaMiC*"') - logger.info("Compressing the 'json_multifile' dataset with the 'clp-json' package.") - compress_action = ClpAction.from_args(compress_args) - compress_result = compress_action.verify_returncode() - assert compress_result, compress_result.failure_message - logger.info("Verifying the compression of the 'json_multifile' dataset.") - compress_result = verify_compress_structured_clp_json( - compress_action, clp_package, json_multifile - ) - assert compress_result, compress_result.failure_message +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_results_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count search on the `json_multifile` dataset. - search_args: SearchArgs = SearchArgs( - script_path=clp_package.path_config.search_path, - config=clp_package.temp_config_file_path, - query='detail: "*maximum dynamic*"', - dataset=metadata.dataset_name, - ) + :param clp_package: + :param json_multifile: + """ + _compress_structured_dataset(clp_package, json_multifile) + _search_count_results(clp_package, json_multifile, 'detail: "*maximum dynamic*"') - logger.info("Searching the 'json_multifile' dataset with the 'clp-json' package.") - search_action = ClpAction.from_args(search_args) - search_result = search_action.verify_returncode() - assert search_result, search_result.failure_message - logger.info("Verifying the search results for the 'json_multifile' dataset.") - search_result = verify_search_clp_json(search_action, clp_package, json_multifile) - assert search_result, search_result.failure_message +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_by_time_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count-by-time search on the `json_multifile` + dataset. + + :param clp_package: + :param json_multifile: + """ + _compress_structured_dataset(clp_package, json_multifile) + _search_count_by_time(clp_package, json_multifile, 'mission: "STS-135"', 10) @pytest.mark.search @@ -165,97 +147,228 @@ def test_clp_json_search_time_range_json_multifile( ) -> None: """ Validate that the `clp-json` package honours `--begin-time`/`--end-time` when searching the - `json-multifile` dataset. The time range is a strict sub-range of the dataset's span + `json_multifile` dataset. The time range is a strict sub-range of the dataset's span (`[begin_ts + 1, end_ts - 1]`), which is guaranteed to exclude the earliest and latest records. :param clp_package: :param json_multifile: """ + _compress_structured_dataset(clp_package, json_multifile) metadata: SampleDatasetMetadata = json_multifile.metadata - compress_args: CompressArgs = CompressArgs( - script_path=clp_package.path_config.compress_path, - config=clp_package.temp_config_file_path, - dataset=metadata.dataset_name, - timestamp_key=metadata.timestamp_key, - unstructured=metadata.unstructured, - paths=[json_multifile.logs_path], - ) + begin_ts = metadata.begin_ts + 1 + end_ts = metadata.end_ts - 1 + _search_time_range(clp_package, json_multifile, 'mission: "STS-135"', begin_ts, end_ts) - logger.info("Compressing the 'json_multifile' dataset with the 'clp-json' package.") - compress_action = ClpAction.from_args(compress_args) - compress_result = compress_action.verify_returncode() - assert compress_result, compress_result.failure_message - logger.info("Verifying the compression of the 'json_multifile' dataset.") - compress_result = verify_compress_structured_clp_json( - compress_action, clp_package, json_multifile - ) - assert compress_result, compress_result.failure_message +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_basic_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a basic search on the `text_multifile` dataset. - search_args: SearchArgs = SearchArgs( - script_path=clp_package.path_config.search_path, - config=clp_package.temp_config_file_path, - query='mission: "STS-135"', - dataset=metadata.dataset_name, - begin_ts=metadata.begin_ts + 1, - end_ts=metadata.end_ts - 1, - ) + :param clp_package: + :param text_multifile: + """ + _compress_unstructured_dataset(clp_package, text_multifile) + _search_basic(clp_package, text_multifile, "*Saturn*") - logger.info( - "Searching the 'json_multifile' dataset over a time range with the 'clp-json' package." - ) - search_action = ClpAction.from_args(search_args) - search_result = search_action.verify_returncode() - assert search_result, search_result.failure_message - logger.info("Verifying the time-range search results for the 'json_multifile' dataset.") - search_result = verify_search_clp_json(search_action, clp_package, json_multifile) - assert search_result, search_result.failure_message +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_ignore_case_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a case-insensitive search on the `text_multifile` + dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_unstructured_dataset(clp_package, text_multifile) + _search_ignore_case(clp_package, text_multifile, "*sAtUrN*") @pytest.mark.search @pytest.mark.usefixtures("clear_package_archives") -def test_clp_json_search_text_multifile( +def test_clp_json_search_count_results_text_multifile( clp_package: ClpPackage, text_multifile: SampleDataset, ) -> None: """ - Validate that the `clp-json` package successfully searches the `text-multifile` dataset. + Validate that the `clp-json` package performs a count search on the `text_multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_unstructured_dataset(clp_package, text_multifile) + _search_count_results(clp_package, text_multifile, "*apollo-17*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_by_time_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count-by-time search on the `text_multifile` + dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_unstructured_dataset(clp_package, text_multifile) + _search_count_by_time(clp_package, text_multifile, "*apollo-17*", 10) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_time_range_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package honours `--begin-time`/`--end-time` when searching the + `text_multifile` dataset. The time range is a strict sub-range of the dataset's span + (`[begin_ts + 1, end_ts - 1]`), which is guaranteed to exclude the earliest and latest records. :param clp_package: :param text_multifile: """ + _compress_unstructured_dataset(clp_package, text_multifile) metadata: SampleDatasetMetadata = text_multifile.metadata - args: CompressArgs = CompressArgs( + begin_ts = metadata.begin_ts + 1 + end_ts = metadata.end_ts - 1 + _search_time_range(clp_package, text_multifile, "*", begin_ts, end_ts) + + +def _compress_structured_dataset(clp_package: ClpPackage, dataset: SampleDataset) -> None: + action = _run_compress(clp_package, dataset) + + logger.info("Verifying the compression of the '%s' dataset.", dataset.metadata.dataset_name) + result = verify_compress_structured_clp_json(action, clp_package, dataset) + assert result, result.failure_message + + +def _compress_unstructured_dataset(clp_package: ClpPackage, dataset: SampleDataset) -> None: + action = _run_compress(clp_package, dataset) + + logger.info("Verifying the compression of the '%s' dataset.", dataset.metadata.dataset_name) + result = verify_compress_unstructured_clp_json(action, clp_package, dataset) + assert result, result.failure_message + + +def _search_basic(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing BASIC search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query) + + +def _search_ignore_case(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing IGNORE-CASE search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, ignore_case=True) + + +def _search_count_results(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing COUNT search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, count=True) + + +def _search_count_by_time( + clp_package: ClpPackage, dataset: SampleDataset, query: str, time_interval: int +) -> None: + logger.info( + "Performing COUNT-BY-TIME search on the '%s' dataset.", dataset.metadata.dataset_name + ) + _run_search(clp_package, dataset, query, count_by_time=time_interval) + + +def _search_time_range( + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + begin_ts: int | None, + end_ts: int | None, +) -> None: + logger.info("Performing TIME-RANGE search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, begin_ts=begin_ts, end_ts=end_ts) + + +def _run_compress(clp_package: ClpPackage, dataset: SampleDataset) -> ClpAction: + """ + Compresses `dataset` with the `clp-json` package and verifies the action's exit code. This is + the mode-agnostic mechanism shared by the module's compression helpers; the caller is + responsible for verifying the compressed output. + + :param clp_package: + :param dataset: + :return: The completed compression action, ready to be passed to a verification helper. + """ + metadata: SampleDatasetMetadata = dataset.metadata + args = CompressArgs( script_path=clp_package.path_config.compress_path, config=clp_package.temp_config_file_path, dataset=metadata.dataset_name, timestamp_key=metadata.timestamp_key, unstructured=metadata.unstructured, - paths=[text_multifile.logs_path], + paths=[dataset.logs_path], ) - logger.info("Compressing the 'text_multifile' dataset with the 'clp-json' package.") + logger.info("Compressing the '%s' dataset with the 'clp-json' package.", metadata.dataset_name) action = ClpAction.from_args(args) result = action.verify_returncode() assert result, result.failure_message - logger.info("Verifying the compression of the 'text_multifile' dataset.") - result = verify_compress_unstructured_clp_json(action, clp_package, text_multifile) - assert result, result.failure_message + return action - search_args: SearchArgs = SearchArgs( + +def _run_search( # noqa: PLR0913 + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + *, + ignore_case: bool = False, + count: bool = False, + count_by_time: int | None = None, + begin_ts: int | None = None, + end_ts: int | None = None, +) -> None: + """ + Searches `dataset` with the `clp-json` package for `query` and verifies both the action's exit + code and its output. This is the mode-agnostic mechanism shared by the module's search helpers; + the caller selects the search variant via the keyword-only flags and logs the variant being + performed. + + :param clp_package: + :param dataset: + :param query: + :param ignore_case: + :param count: + :param count_by_time: + :param begin_ts: + :param end_ts: + """ + metadata: SampleDatasetMetadata = dataset.metadata + args = SearchArgs( script_path=clp_package.path_config.search_path, config=clp_package.temp_config_file_path, - query="*Saturn*", + query=query, dataset=metadata.dataset_name, + ignore_case=ignore_case, + count=count, + count_by_time=count_by_time, + begin_ts=begin_ts, + end_ts=end_ts, ) - logger.info("Searching the 'text_multifile' dataset with the 'clp-json' package.") - search_action = ClpAction.from_args(search_args) - search_result = search_action.verify_returncode() - assert search_result, search_result.failure_message + action = ClpAction.from_args(args) + result = action.verify_returncode() + assert result, result.failure_message - logger.info("Verifying the search results for the 'text_multifile' dataset.") - search_result = verify_search_clp_json(search_action, clp_package, text_multifile) - assert search_result, search_result.failure_message + logger.info("Verifying the search results on the '%s' dataset.", metadata.dataset_name) + verification = verify_search_clp_json(action, clp_package, dataset) + assert verification, verification.failure_message diff --git a/integration-tests/tests/package_tests/clp_json/verification/search.py b/integration-tests/tests/package_tests/clp_json/verification/search.py index 44c430695a..4291866a59 100644 --- a/integration-tests/tests/package_tests/clp_json/verification/search.py +++ b/integration-tests/tests/package_tests/clp_json/verification/search.py @@ -24,6 +24,7 @@ WILDCARD_MULTIMATCH_CHAR = "*" KV_DELIMITER_COLON = ":" MIN_QUOTED_LENGTH = 2 +UNSTRUCTURED_TIMESTAMP_KEY = "timestamp" def verify_search_clp_json( @@ -201,13 +202,13 @@ def _search_all_logs_for_kv_pair( # Filter the found entries if a time-range filter was used in the original query. if args.begin_ts is not None or args.end_ts is not None: - timestamp_key = metadata.timestamp_key timestamp_format = metadata.timestamp_format - if timestamp_key is None or timestamp_format is None: + if timestamp_format is None: pytest.fail( "clp-json time-range search verification requires the original dataset's metadata" - " to define a top-level 'timestamp_key' and a 'timestamp_format'." + " to define a 'timestamp_format'." ) + timestamp_key = _resolve_timestamp_key(metadata) found_entries = _filter_entries_by_time_range( found_entries, @@ -220,6 +221,28 @@ def _search_all_logs_for_kv_pair( return found_entries +def _resolve_timestamp_key(metadata: SampleDatasetMetadata) -> str: + """ + Resolves the top-level key under which each entry's timestamp is stored in the clp-json + structured representation. Unstructured logs are assigned the `UNSTRUCTURED_TIMESTAMP_KEY` key + when compressed with clp-json, whereas structured datasets carry the `timestamp_key` declared + in their metadata. + + :param metadata: + :return: The top-level timestamp key. + """ + if metadata.unstructured: + return UNSTRUCTURED_TIMESTAMP_KEY + + if metadata.timestamp_key is None: + pytest.fail( + "clp-json time-range search verification of a structured dataset requires the original" + " dataset's metadata to define a top-level 'timestamp_key'." + ) + + return metadata.timestamp_key + + def _convert_wildcard_to_regex(pattern: str, ignore_case: bool) -> re.Pattern[str]: """ Compiles a wildcard `pattern` into a regex in which each `WILDCARD_MULTIMATCH_CHAR` matches @@ -305,14 +328,15 @@ def _resolve_timestamp_to_ms( def _extract_count_from_search_output(search_output: str) -> str: """ - Extracts the count reported by a count-style search. + Extracts the total count reported by a count-style search. A count-by-time search reports a + separate count for each time bucket, so all per-bucket counts are summed. :param search_output: - :return: The reported count, followed by a newline. + :return: The total reported count, followed by a newline. """ - match = re.search(r"count: (\d+)", search_output) - if match: - return match.group(1) + "\n" + matches = re.findall(r"count: (\d+)", search_output) + if matches: + return str(sum(int(count) for count in matches)) + "\n" pytest.fail(f"The search result '{search_output}' wasn't in the correct format.") diff --git a/integration-tests/tests/package_tests/clp_text/verification/search.py b/integration-tests/tests/package_tests/clp_text/verification/search.py index bc837d6366..21a44230aa 100644 --- a/integration-tests/tests/package_tests/clp_text/verification/search.py +++ b/integration-tests/tests/package_tests/clp_text/verification/search.py @@ -167,12 +167,13 @@ def _parse_leading_timestamp_ms(line: str, timestamp_format: TimestampFormat) -> def _extract_count_from_search_output(search_output: str) -> str: """ - Extracts the count reported by a count-style search. + Extracts the total count reported by a count-style search. A count-by-time search reports a + separate count for each time bucket, so all per-bucket counts are summed. :param search_output: - :return: The reported count, followed by a newline. + :return: The total reported count, followed by a newline. """ - match = re.search(r"count: (\d+)", search_output) - if match: - return match.group(1) + "\n" + matches = re.findall(r"count: (\d+)", search_output) + if matches: + return str(sum(int(count) for count in matches)) + "\n" pytest.fail(f"The search result '{search_output}' wasn't in the correct format.") From bf83a998879d8e26721df9c0aea0c371eac3a6d9 Mon Sep 17 00:00:00 2001 From: Quinn Date: Sun, 5 Jul 2026 16:12:00 +0000 Subject: [PATCH 19/19] Add clp-text tests. --- .../package_tests/clp_json/test_clp_json.py | 66 ++++ .../package_tests/clp_text/test_clp_text.py | 344 ++++++++++++++++-- 2 files changed, 374 insertions(+), 36 deletions(-) diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index 71fa56ea00..1967e5c732 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -247,6 +247,72 @@ def test_clp_json_search_time_range_text_multifile( _search_time_range(clp_package, text_multifile, "*", begin_ts, end_ts) +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_basic_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a basic search on the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_unstructured_dataset(clp_package, text_singlefile) + _search_basic(clp_package, text_singlefile, "*TEST1*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_ignore_case_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a case-insensitive search on the + `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_unstructured_dataset(clp_package, text_singlefile) + _search_ignore_case(clp_package, text_singlefile, "*tEsT1*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_results_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count search on the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_unstructured_dataset(clp_package, text_singlefile) + _search_count_results(clp_package, text_singlefile, "*TEST*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_by_time_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count-by-time search on the `text_singlefile` + dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_unstructured_dataset(clp_package, text_singlefile) + _search_count_by_time(clp_package, text_singlefile, "*TEST*", 10) + + def _compress_structured_dataset(clp_package: ClpPackage, dataset: SampleDataset) -> None: action = _run_compress(clp_package, dataset) diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index 304a958e63..19d0a93d00 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -1,6 +1,7 @@ """Tests for the clp-text package.""" import logging +from pathlib import Path import pytest @@ -41,25 +42,93 @@ def test_clp_text_compression_text_multifile( text_multifile: SampleDataset, ) -> None: """ - Validate that the `clp-text` package successfully compresses the `text-multifile` dataset. + Validate that the `clp-text` package successfully compresses the `text_multifile` dataset. :param clp_package: :param text_multifile: """ - args = CompressArgs( - script_path=clp_package.path_config.compress_path, - config=clp_package.temp_config_file_path, - paths=[text_multifile.logs_path], - ) + _compress_dataset(clp_package, text_multifile) - logger.info("Compressing the 'text_multifile' dataset with the 'clp-text' package.") - action = ClpAction.from_args(args) - result = action.verify_returncode() - assert result, result.failure_message - logger.info("Verifying the compression of the 'text_multifile' dataset.") - result = verify_compress_clp_text(action, clp_package, text_multifile) - assert result, result.failure_message +@pytest.mark.compression +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_compression_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package successfully compresses the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_basic_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a basic search on the `text_multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + _search_basic(clp_package, text_multifile, "Saturn") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_ignore_case_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a case-insensitive search on the `text_multifile` + dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + _search_ignore_case(clp_package, text_multifile, "sAtUrN") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_count_results_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a count search on the `text_multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + _search_count_results(clp_package, text_multifile, "apollo-17") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_count_by_time_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a count-by-time search on the `text_multifile` + dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + _search_count_by_time(clp_package, text_multifile, "apollo-17", 10) @pytest.mark.search @@ -70,44 +139,247 @@ def test_clp_text_search_time_range_text_multifile( ) -> None: """ Validate that the `clp-text` package honours `--begin-time`/`--end-time` when searching the - `text-multifile` dataset. The time range is a strict sub-range of the dataset's span + `text_multifile` dataset. The time range is a strict sub-range of the dataset's span (`[begin_ts + 1, end_ts - 1]`), which is guaranteed to exclude the earliest and latest log events. :param clp_package: :param text_multifile: """ + _compress_dataset(clp_package, text_multifile) metadata: SampleDatasetMetadata = text_multifile.metadata - compress_args = CompressArgs( + begin_ts = metadata.begin_ts + 1 + end_ts = metadata.end_ts - 1 + _search_time_range(clp_package, text_multifile, "nominal", begin_ts, end_ts) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_file_path_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a search correctly with the `--file-path` flag on + the `text_multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + metadata: SampleDatasetMetadata = text_multifile.metadata + file_path = text_multifile.logs_path / metadata.file_names[0] + _search_file_path(clp_package, text_multifile, "nominal", file_path) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_basic_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a basic search on the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + _search_basic(clp_package, text_singlefile, "TEST1") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_ignore_case_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a case-insensitive search on the `text_singlefile` + dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + _search_ignore_case(clp_package, text_singlefile, "tEsT1") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_count_results_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a count search on the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + _search_count_results(clp_package, text_singlefile, "TEST") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_count_by_time_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a count-by-time search on the `text_singlefile` + dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + _search_count_by_time(clp_package, text_singlefile, "TEST", 10) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_file_path_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a search correctly with the `--file-path` flag on + the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + metadata: SampleDatasetMetadata = text_singlefile.metadata + file_path = text_singlefile.logs_path / metadata.file_names[0] + _search_file_path(clp_package, text_singlefile, "TEST5", file_path) + + +def _compress_dataset(clp_package: ClpPackage, dataset: SampleDataset) -> None: + action = _run_compress(clp_package, dataset) + + logger.info("Verifying the compression of the '%s' dataset.", dataset.metadata.dataset_name) + result = verify_compress_clp_text(action, clp_package, dataset) + assert result, result.failure_message + + +def _search_basic(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing BASIC search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query) + + +def _search_ignore_case(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing IGNORE-CASE search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, ignore_case=True) + + +def _search_count_results(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing COUNT search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, count=True) + + +def _search_count_by_time( + clp_package: ClpPackage, dataset: SampleDataset, query: str, time_interval: int +) -> None: + logger.info( + "Performing COUNT-BY-TIME search on the '%s' dataset.", dataset.metadata.dataset_name + ) + _run_search(clp_package, dataset, query, count_by_time=time_interval) + + +def _search_time_range( + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + begin_ts: int | None, + end_ts: int | None, +) -> None: + logger.info("Performing TIME-RANGE search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, begin_ts=begin_ts, end_ts=end_ts) + + +def _search_file_path( + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + file_path: Path, +) -> None: + logger.info("Performing FILE-PATH search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, file_path=file_path) + + +def _run_compress(clp_package: ClpPackage, dataset: SampleDataset) -> ClpAction: + """ + Compresses `dataset` with the `clp-text` package and verifies the action's exit code. This is + the mode-agnostic mechanism shared by the module's compression helpers; the caller is + responsible for verifying the compressed output. + + :param clp_package: + :param dataset: + :return: The completed compression action, ready to be passed to a verification helper. + """ + metadata: SampleDatasetMetadata = dataset.metadata + args = CompressArgs( script_path=clp_package.path_config.compress_path, config=clp_package.temp_config_file_path, - paths=[text_multifile.logs_path], + paths=[dataset.logs_path], ) - logger.info("Compressing the 'text_multifile' dataset with the 'clp-text' package.") - compress_action = ClpAction.from_args(compress_args) - compress_result = compress_action.verify_returncode() - assert compress_result, compress_result.failure_message + logger.info("Compressing the '%s' dataset with the 'clp-text' package.", metadata.dataset_name) + action = ClpAction.from_args(args) + result = action.verify_returncode() + assert result, result.failure_message - logger.info("Verifying the compression of the 'text_multifile' dataset.") - compress_result = verify_compress_clp_text(compress_action, clp_package, text_multifile) - assert compress_result, compress_result.failure_message + return action - search_args = SearchArgs( + +def _run_search( # noqa: PLR0913 + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + *, + file_path: Path | None = None, + ignore_case: bool = False, + count: bool = False, + count_by_time: int | None = None, + begin_ts: int | None = None, + end_ts: int | None = None, +) -> None: + """ + Searches `dataset` with the `clp-text` package for `query` and verifies both the action's exit + code and its output. This is the mode-agnostic mechanism shared by the module's search helpers; + the caller selects the search variant via the keyword-only flags and logs the variant being + performed. + + :param clp_package: + :param dataset: + :param query: + :param file_path: + :param ignore_case: + :param count: + :param count_by_time: + :param begin_ts: + :param end_ts: + """ + metadata: SampleDatasetMetadata = dataset.metadata + args = SearchArgs( script_path=clp_package.path_config.search_path, config=clp_package.temp_config_file_path, - query="nominal", - begin_ts=metadata.begin_ts + 1, - end_ts=metadata.end_ts - 1, + query=query, + file_path=file_path, + ignore_case=ignore_case, + count=count, + count_by_time=count_by_time, + begin_ts=begin_ts, + end_ts=end_ts, ) - logger.info( - "Searching the 'text_multifile' dataset over a time range with the 'clp-text' package." - ) - search_action = ClpAction.from_args(search_args) - search_result = search_action.verify_returncode() - assert search_result, search_result.failure_message + action = ClpAction.from_args(args) + result = action.verify_returncode() + assert result, result.failure_message - logger.info("Verifying the time-range search results for the 'text_multifile' dataset.") - search_result = verify_search_clp_text(search_action, text_multifile) - assert search_result, search_result.failure_message + logger.info("Verifying the search results on the '%s' dataset.", metadata.dataset_name) + verification = verify_search_clp_text(action, dataset) + assert verification, verification.failure_message