diff --git a/integration-tests/tests/data/README.md b/integration-tests/tests/data/README.md index f0720a71d7..fa9c1cd16b 100644 --- a/integration-tests/tests/data/README.md +++ b/integration-tests/tests/data/README.md @@ -28,6 +28,7 @@ following rules must be observed: "dataset_name": "str", "unstructured": bool, "timestamp_key": "str" | null, + "timestamp_format": | null, "begin_ts": int, "end_ts": int, "logs_subdir": "str", @@ -35,8 +36,7 @@ following rules must be observed: "str", "str", ... - ], - "single_match_wildcard_query": "str" + ] } ``` @@ -45,11 +45,43 @@ following rules must be observed: | `dataset_name` | The name of the sample dataset directory. | | `unstructured` | `True` if logs are unstructured, else `False`. | | `timestamp_key` | The authoritative timestamp key, or `null` if there is no such key. | + | `timestamp_format` | Description of authoritative timestamp encoding (see below). | | `begin_ts` | The earliest timestamp present in the dataset (ms). | | `end_ts` | The latest timestamp present in the dataset (ms). | | `logs_subdir` | The name of the subdirectory containing logs. | | `file_names` | A list of the files within `logs_subdir`. | - | `single_match_wildcard_query` | A wildcard query that matches exactly one log message in the dataset. | + + `timestamp_format` is a field tagged by by `kind` and containing `pattern` if applicable: + + ```json + {"kind": "epoch_ms"} + {"kind": "strptime", "pattern": "str"} + ``` + + | Field | Description | + | --- | --- | + | `kind` | `"epoch_ms"` for epoch ms timestamps, or `"strptime"` for formatted strings. | + | `pattern` | (`strptime` only) The Python [`strptime`][strptime] pattern for the timestamp. | + +3. For **structured** datasets: files in the dataset must be `.jsonl`, with only one log record per + line. + +## Time-range search verification + +Search verification reproduces the package's `--begin-time`/`--end-time` filter by independently +determining each log event's timestamp. For a dataset to be usable in a time-range search test, the +following restrictions must be observed: + +1. `begin_ts` must equal the earliest timestamp present in the dataset, and `end_ts` the latest. +2. `begin_ts` must be less than or equal to `end_ts`. +3. `timestamp_format` must not be `null`. +4. For **structured** datasets: + * `timestamp_key` must not be `null`, and must be a *top-level* key. Nested/dotted keys are + not supported. + * Each log in the dataset must contain `timestamp_key`. +5. For **unstructured** datasets: + * Every log line must begin with a single whitespace-delimited timestamp token written in the + format described by `timestamp_format.pattern`. ## Accessing sample datasets within the testing system @@ -74,3 +106,5 @@ To access a sample dataset from within the test system, the following rules shou Tests should use sample dataset fixtures instead of reading the logs directly, because many verification flows rely on dataset metadata. + +[strptime]: https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes diff --git a/integration-tests/tests/data/json_multifile/metadata.json b/integration-tests/tests/data/json_multifile/metadata.json index 39cbe4b33d..c6a12e5782 100644 --- a/integration-tests/tests/data/json_multifile/metadata.json +++ b/integration-tests/tests/data/json_multifile/metadata.json @@ -2,6 +2,9 @@ "dataset_name": "json_multifile", "unstructured": false, "timestamp_key": "timestamp", + "timestamp_format": { + "kind": "epoch_ms" + }, "begin_ts": 1310138944000, "end_ts": 1311208074120, "logs_subdir": "logs", @@ -11,6 +14,5 @@ "sts-135-2011-07-11.jsonl", "sts-135-2011-07-19.jsonl", "sts-135-2011-07-21.jsonl" - ], - "single_match_wildcard_query": "\"detail\":\"Roll program complete, heads down attitude achieved for ascent\"" + ] } diff --git a/integration-tests/tests/data/text_multifile/metadata.json b/integration-tests/tests/data/text_multifile/metadata.json index 0f075c969a..38e86ff427 100644 --- a/integration-tests/tests/data/text_multifile/metadata.json +++ b/integration-tests/tests/data/text_multifile/metadata.json @@ -2,6 +2,10 @@ "dataset_name": "text_multifile", "unstructured": true, "timestamp_key": null, + "timestamp_format": { + "kind": "strptime", + "pattern": "%Y-%m-%dT%H:%M:%S.%f" + }, "begin_ts": 92554380000, "end_ts": 93596007004, "logs_subdir": "logs", @@ -11,6 +15,5 @@ "apollo-17_day07.txt", "apollo-17_day10.txt", "apollo-17_day13.txt" - ], - "single_match_wildcard_query": "Saturn" + ] } diff --git a/integration-tests/tests/data/text_singlefile/metadata.json b/integration-tests/tests/data/text_singlefile/metadata.json index 1ef6c24929..cc159aede8 100644 --- a/integration-tests/tests/data/text_singlefile/metadata.json +++ b/integration-tests/tests/data/text_singlefile/metadata.json @@ -2,11 +2,11 @@ "dataset_name": "text_singlefile", "unstructured": true, "timestamp_key": null, + "timestamp_format": null, "begin_ts": 1427089710122, "end_ts": 1427089710122, "logs_subdir": "logs", "file_names": [ "simple.txt" - ], - "single_match_wildcard_query": "TEST1" + ] } diff --git a/integration-tests/tests/package_tests/clp_json/test_clp_json.py b/integration-tests/tests/package_tests/clp_json/test_clp_json.py index 75a7b87112..1967e5c732 100644 --- a/integration-tests/tests/package_tests/clp_json/test_clp_json.py +++ b/integration-tests/tests/package_tests/clp_json/test_clp_json.py @@ -10,9 +10,9 @@ verify_compress_structured_clp_json, verify_compress_unstructured_clp_json, ) -from tests.package_tests.utils.compress import ( - CompressArgs, -) +from tests.package_tests.clp_json.verification.search import verify_search_clp_json +from tests.package_tests.utils.compress import CompressArgs +from tests.package_tests.utils.search import SearchArgs from tests.utils.classes import ( ClpAction, SampleDataset, @@ -49,29 +49,12 @@ def test_clp_json_compression_json_multifile( json_multifile: SampleDataset, ) -> None: """ - Validate that the `clp-json` package successfully compresses the `json-multifile` dataset. + Validate that the `clp-json` package successfully compresses the `json_multifile` dataset. :param clp_package: :param json_multifile: """ - metadata: SampleDatasetMetadata = json_multifile.metadata - args: CompressArgs = CompressArgs( - script_path=clp_package.path_config.compress_path, - config=clp_package.temp_config_file_path, - dataset=metadata.dataset_name, - timestamp_key=metadata.timestamp_key, - unstructured=metadata.unstructured, - paths=[json_multifile.logs_path], - ) - - logger.info("Compressing the 'json_multifile' dataset with the 'clp-json' package.") - action = ClpAction.from_args(args) - result = action.verify_returncode() - assert result, result.failure_message - - logger.info("Verifying the compression of the 'json_multifile' dataset.") - result = verify_compress_structured_clp_json(action, clp_package, json_multifile) - assert result, result.failure_message + _compress_structured_dataset(clp_package, json_multifile) @pytest.mark.compression @@ -81,44 +64,377 @@ def test_clp_json_compression_text_multifile( text_multifile: SampleDataset, ) -> None: """ - Validate that the `clp-json` package successfully compresses the `text-multifile` dataset as + Validate that the `clp-json` package successfully compresses the `text_multifile` dataset as unstructured text. :param clp_package: :param text_multifile: """ + _compress_unstructured_dataset(clp_package, text_multifile) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_basic_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a basic search on the `json_multifile` dataset. + + :param clp_package: + :param json_multifile: + """ + _compress_structured_dataset(clp_package, json_multifile) + _search_basic(clp_package, json_multifile, 'detail: "*maximum dynamic*"') + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_ignore_case_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a case-insensitive search on the `json_multifile` + dataset. + + :param clp_package: + :param json_multifile: + """ + _compress_structured_dataset(clp_package, json_multifile) + _search_ignore_case(clp_package, json_multifile, 'detail: "*mAxImUm DyNaMiC*"') + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_results_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count search on the `json_multifile` dataset. + + :param clp_package: + :param json_multifile: + """ + _compress_structured_dataset(clp_package, json_multifile) + _search_count_results(clp_package, json_multifile, 'detail: "*maximum dynamic*"') + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_by_time_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count-by-time search on the `json_multifile` + dataset. + + :param clp_package: + :param json_multifile: + """ + _compress_structured_dataset(clp_package, json_multifile) + _search_count_by_time(clp_package, json_multifile, 'mission: "STS-135"', 10) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_time_range_json_multifile( + clp_package: ClpPackage, + json_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package honours `--begin-time`/`--end-time` when searching the + `json_multifile` dataset. The time range is a strict sub-range of the dataset's span + (`[begin_ts + 1, end_ts - 1]`), which is guaranteed to exclude the earliest and latest records. + + :param clp_package: + :param json_multifile: + """ + _compress_structured_dataset(clp_package, json_multifile) + metadata: SampleDatasetMetadata = json_multifile.metadata + begin_ts = metadata.begin_ts + 1 + end_ts = metadata.end_ts - 1 + _search_time_range(clp_package, json_multifile, 'mission: "STS-135"', begin_ts, end_ts) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_basic_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a basic search on the `text_multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_unstructured_dataset(clp_package, text_multifile) + _search_basic(clp_package, text_multifile, "*Saturn*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_ignore_case_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a case-insensitive search on the `text_multifile` + dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_unstructured_dataset(clp_package, text_multifile) + _search_ignore_case(clp_package, text_multifile, "*sAtUrN*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_results_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count search on the `text_multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_unstructured_dataset(clp_package, text_multifile) + _search_count_results(clp_package, text_multifile, "*apollo-17*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_by_time_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count-by-time search on the `text_multifile` + dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_unstructured_dataset(clp_package, text_multifile) + _search_count_by_time(clp_package, text_multifile, "*apollo-17*", 10) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_time_range_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package honours `--begin-time`/`--end-time` when searching the + `text_multifile` dataset. The time range is a strict sub-range of the dataset's span + (`[begin_ts + 1, end_ts - 1]`), which is guaranteed to exclude the earliest and latest records. + + :param clp_package: + :param text_multifile: + """ + _compress_unstructured_dataset(clp_package, text_multifile) metadata: SampleDatasetMetadata = text_multifile.metadata - args: CompressArgs = CompressArgs( + begin_ts = metadata.begin_ts + 1 + end_ts = metadata.end_ts - 1 + _search_time_range(clp_package, text_multifile, "*", begin_ts, end_ts) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_basic_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a basic search on the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_unstructured_dataset(clp_package, text_singlefile) + _search_basic(clp_package, text_singlefile, "*TEST1*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_ignore_case_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a case-insensitive search on the + `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_unstructured_dataset(clp_package, text_singlefile) + _search_ignore_case(clp_package, text_singlefile, "*tEsT1*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_results_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count search on the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_unstructured_dataset(clp_package, text_singlefile) + _search_count_results(clp_package, text_singlefile, "*TEST*") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_json_search_count_by_time_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-json` package performs a count-by-time search on the `text_singlefile` + dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_unstructured_dataset(clp_package, text_singlefile) + _search_count_by_time(clp_package, text_singlefile, "*TEST*", 10) + + +def _compress_structured_dataset(clp_package: ClpPackage, dataset: SampleDataset) -> None: + action = _run_compress(clp_package, dataset) + + logger.info("Verifying the compression of the '%s' dataset.", dataset.metadata.dataset_name) + result = verify_compress_structured_clp_json(action, clp_package, dataset) + assert result, result.failure_message + + +def _compress_unstructured_dataset(clp_package: ClpPackage, dataset: SampleDataset) -> None: + action = _run_compress(clp_package, dataset) + + logger.info("Verifying the compression of the '%s' dataset.", dataset.metadata.dataset_name) + result = verify_compress_unstructured_clp_json(action, clp_package, dataset) + assert result, result.failure_message + + +def _search_basic(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing BASIC search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query) + + +def _search_ignore_case(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing IGNORE-CASE search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, ignore_case=True) + + +def _search_count_results(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing COUNT search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, count=True) + + +def _search_count_by_time( + clp_package: ClpPackage, dataset: SampleDataset, query: str, time_interval: int +) -> None: + logger.info( + "Performing COUNT-BY-TIME search on the '%s' dataset.", dataset.metadata.dataset_name + ) + _run_search(clp_package, dataset, query, count_by_time=time_interval) + + +def _search_time_range( + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + begin_ts: int | None, + end_ts: int | None, +) -> None: + logger.info("Performing TIME-RANGE search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, begin_ts=begin_ts, end_ts=end_ts) + + +def _run_compress(clp_package: ClpPackage, dataset: SampleDataset) -> ClpAction: + """ + Compresses `dataset` with the `clp-json` package and verifies the action's exit code. This is + the mode-agnostic mechanism shared by the module's compression helpers; the caller is + responsible for verifying the compressed output. + + :param clp_package: + :param dataset: + :return: The completed compression action, ready to be passed to a verification helper. + """ + metadata: SampleDatasetMetadata = dataset.metadata + args = CompressArgs( script_path=clp_package.path_config.compress_path, config=clp_package.temp_config_file_path, dataset=metadata.dataset_name, timestamp_key=metadata.timestamp_key, unstructured=metadata.unstructured, - paths=[text_multifile.logs_path], + paths=[dataset.logs_path], ) - logger.info("Compressing the 'text_multifile' dataset with the 'clp-json' package.") + logger.info("Compressing the '%s' dataset with the 'clp-json' package.", metadata.dataset_name) action = ClpAction.from_args(args) result = action.verify_returncode() assert result, result.failure_message - logger.info("Verifying the compression of the 'text_multifile' dataset.") - result = verify_compress_unstructured_clp_json(action, clp_package, text_multifile) - assert result, result.failure_message + return action -@pytest.mark.search -@pytest.mark.usefixtures("clear_package_archives") -def test_clp_json_search(clp_package: ClpPackage) -> None: +def _run_search( # noqa: PLR0913 + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + *, + ignore_case: bool = False, + count: bool = False, + count_by_time: int | None = None, + begin_ts: int | None = None, + end_ts: int | None = None, +) -> None: """ - Validate that the `clp-json` package successfully searches some dataset. + Searches `dataset` with the `clp-json` package for `query` and verifies both the action's exit + code and its output. This is the mode-agnostic mechanism shared by the module's search helpers; + the caller selects the search variant via the keyword-only flags and logs the variant being + performed. :param clp_package: + :param dataset: + :param query: + :param ignore_case: + :param count: + :param count_by_time: + :param begin_ts: + :param end_ts: """ - # TODO: compress a dataset - - # TODO: check the correctness of the compression + metadata: SampleDatasetMetadata = dataset.metadata + args = SearchArgs( + script_path=clp_package.path_config.search_path, + config=clp_package.temp_config_file_path, + query=query, + dataset=metadata.dataset_name, + ignore_case=ignore_case, + count=count, + count_by_time=count_by_time, + begin_ts=begin_ts, + end_ts=end_ts, + ) - # TODO: search through that dataset and check the correctness of the search results. + action = ClpAction.from_args(args) + result = action.verify_returncode() + assert result, result.failure_message - assert clp_package + logger.info("Verifying the search results on the '%s' dataset.", metadata.dataset_name) + verification = verify_search_clp_json(action, clp_package, dataset) + assert verification, verification.failure_message diff --git a/integration-tests/tests/package_tests/clp_json/verification/search.py b/integration-tests/tests/package_tests/clp_json/verification/search.py new file mode 100644 index 0000000000..4291866a59 --- /dev/null +++ b/integration-tests/tests/package_tests/clp_json/verification/search.py @@ -0,0 +1,350 @@ +"""Search verification helpers specific to the clp-json package.""" + +import json +import logging +import re +from typing import Any + +import pytest + +from tests.package_tests.classes import ClpPackage +from tests.package_tests.utils.search import parse_timestamp_to_epoch_ms, SearchArgs +from tests.utils.classes import ( + ClpAction, + ClpVerificationResult, + EpochMsTimestampFormat, + SampleDataset, + SampleDatasetMetadata, + TimestampFormat, +) + +logger = logging.getLogger(__name__) + + +WILDCARD_MULTIMATCH_CHAR = "*" +KV_DELIMITER_COLON = ":" +MIN_QUOTED_LENGTH = 2 +UNSTRUCTURED_TIMESTAMP_KEY = "timestamp" + + +def verify_search_clp_json( + action: ClpAction, + clp_package: ClpPackage, + dataset: SampleDataset, +) -> ClpVerificationResult: + """ + Verifies that the search action performed on the `clp-json` package returns the expected + results. Verification is performed as follows: + + 1. The query string from the search action is parsed into a key-value pair. + 2. The dataset's log entries are loaded into a list of dicts for processing. + 3. The list of dicts is searched for entries that match or contain the key-value pair. + 4. The found entries are compared against the original search action's output. For a count-style + search (`--count`), the number of found entries is compared against the reported count; + otherwise the set of found entries is compared against the set of output logs. + + :param action: + :param clp_package: + :param dataset: + :return: A `ClpVerificationResult` indicating the success or failure of the verification. + """ + args = action.args + if not isinstance(args, SearchArgs): + err_msg = "Verification expects a 'SearchArgs' action." + raise TypeError(err_msg) + + kv_pair = _construct_kv_pair_from_query(args.query) + + metadata: SampleDatasetMetadata = dataset.metadata + if metadata.unstructured: + # Use search.sh to get the structured version of the unstructured data. + supporting_search_args: SearchArgs = SearchArgs( + script_path=clp_package.path_config.search_path, + config=clp_package.temp_config_file_path, + query="*", + dataset=metadata.dataset_name, + ) + supporting_search_action = ClpAction.from_args(supporting_search_args) + search_result = supporting_search_action.verify_returncode() + if not search_result: + return action.fail_verification( + "During search action verification, supporting call to 'search.sh' returned a" + " non-zero exit code.", + supporting_action=supporting_search_action, + ) + + supporting_output = supporting_search_action.completed_proc.stdout + all_logs = [json.loads(line) for line in supporting_output.splitlines() if line.strip()] + else: + all_logs = _load_all_jsonl_logs_from_dataset(dataset) + + found_entries: list[dict[str, Any]] = _search_all_logs_for_kv_pair( + all_logs, kv_pair, args, metadata + ) + + search_output = action.completed_proc.stdout + if args.is_count_query: + expected_count = str(len(found_entries)) + "\n" + actual_count = _extract_count_from_search_output(search_output) + + if expected_count == actual_count: + return action.pass_verification() + + return action.fail_verification( + "Search verification failure: mismatch between search count and expected count.\n" + f"Expected count: '{expected_count}'\nActual count: '{actual_count}'", + ) + + search_output_list = [json.loads(line) for line in search_output.splitlines() if line.strip()] + if _normalize_entries(found_entries) == _normalize_entries(search_output_list): + return action.pass_verification() + + return action.fail_verification( + "Search verification failure: mismatch between search output and expected output.\n" + f"Expected output: '{found_entries}'\nActual output: '{search_output_list}'", + ) + + +def _construct_kv_pair_from_query(query: str) -> tuple[str, Any]: + """ + Constructs a key-value pair from a query string. The query is split on the first + `KV_DELIMITER_COLON` that is not inside a quoted group (`"..."` or `'...'`). + + :param query: + :return: `(KEY, VALUE)` if `KV_DELIMITER_COLON` is present, else + `(WILDCARD_MULTIMATCH_CHAR, VALUE)`. + """ + colon_index = _find_unquoted_colon(query) + if colon_index is None: + return WILDCARD_MULTIMATCH_CHAR, _strip_surrounding_quotes(query.strip()) + key = _strip_surrounding_quotes(query[:colon_index].strip()) + value = _strip_surrounding_quotes(query[colon_index + 1 :].strip()) + return key, value + + +def _find_unquoted_colon(query: str) -> int | None: + """ + Finds the first colon in `query` that is not inside a quoted group (`"..."` or `'...'`). + + :param query: + :return: The index of the first unquoted colon, or `None` if there is none. + """ + open_quote: str | None = None + for index, char in enumerate(query): + if open_quote is not None: + if char == open_quote: + open_quote = None + elif char in ('"', "'"): + open_quote = char + elif char == KV_DELIMITER_COLON: + return index + return None + + +def _strip_surrounding_quotes(text: str) -> str: + """ + Removes a single pair of matching surrounding quotes (`"` or `'`) from `text`, if present. + + :param text: + :return: The processed text string. + """ + if len(text) >= MIN_QUOTED_LENGTH and text[0] == text[-1] and text[0] in ('"', "'"): + return text[1:-1] + return text + + +def _load_all_jsonl_logs_from_dataset(dataset: SampleDataset) -> list[dict[str, Any]]: + """ + Loads every JSONL log from a dataset into a single list of dicts. + + :param dataset: + :return: A list of dicts, each of which represents a JSONL record. + """ + records: list[dict[str, Any]] = [] + for path in dataset.metadata.file_names: + absolute_path = dataset.logs_path / path + with absolute_path.open("r", encoding="utf-8") as f: + for line in f: + stripped = line.strip() + if not stripped: + continue + records.append(json.loads(stripped)) + return records + + +def _search_all_logs_for_kv_pair( + all_logs: list[dict[str, Any]], + kv_pair: tuple[str, Any], + args: SearchArgs, + metadata: SampleDatasetMetadata, +) -> list[dict[str, Any]]: + """ + Searches `all_logs` for entries that contain `kv_pair`. The search respects the various + constraints posed by the content of `kv_pair`, `args` and `metadata`. + + :param all_logs: + :param kv_pair: + :param args: + :param metadata: + :return: A list of dicts that contain the `kv_pair` when searched w.r.t. the constraints. + """ + key, value = kv_pair + key_regex = _convert_wildcard_to_regex(str(key), ignore_case=False) + value_regex = _convert_wildcard_to_regex(str(value), ignore_case=args.ignore_case) + found_entries = [ + entry + for entry in all_logs + if any( + key_regex.fullmatch(str(entry_key)) and value_regex.fullmatch(str(entry_value)) + for entry_key, entry_value in entry.items() + ) + ] + + # Filter the found entries if a time-range filter was used in the original query. + if args.begin_ts is not None or args.end_ts is not None: + timestamp_format = metadata.timestamp_format + if timestamp_format is None: + pytest.fail( + "clp-json time-range search verification requires the original dataset's metadata" + " to define a 'timestamp_format'." + ) + timestamp_key = _resolve_timestamp_key(metadata) + + found_entries = _filter_entries_by_time_range( + found_entries, + timestamp_key, + timestamp_format, + args.begin_ts, + args.end_ts, + ) + + return found_entries + + +def _resolve_timestamp_key(metadata: SampleDatasetMetadata) -> str: + """ + Resolves the top-level key under which each entry's timestamp is stored in the clp-json + structured representation. Unstructured logs are assigned the `UNSTRUCTURED_TIMESTAMP_KEY` key + when compressed with clp-json, whereas structured datasets carry the `timestamp_key` declared + in their metadata. + + :param metadata: + :return: The top-level timestamp key. + """ + if metadata.unstructured: + return UNSTRUCTURED_TIMESTAMP_KEY + + if metadata.timestamp_key is None: + pytest.fail( + "clp-json time-range search verification of a structured dataset requires the original" + " dataset's metadata to define a top-level 'timestamp_key'." + ) + + return metadata.timestamp_key + + +def _convert_wildcard_to_regex(pattern: str, ignore_case: bool) -> re.Pattern[str]: + """ + Compiles a wildcard `pattern` into a regex in which each `WILDCARD_MULTIMATCH_CHAR` matches + zero or more characters and all other characters are matched literally. When `ignore_case` is + set, the compiled regex matches case-insensitively. + + :param pattern: + :param ignore_case: + :return: A compiled regex pattern. + """ + regex = ".*".join(re.escape(segment) for segment in pattern.split(WILDCARD_MULTIMATCH_CHAR)) + flags = re.DOTALL + if ignore_case: + flags |= re.IGNORECASE + return re.compile(regex, flags) + + +def _filter_entries_by_time_range( + entries: list[dict[str, Any]], + timestamp_key: str, + timestamp_format: TimestampFormat, + begin_ts: int | None, + end_ts: int | None, +) -> list[dict[str, Any]]: + """ + Filters `entries` down to those whose timestamp falls within the inclusive `[begin_ts, end_ts]` + range. Each entry's timestamp is read from the top-level `timestamp_key` and interpreted + according to `timestamp_format`. + + :param entries: + :param timestamp_key: + :param timestamp_format: + :param begin_ts: + :param end_ts: + :return: The entries that fall within the time range. + """ + filtered_entries: list[dict[str, Any]] = [] + for entry in entries: + if timestamp_key not in entry: + pytest.fail(f"Log entry '{entry}' is missing the timestamp key '{timestamp_key}'.") + timestamp = _resolve_timestamp_to_ms(entry[timestamp_key], timestamp_key, timestamp_format) + if (begin_ts is None or begin_ts <= timestamp) and (end_ts is None or timestamp <= end_ts): + filtered_entries.append(entry) + return filtered_entries + + +def _resolve_timestamp_to_ms( + raw_timestamp: Any, + timestamp_key: str, + timestamp_format: TimestampFormat, +) -> int: + """ + Converts a log entry's raw timestamp value into UNIX epoch ms according to `timestamp_format`, + if not already given in UNIX epoch ms. + + :param raw_timestamp: + :param timestamp_key: + :param timestamp_format: + :return: The timestamp as an integer number of milliseconds since the UNIX epoch. + """ + if isinstance(timestamp_format, EpochMsTimestampFormat): + if not isinstance(raw_timestamp, int): + pytest.fail( + f"Log entry timestamp '{raw_timestamp}' under key '{timestamp_key}' should be an" + " integer, but it is not." + ) + return raw_timestamp + + if not isinstance(raw_timestamp, str): + pytest.fail( + f"Log entry timestamp '{raw_timestamp}' under key '{timestamp_key}' should be a string," + " but it is not." + ) + + try: + return parse_timestamp_to_epoch_ms(raw_timestamp, timestamp_format.pattern) + except ValueError: + pytest.fail( + f"Failed to parse log entry timestamp '{raw_timestamp}' under key '{timestamp_key}'" + f" using pattern '{timestamp_format.pattern}'." + ) + + +def _extract_count_from_search_output(search_output: str) -> str: + """ + Extracts the total count reported by a count-style search. A count-by-time search reports a + separate count for each time bucket, so all per-bucket counts are summed. + + :param search_output: + :return: The total reported count, followed by a newline. + """ + matches = re.findall(r"count: (\d+)", search_output) + if matches: + return str(sum(int(count) for count in matches)) + "\n" + pytest.fail(f"The search result '{search_output}' wasn't in the correct format.") + + +def _normalize_entries(entries: list[dict[str, Any]]) -> list[str]: + """ + Serializes each entry into a canonical JSON string and returns the sorted list. + + :param entries: + :return: A sorted list of canonical JSON strings. + """ + return sorted(json.dumps(entry, sort_keys=True) for entry in entries) diff --git a/integration-tests/tests/package_tests/clp_text/test_clp_text.py b/integration-tests/tests/package_tests/clp_text/test_clp_text.py index b802cbe59a..19d0a93d00 100644 --- a/integration-tests/tests/package_tests/clp_text/test_clp_text.py +++ b/integration-tests/tests/package_tests/clp_text/test_clp_text.py @@ -1,14 +1,17 @@ """Tests for the clp-text package.""" import logging +from pathlib import Path import pytest from tests.package_tests.classes import ClpPackage from tests.package_tests.clp_text.utils.mode import CLP_TEXT_MODE from tests.package_tests.clp_text.verification.compress import verify_compress_clp_text +from tests.package_tests.clp_text.verification.search import verify_search_clp_text from tests.package_tests.utils.compress import CompressArgs -from tests.utils.classes import ClpAction, SampleDataset +from tests.package_tests.utils.search import SearchArgs +from tests.utils.classes import ClpAction, SampleDataset, SampleDatasetMetadata logger = logging.getLogger(__name__) @@ -39,22 +42,344 @@ def test_clp_text_compression_text_multifile( text_multifile: SampleDataset, ) -> None: """ - Validate that the `clp-text` package successfully compresses the `text-multifile` dataset. + Validate that the `clp-text` package successfully compresses the `text_multifile` dataset. :param clp_package: :param text_multifile: """ + _compress_dataset(clp_package, text_multifile) + + +@pytest.mark.compression +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_compression_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package successfully compresses the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_basic_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a basic search on the `text_multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + _search_basic(clp_package, text_multifile, "Saturn") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_ignore_case_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a case-insensitive search on the `text_multifile` + dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + _search_ignore_case(clp_package, text_multifile, "sAtUrN") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_count_results_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a count search on the `text_multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + _search_count_results(clp_package, text_multifile, "apollo-17") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_count_by_time_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a count-by-time search on the `text_multifile` + dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + _search_count_by_time(clp_package, text_multifile, "apollo-17", 10) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_time_range_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package honours `--begin-time`/`--end-time` when searching the + `text_multifile` dataset. The time range is a strict sub-range of the dataset's span + (`[begin_ts + 1, end_ts - 1]`), which is guaranteed to exclude the earliest and latest log + events. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + metadata: SampleDatasetMetadata = text_multifile.metadata + begin_ts = metadata.begin_ts + 1 + end_ts = metadata.end_ts - 1 + _search_time_range(clp_package, text_multifile, "nominal", begin_ts, end_ts) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_file_path_text_multifile( + clp_package: ClpPackage, + text_multifile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a search correctly with the `--file-path` flag on + the `text_multifile` dataset. + + :param clp_package: + :param text_multifile: + """ + _compress_dataset(clp_package, text_multifile) + metadata: SampleDatasetMetadata = text_multifile.metadata + file_path = text_multifile.logs_path / metadata.file_names[0] + _search_file_path(clp_package, text_multifile, "nominal", file_path) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_basic_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a basic search on the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + _search_basic(clp_package, text_singlefile, "TEST1") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_ignore_case_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a case-insensitive search on the `text_singlefile` + dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + _search_ignore_case(clp_package, text_singlefile, "tEsT1") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_count_results_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a count search on the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + _search_count_results(clp_package, text_singlefile, "TEST") + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_count_by_time_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a count-by-time search on the `text_singlefile` + dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + _search_count_by_time(clp_package, text_singlefile, "TEST", 10) + + +@pytest.mark.search +@pytest.mark.usefixtures("clear_package_archives") +def test_clp_text_search_file_path_text_singlefile( + clp_package: ClpPackage, + text_singlefile: SampleDataset, +) -> None: + """ + Validate that the `clp-text` package performs a search correctly with the `--file-path` flag on + the `text_singlefile` dataset. + + :param clp_package: + :param text_singlefile: + """ + _compress_dataset(clp_package, text_singlefile) + metadata: SampleDatasetMetadata = text_singlefile.metadata + file_path = text_singlefile.logs_path / metadata.file_names[0] + _search_file_path(clp_package, text_singlefile, "TEST5", file_path) + + +def _compress_dataset(clp_package: ClpPackage, dataset: SampleDataset) -> None: + action = _run_compress(clp_package, dataset) + + logger.info("Verifying the compression of the '%s' dataset.", dataset.metadata.dataset_name) + result = verify_compress_clp_text(action, clp_package, dataset) + assert result, result.failure_message + + +def _search_basic(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing BASIC search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query) + + +def _search_ignore_case(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing IGNORE-CASE search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, ignore_case=True) + + +def _search_count_results(clp_package: ClpPackage, dataset: SampleDataset, query: str) -> None: + logger.info("Performing COUNT search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, count=True) + + +def _search_count_by_time( + clp_package: ClpPackage, dataset: SampleDataset, query: str, time_interval: int +) -> None: + logger.info( + "Performing COUNT-BY-TIME search on the '%s' dataset.", dataset.metadata.dataset_name + ) + _run_search(clp_package, dataset, query, count_by_time=time_interval) + + +def _search_time_range( + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + begin_ts: int | None, + end_ts: int | None, +) -> None: + logger.info("Performing TIME-RANGE search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, begin_ts=begin_ts, end_ts=end_ts) + + +def _search_file_path( + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + file_path: Path, +) -> None: + logger.info("Performing FILE-PATH search on the '%s' dataset.", dataset.metadata.dataset_name) + _run_search(clp_package, dataset, query, file_path=file_path) + + +def _run_compress(clp_package: ClpPackage, dataset: SampleDataset) -> ClpAction: + """ + Compresses `dataset` with the `clp-text` package and verifies the action's exit code. This is + the mode-agnostic mechanism shared by the module's compression helpers; the caller is + responsible for verifying the compressed output. + + :param clp_package: + :param dataset: + :return: The completed compression action, ready to be passed to a verification helper. + """ + metadata: SampleDatasetMetadata = dataset.metadata args = CompressArgs( script_path=clp_package.path_config.compress_path, config=clp_package.temp_config_file_path, - paths=[text_multifile.logs_path], + paths=[dataset.logs_path], ) - logger.info("Compressing the 'text_multifile' dataset with the 'clp-text' package.") + logger.info("Compressing the '%s' dataset with the 'clp-text' package.", metadata.dataset_name) action = ClpAction.from_args(args) result = action.verify_returncode() assert result, result.failure_message - logger.info("Verifying the compression of the 'text_multifile' dataset.") - result = verify_compress_clp_text(action, clp_package, text_multifile) + return action + + +def _run_search( # noqa: PLR0913 + clp_package: ClpPackage, + dataset: SampleDataset, + query: str, + *, + file_path: Path | None = None, + ignore_case: bool = False, + count: bool = False, + count_by_time: int | None = None, + begin_ts: int | None = None, + end_ts: int | None = None, +) -> None: + """ + Searches `dataset` with the `clp-text` package for `query` and verifies both the action's exit + code and its output. This is the mode-agnostic mechanism shared by the module's search helpers; + the caller selects the search variant via the keyword-only flags and logs the variant being + performed. + + :param clp_package: + :param dataset: + :param query: + :param file_path: + :param ignore_case: + :param count: + :param count_by_time: + :param begin_ts: + :param end_ts: + """ + metadata: SampleDatasetMetadata = dataset.metadata + args = SearchArgs( + script_path=clp_package.path_config.search_path, + config=clp_package.temp_config_file_path, + query=query, + file_path=file_path, + ignore_case=ignore_case, + count=count, + count_by_time=count_by_time, + begin_ts=begin_ts, + end_ts=end_ts, + ) + + action = ClpAction.from_args(args) + result = action.verify_returncode() assert result, result.failure_message + + logger.info("Verifying the search results on the '%s' dataset.", metadata.dataset_name) + verification = verify_search_clp_text(action, dataset) + assert verification, verification.failure_message diff --git a/integration-tests/tests/package_tests/clp_text/verification/search.py b/integration-tests/tests/package_tests/clp_text/verification/search.py new file mode 100644 index 0000000000..21a44230aa --- /dev/null +++ b/integration-tests/tests/package_tests/clp_text/verification/search.py @@ -0,0 +1,179 @@ +"""Search verification helpers specific to the clp-text package.""" + +import logging +import re + +import pytest + +from tests.package_tests.utils.search import parse_timestamp_to_epoch_ms, SearchArgs +from tests.utils.classes import ( + ClpAction, + ClpVerificationResult, + EpochMsTimestampFormat, + NonClpAction, + SampleDataset, + TimestampFormat, +) +from tests.utils.utils import get_binary_path + +logger = logging.getLogger(__name__) + + +def verify_search_clp_text( + action: ClpAction, + dataset: SampleDataset, +) -> ClpVerificationResult: + """ + Verifies that the search action performed on the `clp-text` package returns the expected + results. Verification is performed as follows: + + 1. A grep command written with respect to the original search arguments is run on the dataset. + 2. The filtered grep lines are compared against the original search action's output. For a + count-style search (`--count`), the number of matched lines is compared against the reported + count; otherwise the set of matched lines is compared against the set of output lines. + + :param action: + :param dataset: + :return: A `ClpVerificationResult` indicating the success or failure of the verification. + """ + args = action.args + if not isinstance(args, SearchArgs): + err_msg = "Verification expects a 'SearchArgs' action." + raise TypeError(err_msg) + + grep_action = NonClpAction(cmd=_construct_grep_verification_cmd(args, dataset)) + grep_action.check_returncode(dependent_action=action) + grep_lines = grep_action.completed_proc.stdout.splitlines() + + # Filter the found entries if a time-range filter was used in the original query. + if args.begin_ts is not None or args.end_ts is not None: + grep_lines = _filter_lines_by_time_range( + grep_lines, dataset.metadata.timestamp_format, args.begin_ts, args.end_ts + ) + + search_output = action.completed_proc.stdout + if args.is_count_query: + expected_count = str(len(grep_lines)) + "\n" + actual_count = _extract_count_from_search_output(search_output) + + if expected_count == actual_count: + return action.pass_verification() + + return action.fail_verification( + "Search verification failure: mismatch between search count and expected count.\n" + f"Expected count: '{expected_count}'\n" + f"Actual count: '{actual_count}'", + supporting_action=grep_action, + ) + + formatted_grep_result = "\n".join(sorted(grep_lines)) + formatted_search_result = "\n".join(sorted(search_output.splitlines())) + if formatted_grep_result == formatted_search_result: + return action.pass_verification() + + return action.fail_verification( + f"Search verification failure: mismatch between formatted search result and formatted grep" + f" result.\n" + f"Formatted search result: '{formatted_search_result}'\n" + f"Formatted grep result: '{formatted_grep_result}'", + supporting_action=grep_action, + ) + + +def _construct_grep_verification_cmd( + args: SearchArgs, + dataset: SampleDataset, +) -> list[str]: + """ + Constructs a grep command that reproduces the search described by `args` on `dataset`'s original + logs. + + :param args: + :param dataset: + :return: The grep command as a list of arguments. + """ + grep_cmd_options: list[str] = [ + "--recursive", + "--no-filename", + "--color=never", + ] + + if args.ignore_case: + grep_cmd_options.append("--ignore-case") + + path_for_grep = args.file_path or dataset.logs_path + return [ + get_binary_path("grep"), + *grep_cmd_options, + args.query, + str(path_for_grep), + ] + + +def _filter_lines_by_time_range( + lines: list[str], + timestamp_format: TimestampFormat | None, + begin_ts: int | None, + end_ts: int | None, +) -> list[str]: + """ + Filters `lines` down to those whose leading timestamp falls within the inclusive + `[begin_ts, end_ts]` range. Each line must begin with a whitespace-delimited timestamp token + interpreted according to `timestamp_format`. + + :param lines: + :param timestamp_format: + :param begin_ts: + :param end_ts: + :return: The lines that fall within the time range. + """ + if timestamp_format is None: + pytest.fail( + "clp-text time-range search verification requires the original dataset's metadata to" + " define a 'timestamp_format'." + ) + + filtered_entries: list[str] = [] + for line in lines: + if not line.strip(): + continue + timestamp = _parse_leading_timestamp_ms(line, timestamp_format) + if (begin_ts is None or begin_ts <= timestamp) and (end_ts is None or timestamp <= end_ts): + filtered_entries.append(line) + return filtered_entries + + +def _parse_leading_timestamp_ms(line: str, timestamp_format: TimestampFormat) -> int: + """ + Parses the leading whitespace-delimited timestamp token of `line` according to + `timestamp_format`. + + :param line: + :param timestamp_format: + :return: The parsed timestamp as an integer number of milliseconds since the UNIX epoch. + """ + token = line.split(maxsplit=1)[0] + if isinstance(timestamp_format, EpochMsTimestampFormat): + return int(token) + + try: + return parse_timestamp_to_epoch_ms(token, timestamp_format.pattern) + except ValueError: + pytest.fail( + f"Failed to parse timestamp token '{token}' from line '{line}' using pattern" + f" '{timestamp_format.pattern}'." + ) + + +def _extract_count_from_search_output(search_output: str) -> str: + """ + Extracts the total count reported by a count-style search. A count-by-time search reports a + separate count for each time bucket, so all per-bucket counts are summed. + + :param search_output: + :return: The total reported count, followed by a newline. + """ + matches = re.findall(r"count: (\d+)", search_output) + if matches: + return str(sum(int(count) for count in matches)) + "\n" + pytest.fail(f"The search result '{search_output}' wasn't in the correct format.") diff --git a/integration-tests/tests/package_tests/utils/search.py b/integration-tests/tests/package_tests/utils/search.py new file mode 100644 index 0000000000..43c6db6c54 --- /dev/null +++ b/integration-tests/tests/package_tests/utils/search.py @@ -0,0 +1,83 @@ +"""Classes to facilitate CLP package search testing.""" + +import logging +from datetime import datetime, timezone +from pathlib import Path + +from tests.utils.classes import CmdArgs + +logger = logging.getLogger(__name__) + + +DEFAULT_COUNT_BY_TIME_INTERVAL = 10 + + +def parse_timestamp_to_epoch_ms(timestamp: str, strptime_pattern: str) -> int: + """ + Parses `timestamp` using `strptime_pattern` into an integer number of milliseconds since the + UNIX epoch. A timestamp without an offset in the pattern is interpreted as UTC. + + :param timestamp: + :param strptime_pattern: + :raises ValueError: If `timestamp` doesn't match `strptime_pattern`. + :return: The parsed timestamp as an integer number of milliseconds since the UNIX epoch. + """ + parsed = datetime.strptime(timestamp, strptime_pattern) # noqa: DTZ007 + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return round(parsed.timestamp() * 1000) + + +class SearchArgs(CmdArgs): + """Command argument model for searching with the CLP package.""" + + script_path: Path + config: Path + query: str + raw: bool = True + dataset: str | None = None + file_path: Path | None = None + ignore_case: bool = False + count: bool = False + count_by_time: int | None = None + begin_ts: int | None = None + end_ts: int | None = None + + @property + def is_count_query(self) -> bool: + """:return: `True` if this object represents a count-style search; else `False`.""" + return self.count or self.count_by_time is not None + + def to_cmd(self) -> list[str]: + """Converts the model attributes to a command list.""" + cmd: list[str] = [ + str(self.script_path), + "--config", + str(self.config), + ] + + if self.dataset: + cmd.append("--dataset") + cmd.append(self.dataset) + if self.file_path: + cmd.append("--file-path") + cmd.append(str(self.file_path)) + if self.ignore_case: + cmd.append("--ignore-case") + if self.count: + cmd.append("--count") + if self.count_by_time is not None: + cmd.append("--count-by-time") + cmd.append(str(self.count_by_time)) + if self.begin_ts is not None: + cmd.append("--begin-time") + cmd.append(str(self.begin_ts)) + if self.end_ts is not None: + cmd.append("--end-time") + cmd.append(str(self.end_ts)) + if self.raw: + cmd.append("--raw") + + cmd.append(self.query) + + return cmd diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index 89c847945f..80249c5de9 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -8,6 +8,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path +from typing import Literal import pytest from pydantic import BaseModel @@ -76,6 +77,22 @@ def _static_paths(self) -> list[Path]: return [self.test_data_dir] +class EpochMsTimestampFormat(BaseModel): + """Indicates a timestamp encoded as an integer number of milliseconds since the UNIX epoch.""" + + kind: Literal["epoch_ms"] + + +class StrptimeTimestampFormat(BaseModel): + """Indicates a timestamp encoded as a string parseable by a `datetime.strptime` pattern.""" + + kind: Literal["strptime"] + pattern: str + + +TimestampFormat = EpochMsTimestampFormat | StrptimeTimestampFormat + + class SampleDatasetMetadata(BaseModel): """ Metadata for a sample dataset. All `/metadata.json` files must conform to this @@ -85,11 +102,11 @@ class SampleDatasetMetadata(BaseModel): dataset_name: str unstructured: bool timestamp_key: str | None + timestamp_format: TimestampFormat | None begin_ts: int end_ts: int logs_subdir: str file_names: list[str] - single_match_wildcard_query: str @dataclass