From 2ef114f4a9d4edecec8a8a7c239bf3b7a402ba3e Mon Sep 17 00:00:00 2001 From: guzmud Date: Wed, 8 Apr 2026 09:03:53 +0200 Subject: [PATCH 01/25] [template] feat(collector): copy-pasting the sentinelone probe as a starting point --- template/.dockerignore | 11 + template/.gitignore | 10 + template/CONTRIBUTING.md | 324 +++++++ template/Dockerfile | 32 + template/README.md | 335 ++++++++ template/docker-compose.yml | 11 + template/manifest-metadata.json | 18 + template/pyproject.toml | 127 +++ template/src/__init__.py | 3 + template/src/__main__.py | 33 + template/src/collector/__init__.py | 3 + template/src/collector/collector.py | 142 ++++ template/src/collector/exception.py | 73 ++ template/src/collector/expectation_handler.py | 204 +++++ template/src/collector/expectation_manager.py | 426 ++++++++++ .../collector/expectation_service_provider.py | 76 ++ template/src/collector/models.py | 168 ++++ template/src/collector/signature_registry.py | 159 ++++ template/src/collector/trace_manager.py | 201 +++++ .../src/collector/trace_service_provider.py | 24 + template/src/config.yml.sample | 10 + template/src/img/sentinelone-logo.png | Bin 0 -> 63856 bytes template/src/models/__init__.py | 3 + template/src/models/configs/__init__.py | 13 + template/src/models/configs/base_settings.py | 23 + .../src/models/configs/collector_configs.py | 65 ++ template/src/models/configs/config_loader.py | 156 ++++ .../src/models/configs/sentinelone_configs.py | 39 + template/src/py.typed | 0 template/src/services/client_api.py | 71 ++ template/src/services/converter.py | 117 +++ template/src/services/exception.py | 43 + template/src/services/expectation_service.py | 703 +++++++++++++++ .../src/services/fetcher_deep_visibility.py | 533 ++++++++++++ template/src/services/fetcher_threat.py | 141 ++++ .../src/services/fetcher_threat_events.py | 139 +++ template/src/services/model_threat.py | 132 +++ template/src/services/trace_service.py | 198 +++++ template/src/services/utils/__init__.py | 9 + template/src/services/utils/config_loader.py | 72 ++ .../src/services/utils/signature_extractor.py | 127 +++ template/src/services/utils/trace_builder.py | 74 ++ template/tests/__init__.py | 0 template/tests/conftest.py | 93 ++ template/tests/gwt_shared.py | 757 +++++++++++++++++ template/tests/services/__init__.py | 0 template/tests/services/conftest.py | 396 +++++++++ template/tests/services/fixtures/__init__.py | 0 template/tests/services/fixtures/factories.py | 324 +++++++ template/tests/services/test_client_api.py | 117 +++ template/tests/services/test_converter.py | 216 +++++ .../services/test_expectation_service.py | 797 ++++++++++++++++++ .../services/test_fetcher_deep_visibility.py | 431 ++++++++++ .../tests/services/test_fetcher_threat.py | 336 ++++++++ .../services/test_fetcher_threat_events.py | 212 +++++ template/tests/services/test_trace_service.py | 235 ++++++ template/tests/test_create_collector.py | 233 +++++ 57 files changed, 9195 insertions(+) create mode 100644 template/.dockerignore create mode 100644 template/.gitignore create mode 100644 template/CONTRIBUTING.md create mode 100644 template/Dockerfile create mode 100644 template/README.md create mode 100644 template/docker-compose.yml create mode 100644 template/manifest-metadata.json create mode 100644 template/pyproject.toml create mode 100644 template/src/__init__.py create mode 100644 template/src/__main__.py create mode 100644 template/src/collector/__init__.py create mode 100644 template/src/collector/collector.py create mode 100644 template/src/collector/exception.py create mode 100644 template/src/collector/expectation_handler.py create mode 100644 template/src/collector/expectation_manager.py create mode 100644 template/src/collector/expectation_service_provider.py create mode 100644 template/src/collector/models.py create mode 100644 template/src/collector/signature_registry.py create mode 100644 template/src/collector/trace_manager.py create mode 100644 template/src/collector/trace_service_provider.py create mode 100644 template/src/config.yml.sample create mode 100644 template/src/img/sentinelone-logo.png create mode 100644 template/src/models/__init__.py create mode 100644 template/src/models/configs/__init__.py create mode 100644 template/src/models/configs/base_settings.py create mode 100644 template/src/models/configs/collector_configs.py create mode 100644 template/src/models/configs/config_loader.py create mode 100644 template/src/models/configs/sentinelone_configs.py create mode 100644 template/src/py.typed create mode 100644 template/src/services/client_api.py create mode 100644 template/src/services/converter.py create mode 100644 template/src/services/exception.py create mode 100644 template/src/services/expectation_service.py create mode 100644 template/src/services/fetcher_deep_visibility.py create mode 100644 template/src/services/fetcher_threat.py create mode 100644 template/src/services/fetcher_threat_events.py create mode 100644 template/src/services/model_threat.py create mode 100644 template/src/services/trace_service.py create mode 100644 template/src/services/utils/__init__.py create mode 100644 template/src/services/utils/config_loader.py create mode 100644 template/src/services/utils/signature_extractor.py create mode 100644 template/src/services/utils/trace_builder.py create mode 100644 template/tests/__init__.py create mode 100644 template/tests/conftest.py create mode 100644 template/tests/gwt_shared.py create mode 100644 template/tests/services/__init__.py create mode 100644 template/tests/services/conftest.py create mode 100644 template/tests/services/fixtures/__init__.py create mode 100644 template/tests/services/fixtures/factories.py create mode 100644 template/tests/services/test_client_api.py create mode 100644 template/tests/services/test_converter.py create mode 100644 template/tests/services/test_expectation_service.py create mode 100644 template/tests/services/test_fetcher_deep_visibility.py create mode 100644 template/tests/services/test_fetcher_threat.py create mode 100644 template/tests/services/test_fetcher_threat_events.py create mode 100644 template/tests/services/test_trace_service.py create mode 100644 template/tests/test_create_collector.py diff --git a/template/.dockerignore b/template/.dockerignore new file mode 100644 index 00000000..20a32394 --- /dev/null +++ b/template/.dockerignore @@ -0,0 +1,11 @@ +# Configuration files +config.yml + +# Build artifacts +dist + +# Cache directories +__pycache__ +.ruff_cache +.mypy_cache +.pytest_cache diff --git a/template/.gitignore b/template/.gitignore new file mode 100644 index 00000000..a96f8e71 --- /dev/null +++ b/template/.gitignore @@ -0,0 +1,10 @@ +config.yml + +# Build artifacts +dist + +# Cache directories +__pycache__ +.ruff_cache +.mypy_cache +.pytest_cache diff --git a/template/CONTRIBUTING.md b/template/CONTRIBUTING.md new file mode 100644 index 00000000..7900626b --- /dev/null +++ b/template/CONTRIBUTING.md @@ -0,0 +1,324 @@ +# Contributing to SentinelOne Collector + +This document provides guidance for contributing to the SentinelOne collector for OpenAEV. This collector is now feature-complete with SentinelOne-specific implementation. + +## Current Implementation Status + +**COMPLETED**: The SentinelOne collector is fully implemented with the following components: + +### Core Components +- **Collector Core** ([`src/collector/collector.py`](src/collector/collector.py)) - Main daemon with SentinelOne service integration +- **Expectation Handler** ([`src/collector/expectation_handler.py`](src/collector/expectation_handler.py)) - Generic handler using service provider pattern +- **Expectation Manager** ([`src/collector/expectation_manager.py`](src/collector/expectation_manager.py)) - Batch processing and API interactions +- **Configuration System** ([`src/models/configs/`](src/models/configs/)) - Hierarchical configuration with SentinelOne settings +- **Service Providers** - Complete SentinelOne-specific implementation + +### SentinelOne Implementation +- **SentinelOne API Client** ([`src/services/client_api.py`](src/services/client_api.py)) - Full API integration +- **Deep Visibility Fetcher** ([`src/services/fetcher_deep_visibility.py`](src/services/fetcher_deep_visibility.py)) - Process event queries +- **Threat Fetcher** ([`src/services/fetcher_threat.py`](src/services/fetcher_threat.py)) - Prevention data correlation +- **Expectation Service** ([`src/services/expectation_service.py`](src/services/expectation_service.py)) - Business logic implementation +- **Trace Service** ([`src/services/trace_service.py`](src/services/trace_service.py)) - Trace creation with SentinelOne links +- **Data Converter** ([`src/services/converter.py`](src/services/converter.py)) - SentinelOne to OAEV format conversion + +### Supported Features +- **Signature Support**: `parent_process_name`, `start_date`, `end_date` +- **Detection Expectations**: Deep Visibility event validation +- **Prevention Expectations**: Combined event + threat validation +- **Retry Mechanism**: Configurable retries with ingestion delay handling +- **Trace Generation**: Links back to SentinelOne console +- **Error Handling**: Comprehensive exception handling and logging +- **Configuration Management**: YAML, environment variables, defaults + +## Installation and Setup + +### Poetry Dependency Groups + +- `--with dev`: Development tools (ruff, mypy, black, etc.) +- `--with test`: Testing tools (pytest, coverage, etc.) + +### Poetry Extras + +- `--extra prod`: Get pyoaev from PyPI (production releases) +- `--extra current`: Get pyoaev from Git release/current branch +- `--extra local`: Get pyoaev locally from `../../client-python` + +### Development Installation + +```bash +# Development setup with current pyoaev version +poetry install -E current --with dev,test + +# Production setup +poetry install -E prod + +# Local development with local pyoaev +poetry install -E local --with dev,test +``` + +### Running the Collector + +```bash +# Direct execution +SentinelOneCollector + +# Using Python module execution +python -m src + +# Using Poetry to run +poetry run python -m src +``` + +## Development Workflow + +### Setting Up Development Environment + +1. **Clone and Install**: + ```bash + git clone + cd sentinelone + poetry install -E current --with dev,test + ``` + +2. **Configure for Development**: + ```bash + # Copy sample config + cp src/config.yml.sample src/config.yml + + # Edit with your SentinelOne details + vim src/config.yml + ``` + +3. **Run Development Tools**: + ```bash + # Format code + poetry run black src/ + + # Lint code + poetry run ruff check src/ + + # Type checking + poetry run mypy src/ + + # Run tests + poetry run pytest + ``` + +### Code Organization + +The codebase follows a clean architecture with clear separation of concerns: + +``` +src/ +├── collector/ # Generic collector framework +│ ├── collector.py # Main collector daemon +│ ├── expectation_handler.py +│ ├── expectation_manager.py +│ ├── trace_manager.py +│ └── models.py # Pydantic data models +├── services/ # SentinelOne-specific implementation +│ ├── client_api.py # API client +│ ├── expectation_service.py # Business logic +│ ├── trace_service.py # Trace creation +│ ├── converter.py # Data conversion +│ ├── fetcher_*.py # API-specific fetchers +│ └── model_*.py # Data models +└── models/ # Configuration management + └── configs/ # Hierarchical config system +``` + +## Testing + +### Test Structure + +```bash +# Run all tests +poetry run pytest + +# Run specific test files +poetry run pytest tests/test_expectation_service.py + +# Run with verbose output +poetry run pytest -v +``` + +### Test Categories + +- **Unit Tests**: Test individual components in isolation +- **Integration Tests**: Test SentinelOne API interactions +- **Configuration Tests**: Validate config loading and validation +- **Service Provider Tests**: Test expectation handling logic + +## Code Quality Standards + +### Formatting and Linting + +- **Black**: Code formatting (line length: 88) +- **Ruff**: Fast Python linter +- **MyPy**: Static type checking +- **Pre-commit**: Automated checks before commits + +### Code Style Guidelines + +- Use type hints throughout +- Follow Python PEP 8 conventions +- Write descriptive docstrings for public methods +- Implement comprehensive error handling +- Add meaningful logging with appropriate levels +- Use Pydantic models for data validation + +### Error Handling Patterns + +```python +# Use custom exceptions from src/collector/exception.py +from src.collector.exception import CollectorProcessingError + +try: + result = process_expectation(expectation) +except SentinelOneServiceError as e: + logger.error(f"SentinelOne API error: {e}") + raise CollectorProcessingError(f"Processing failed: {e}") from e +``` + +### Logging Best Practices + +```python +# Use consistent log prefixes +LOG_PREFIX = "[ComponentName]" + +# Include context in error logs +logger.error( + f"{LOG_PREFIX} Error processing expectation: {e} " + f"(Context: expectation_id={expectation_id}, retry_count={retries})" +) +``` + +## Contributing Guidelines + +### Making Changes + +1. **Create Feature Branch**: + ```bash + git checkout -b feature/your-feature-name + ``` + +2. **Make Changes**: + - Follow existing code patterns + - Add/update tests + - Update documentation + - Ensure type hints are complete + +3. **Test Changes**: + ```bash + poetry run pytest + poetry run mypy src/ + poetry run ruff check src/ + ``` + +4. **Commit and Push**: + ```bash + git add . + git commit -m "feat: description of your changes" + git push origin feature/your-feature-name + ``` + +### Pull Request Guidelines + +- Provide clear description of changes +- Update documentation as needed +- Ensure all CI checks pass +- Request review from maintainers + +### Extending the Collector + +#### Adding New Signature Types + +1. Update `SUPPORTED_SIGNATURES` in [`src/services/expectation_service.py`](src/services/expectation_service.py) +2. Modify query building in [`src/services/client_api.py`](src/services/client_api.py) +3. Update data conversion logic in [`src/services/converter.py`](src/services/converter.py) +4. Add corresponding tests + +#### Adding New API Endpoints + +1. Create fetcher class following pattern of existing fetchers +2. Update client API to use new fetcher +3. Add data models in `src/services/model_*.py` +4. Update service provider logic + +#### Configuration Changes + +1. Add fields to appropriate config models in `src/models/configs/` +2. Update config loader and validation +3. Update sample configuration files +4. Document new configuration options + +## Template Adaptation + +This collector is built on a reusable foundation that can be adapted for other security platforms. If you want to create a similar collector for another platform (e.g., CrowdStrike, Microsoft Defender): + +### SentinelOne-Specific References to Change + +#### Configuration Files +- [ ] `pyproject.toml` - Update project name and script names +- [ ] [`src/config.yml`](src/config.yml) - Update collector ID +- [ ] [`src/config.yml.sample`](src/config.yml.sample) - Update sample configuration + +#### Code References +- [ ] [`src/services/utils/config_loader.py`](src/services/utils/config_loader.py) - Rename config classes +- [ ] [`src/collector/collector.py`](src/collector/collector.py) - Update service imports +- [ ] [`src/models/configs/collector_configs.py`](src/models/configs/collector_configs.py) - Update defaults +- [ ] Platform-specific service implementations in `src/services/` + +### Reusable Components + +The following components are platform-agnostic and can be reused: +- Generic collector daemon +- Service provider protocols +- Configuration management system +- Expectation processing pipeline +- Signature registry system +- Trace management system + +## Common Issues and Solutions + +### Development Issues + +#### Import Errors +- Ensure Poetry environment is activated +- Check that all dependencies are installed with correct extras + +#### Configuration Loading +- Verify YAML structure matches Pydantic models +- Check environment variable naming conventions +- Validate required fields are present + +#### API Integration Testing +- Use mock objects for unit tests +- Set up test SentinelOne environment for integration tests +- Handle rate limits in test environments + +### Production Issues + +#### Performance Optimization +- Monitor API response times and adjust retry intervals +- Use batch processing for large expectation sets +- Optimize query time windows based on data volume + +#### Error Recovery +- Implement circuit breakers for persistent API failures +- Add health checks for service monitoring +- Use graceful degradation when possible + +## Documentation + +### Code Documentation +- Write clear docstrings for all public interfaces +- Include type hints and parameter descriptions +- Provide usage examples for complex functions + +### Configuration Documentation +- Document all configuration options +- Provide example configurations for different scenarios +- Include troubleshooting guides for common issues + +This collector provides a production-ready SentinelOne integration for OpenAEV with comprehensive error handling, configurable retry logic, and detailed trace generation. diff --git a/template/Dockerfile b/template/Dockerfile new file mode 100644 index 00000000..ae670616 --- /dev/null +++ b/template/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.13-alpine AS builder + +# poetry version available on Ubuntu 24.04 +RUN pip3 install poetry==2.1.3 + +RUN apk update && apk upgrade + +ARG installdir=/collector +ADD . ${installdir} +RUN cd ${installdir} && poetry build + +FROM python:3.13-alpine AS runner + +# Declare the build argument +ARG PYOAEV_GIT_BRANCH_OVERRIDE + +ARG installdir=/collector +COPY --from=builder ${installdir} ${installdir} +RUN cd ${installdir}/dist && pip3 install --no-cache-dir "$(ls *.whl)[prod]" + +RUN if [[ ${PYOAEV_GIT_BRANCH_OVERRIDE} ]] ; then \ + echo "Forcing specific version of client-python" && \ + apk add --no-cache git && \ + pip install pip3-autoremove && \ + pip-autoremove pyoaev -y && \ + pip install git+https://github.com/OpenAEV-Platform/client-python@${PYOAEV_GIT_BRANCH_OVERRIDE} ; \ + fi + +# necessary for icon location +WORKDIR ${installdir} + +CMD ["python3", "-m", "src"] diff --git a/template/README.md b/template/README.md new file mode 100644 index 00000000..d1742ebc --- /dev/null +++ b/template/README.md @@ -0,0 +1,335 @@ +# OpenAEV SentinelOne Collector + +A SentinelOne EDR integration for OpenAEV that validates security expectations by querying SentinelOne's Deep Visibility and Threats APIs. + +**Note**: Requires access to a SentinelOne Management Console with appropriate API permissions. + +**⚠️ Deep Visibility License Warning**: All static engine alerts rely on Deep Visibility, which is only compatible with Complete licenses. However, behavioral detection will be properly handled even with Core or Control licenses. The `enable_deep_visibility_search` configuration option (defaulted to False) allows you to enable this feature when you have the appropriate license. + +## Overview + +This collector validates OpenAEV expectations by querying your SentinelOne environment for threat data via the SentinelOne API. When OpenAEV runs security exercises, this collector automatically checks if the expected security threats were detected in your EDR by matching threat information and associated events, providing visibility into your detection capabilities. + +The collector uses SentinelOne's Threats API to fetch threat data and correlates it with threat events to validate expectations. + +## Features + +- **Threat-Based Validation**: Queries SentinelOne Threats API to validate security expectations against detected threats +- **Batch Processing**: Processes expectations in configurable batches for improved performance +- **Event Correlation**: Correlates threat data with threat events to extract process execution details +- **Trace Generation**: Creates detailed traces with links back to SentinelOne console +- **Flexible Configuration**: Support for YAML, environment variables, and multiple deployment scenarios + +## Requirements + +- OpenAEV Platform +- SentinelOne Management Console with API access +- Python 3.12+ (for manual deployment) +- SentinelOne API token with Threats and Threat Events permissions + +## Configuration + +## Configuration variables + +There are a number of configuration options, which are set either in `docker-compose.yml` (for Docker) or +in `config.yml` (for manual deployment). + +The collector supports multiple configuration sources in order of precedence: +1. `.env` file (if present in src directory) +2. YAML configuration file (`src/config.yml`, if present) +3. Environment variables (fallback) + +### OpenAEV environment variables + +Below are the parameters you'll need to set for OpenAEV: + +| Parameter | config.yml | Docker environment variable | Mandatory | Description | +|---------------|---------------|-----------------------------|-----------|------------------------------------------------------| +| OpenAEV URL | openaev.url | `OPENAEV_URL` | Yes | The URL of the OpenAEV platform. | +| OpenAEV Token | openaev.token | `OPENAEV_TOKEN` | Yes | The default admin token set in the OpenAEV platform.| + +### Base collector environment variables + +Below are the parameters you'll need to set for running the collector properly: + +| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | +|------------------|---------------------|-----------------------------|-------------------------|-----------|-----------------------------------------------------------------------------------------------| +| Collector ID | collector.id | `COLLECTOR_ID` | sentinelone--0b13e3f7-5c9e-46f5-acc4-33032e9b4921 | Yes | A unique `UUIDv4` identifier for this collector instance. | +| Collector Name | collector.name | `COLLECTOR_NAME` | SentinelOne | No | Name of the collector. | +| Collector Period | collector.period | `COLLECTOR_PERIOD` | PT2M | No | Collection interval (ISO 8601 format). | +| Log Level | collector.log_level | `COLLECTOR_LOG_LEVEL` | error | No | Determines the verbosity of the logs. Options are `debug`, `info`, `warn`, or `error`. | +| Platform | collector.platform | `COLLECTOR_PLATFORM` | EDR | No | Type of security platform this collector works for. One of: `EDR, XDR, SIEM, SOAR, NDR, ISPM` | +| Icon Filepath | collector.icon_filepath | `COLLECTOR_ICON_FILEPATH` | src/img/sentinelone-logo.png | No | Path to the icon file of the collector. | + +### Collector extra parameters environment variables + +Below are the parameters you'll need to set for the collector: + +| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | +|--------------------------|--------------------------------------|----------------------------------------|-----------------------------|-----------|----------------------------------------------------------------------------------------------------| +| Base URL | sentinelone.base_url | `SENTINELONE_BASE_URL` | https://api.sentinelone.com | No | SentinelOne Management Console URL | +| API Key | sentinelone.api_key | `SENTINELONE_API_KEY` | | Yes | SentinelOne API token with Threats and Threat Events permissions | +| Time Window | sentinelone.time_window | `SENTINELONE_TIME_WINDOW` | PT1H | No | Default search time window when no date signatures are provided (ISO 8601 format) | +| Expectation Batch Size | sentinelone.expectation_batch_size | `SENTINELONE_EXPECTATION_BATCH_SIZE` | 50 | No | Number of expectations to process in each batch for batch-based processing | +| Enable Deep Visibility | sentinelone.enable_deep_visibility_search | `SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH` | false | No | Enable Deep Visibility search for advanced threat detection (requires Complete license) | + +### Example Configuration Files + +#### YAML Configuration (`src/config.yml`) +```yaml +openaev: + url: "https://your-openaev-instance.com" + token: "your-openaev-token" + +collector: + id: "sentinelone--your-unique-uuid" + name: "SentinelOne Production" + period: "PT10M" + log_level: "info" + +sentinelone: + base_url: "https://your-sentinelone-console.sentinelone.net" + api_key: "your-sentinelone-api-token" + time_window: "PT1H" + expectation_batch_size: 50 + enable_deep_visibility_search: false +``` + +#### Environment Variables +```bash +export OPENAEV_URL="https://your-openaev-instance.com" +export OPENAEV_TOKEN="your-openaev-token" +export COLLECTOR_ID="sentinelone--your-unique-uuid" +export SENTINELONE_BASE_URL="https://your-sentinelone-console.sentinelone.net" +export SENTINELONE_API_KEY="your-sentinelone-api-token" +export SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH="false" +``` + +## Deployment + +### Manual Deployment with Poetry + +1. **Clone and Install Dependencies**: + ```bash + git clone + cd sentinelone + poetry install --extras local + ``` + +2. **Configure the Collector**: + - Copy `src/config.yml.sample` to `src/config.yml` + - Update configuration values or set environment variables + +3. **Run the Collector**: + ```bash + # Using Poetry + poetry run python -m src + + # Or direct execution after installation + SentinelOneCollector + ``` + +### Docker Deployment + +```bash +# Build the container +docker build -t openaev-sentinelone-collector . + +# Run with environment variables +docker run -d \ + -e OPENAEV_URL="https://your-openaev-instance.com" \ + -e OPENAEV_TOKEN="your-token" \ + -e COLLECTOR_ID="sentinelone--your-uuid" \ + -e SENTINELONE_BASE_URL="https://your-console.sentinelone.net" \ + -e SENTINELONE_API_KEY="your-api-key" \ + openaev-sentinelone-collector + +# Or run with configuration file mounted +docker run -d \ + -v /path/to/config.yml:/app/src/config.yml:ro \ + openaev-sentinelone-collector +``` + +## Behavior + +### Supported Signature Types + +The collector supports the following OpenAEV signature types: + +- **`parent_process_name`**: Process names to match against threat event data +- **`target_hostname_address`**: Target hostnames to filter threat queries +- **`end_date`**: End time for the threat search query (ISO 8601 format) + +### Processing Flow + +1. **Expectation Retrieval**: Fetches pending expectations from OpenAEV +2. **Batch Creation**: Groups expectations into configurable batches for processing +3. **Time Window Determination**: Extracts time windows from expectations or uses default configuration +4. **Threat Fetching**: Queries SentinelOne Threats API for the determined time window +5. **Event Correlation**: Fetches threat events for each identified threat +6. **Expectation Matching**: Matches threat data and events against expectation criteria using detection helper +7. **Result Reporting**: Updates expectation status in OpenAEV +8. **Trace Creation**: Creates detailed traces linking back to SentinelOne console + +### Threat Matching Logic + +The collector validates expectations by: + +1. **Threat Data Conversion**: Converts SentinelOne threat objects to OpenAEV-compatible format +2. **Process Name Extraction**: Extracts parent process names from threat events, focusing on `oaev-implant-*` prefixed processes +3. **Signature Matching**: Uses OpenAEV detection helper to match extracted data against expectation signatures +4. **Static vs Dynamic Threats**: Handles both static threat indicators and dynamic threats with associated events + +### Deep Visibility Integration + +The collector's behavior varies based on the Deep Visibility feature availability: + +- **Deep Visibility Enabled** (`enable_deep_visibility_search: true`): + - Requires SentinelOne Complete license + - Provides comprehensive threat detection including static engine alerts + - Supports full range of static threat indicators + +- **Deep Visibility Disabled** (`enable_deep_visibility_search: false`, default): + - Compatible with Core and Control licenses + - Focuses on behavioral threat detection + - Maintains full functionality for dynamic threats and behavioral analysis + +**Important**: When Deep Visibility is disabled, static engine-based expectations may not be fully validated, but all behavioral-based detections will continue to work normally. + +### Batch Processing + +The collector implements efficient batch processing to handle large volumes of expectations: + +1. **Configurable Batch Size**: Processes expectations in batches based on `expectation_batch_size` configuration +2. **Time Window Optimization**: Extracts and consolidates time windows across batch expectations +3. **Bulk Threat Fetching**: Fetches threats for the entire time window rather than individual queries +4. **Parallel Event Processing**: Efficiently correlates threat events across the batch + +## API Requirements + +### SentinelOne API Permissions + +Your SentinelOne API token requires the following permissions: + +- **Threats**: Read access to query threat information +- **Threat Events**: Read access to retrieve threat event details +- **Console Access**: General API access to the Management Console +- **Deep Visibility** (Optional): Required when `enable_deep_visibility_search` is enabled, needs Complete license + +### API Endpoints Used + +- `GET /web/api/v2.1/threats`: Query threat information using time-based filters +- `GET /web/api/v2.1/threat-events`: Retrieve detailed threat event information +- **Deep Visibility endpoints** (when enabled): + - `POST /web/api/v2.1/dv/init-query`: Initialize Deep Visibility query with SHA1 hashes and time range + - `GET /web/api/v2.1/dv/query-status`: Poll query status and progress until completion + - `GET /web/api/v2.1/dv/events`: Retrieve Deep Visibility events for completed query + +### Rate Limiting + +The collector respects SentinelOne's API rate limits by: +- Processing expectations in configurable batches +- Consolidating time windows to minimize API calls +- Adjusting query complexity based on Deep Visibility availability + +## Troubleshooting + +### Common Issues + +#### No Threats Found +- **Symptom**: Collector reports no matching threats despite expecting them +- **Causes**: + - Threat ingestion delay in SentinelOne + - Incorrect process names or hostnames in expectations + - Time window too narrow for threat detection +- **Solutions**: + - Verify process names match threat event data + - Extend `sentinelone.time_window` for broader searches + +#### API Authentication Errors +- **Symptom**: HTTP 401/403 errors in logs +- **Causes**: + - Invalid or expired API token + - Insufficient API permissions +- **Solutions**: + - Verify API token in SentinelOne console + +#### Connection Timeouts +- **Symptom**: HTTP timeout errors or connection failures +- **Causes**: + - Network connectivity issues + - SentinelOne console unavailability + - Incorrect base URL +- **Solutions**: + - Verify network connectivity to SentinelOne + - Check `sentinelone.base_url` configuration + - Review firewall and proxy settings + +#### Deep Visibility Issues +- **Symptom**: Static threats not being detected or matched +- **Causes**: + - Deep Visibility disabled in configuration + - Insufficient SentinelOne license (Core/Control instead of Complete) + - Deep Visibility feature not properly configured in SentinelOne +- **Solutions**: + - Set `sentinelone.enable_deep_visibility_search: true` if you have Complete license + - Verify your SentinelOne license includes Deep Visibility features + - Focus on behavioral expectations when using Core/Control licenses + - Check SentinelOne console for Deep Visibility configuration status + +### Logging + +The collector provides comprehensive logging at multiple levels: + +- **Error**: Critical failures and exceptions +- **Warn**: Recoverable issues and misconfigurations +- **Info**: Processing progress and results summary +- **Debug**: Detailed API interactions and data processing + +#### Log Configuration +```yaml +collector: + log_level: "debug" # For maximum verbosity during troubleshooting +``` + +#### Key Log Patterns +- `[SentinelOneClientAPI]`: API communication and responses +- `[SentinelOneExpectationService]`: Batch expectation processing logic +- `[SentinelOneThreatFetcher]`: Threat data fetching operations +- `[SentinelOneThreatEventsFetcher]`: Threat events fetching operations +- `[CollectorExpectationManager]`: High-level processing flow +- `[SentinelOneTraceService]`: Trace creation and submission + +### Performance Tuning + +#### For High-Volume Environments +- Reduce `collector.period` for more frequent processing +- Increase `sentinelone.expectation_batch_size` for better throughput +- Monitor API rate limits and ingestion patterns in your environment + +#### For Low-Latency Requirements +- Use shorter time windows in expectations for faster queries +- Reduce `collector.period` for more frequent collection cycles +- Monitor API rate limits and ingestion patterns accordingly + +## Architecture + +The collector uses a modular, service-provider architecture: + +- **Collector Core**: Main daemon handling scheduling and coordination +- **Expectation Service**: Batch processing and threat correlation logic +- **Threat Fetcher**: Dedicated service for fetching threat data +- **Threat Events Fetcher**: Service for retrieving threat event details +- **Client API**: SentinelOne API communication layer +- **Trace Service**: Trace creation and submission +- **Configuration System**: Hierarchical configuration management + +This architecture allows for easy extension and customization while maintaining clean separation of concerns. + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, coding standards, and contribution guidelines. + +## License + +This project is licensed under the terms specified in the main OpenAEV project. diff --git a/template/docker-compose.yml b/template/docker-compose.yml new file mode 100644 index 00000000..57477ea3 --- /dev/null +++ b/template/docker-compose.yml @@ -0,0 +1,11 @@ +version: "3" +services: + collector-sentinelone: + image: openaev/collector-sentinelone:rolling + environment: + - OPENAEV_URL=http://localhost + - OPENAEV_TOKEN=ChangeMe + - COLLECTOR_ID=ChangeMe + - SENTINELONE_BASE_URL=https://change.me + - SENTINELONE_API_KEY=ChangeMe + restart: always diff --git a/template/manifest-metadata.json b/template/manifest-metadata.json new file mode 100644 index 00000000..1cdfd0d6 --- /dev/null +++ b/template/manifest-metadata.json @@ -0,0 +1,18 @@ +{ + "title": "SentinelOne", + "slug": "openaev_sentinelone", + "description": "Collect responses from SentinelOne", + "short_description": "Collect responses from SentinelOne", + "use_cases": ["Security response"], + "verified": true, + "last_verified_date": "", + "playbook_supported": false, + "max_confidence_level": 80, + "support_version": "", + "subscription_link": "https://www.sentinelone.com/", + "source_code": "", + "manager_supported": true, + "container_version": "rolling", + "container_image": "openaev/collector-sentinelone", + "container_type": "COLLECTOR" +} \ No newline at end of file diff --git a/template/pyproject.toml b/template/pyproject.toml new file mode 100644 index 00000000..b42e197b --- /dev/null +++ b/template/pyproject.toml @@ -0,0 +1,127 @@ +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +packages = [{ include = "src" }, { include = "tests" }] + +[project] +name = "SentinelOneCollector" +version = "2.3.2" +description = "Collector for SentinelOne EDR." +readme = "README.md" + +requires-python = ">=3.11,<3.14" +dynamic = ["dependencies"] + +[tool.poetry.dependencies] +pyoaev = [ + {markers = "extra == 'prod' and extra != 'local' and extra != 'current'",version = "^2.0.2"}, + {markers = "extra == 'local' and extra != 'current' and extra != 'prod'",path = "../../client-python", develop = true}, +] +pydantic = "^2.11.7" +pydantic-settings = "^2.11.0" +requests = "^2.32.5" + +[tool.poetry.extras] +prod = ["pyoaev"] +local = ["pyoaev"] + +[tool.poetry.group.dev.dependencies] +isort = "^6.0.1" +ruff = "^0.12.11" +mypy = "^1.17.1" +black = "^25.1.0" +flake8 = "^7.3.0" +pip-audit = "^2.9.0" +pre-commit = "^4.3.0" + +[tool.poetry.group.test.dependencies] +pytest = "^8.4.1" +polyfactory = "^2.22.2" + +[project.scripts] +SentinelOneCollector = "src.__main__:main" + +[tool.pytest.ini_options] +testpaths = ["./tests"] + +[tool.ruff] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +target-version = "py312" + +[tool.ruff.lint] +# Never enforce `I001` (unsorted import). Already handle with isort +# Never enforce `E501` (line length violations). Already handle with black +# Never enforce `F821` (Undefined name `null`). incorrect issue with notebook +# Never enforce `D213` (Multi-line docstring summary should start at the second line) conflict with our docstring convention +# Never enforce `D211` (NoBlankLinesBeforeClass)` +# Never enforce `G004` (logging-f-string) Logging statement uses f-string +# Never enforce `TRY003`() Avoid specifying long messages outside the exception class not useful +# Never enforce `D104` (Missing docstring in public package) +# Never enforce `D407` (Missing dashed underline after section) +# Never enforce `D408` (Section underline should be in the line following the section’s name) +# Never enforce `D409` (Section underline should match the length of its name) +ignore = [ + "I001", + "D203", + "E501", + "F821", + "D205", + "D213", + "D211", + "G004", + "TRY003", + "D104", + "D407", + "D408", + "D409", +] +select = ["E", "F", "W", "D", "G", "T", "B", "C", "N", "I", "S"] + +[tool.mypy] +strict = true +exclude = [ + '^tests', + '^docs', + '^build', + '^dist', + '^venv', + '^site-packages', + '^__pypackages__', + '^.venv', +] +plugins = ["pydantic.mypy"] + + +[tool.cmw] +install-command = "poetry install --extras local" +config-dump-command = "poetry run python -m src --dump-config-schema" +icon-path = "src/img/sentinelone-logo.png" diff --git a/template/src/__init__.py b/template/src/__init__.py new file mode 100644 index 00000000..fab18bf4 --- /dev/null +++ b/template/src/__init__.py @@ -0,0 +1,3 @@ +from src.models import ConfigLoader + +__all__ = ["ConfigLoader"] diff --git a/template/src/__main__.py b/template/src/__main__.py new file mode 100644 index 00000000..f025e84b --- /dev/null +++ b/template/src/__main__.py @@ -0,0 +1,33 @@ +"""Main entry point for the collector.""" + +import logging +import os +import sys + +from src.collector import Collector +from src.collector.exception import CollectorConfigError + +LOG_PREFIX = "[Main]" + + +def main() -> None: + """Define the main function to run the collector.""" + logger = logging.getLogger(__name__) + + try: + logger.info(f"{LOG_PREFIX} Starting SentinelOne collector...") + collector = Collector() + collector.start() + except KeyboardInterrupt: + logger.info(f"{LOG_PREFIX} Collector stopped by user (Ctrl+C)") + os._exit(0) + except CollectorConfigError as e: + logger.error(f"{LOG_PREFIX} Configuration error: {e}") + sys.exit(2) + except Exception as e: + logger.exception(f"{LOG_PREFIX} Fatal error starting collector: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/template/src/collector/__init__.py b/template/src/collector/__init__.py new file mode 100644 index 00000000..36918ee8 --- /dev/null +++ b/template/src/collector/__init__.py @@ -0,0 +1,3 @@ +from src.collector.collector import Collector + +__all__ = ["Collector"] diff --git a/template/src/collector/collector.py b/template/src/collector/collector.py new file mode 100644 index 00000000..442b3683 --- /dev/null +++ b/template/src/collector/collector.py @@ -0,0 +1,142 @@ +"""Core collector.""" + +import os + +from pyoaev.daemons import CollectorDaemon # type: ignore[import-untyped] +from pyoaev.helpers import OpenAEVDetectionHelper # type: ignore[import-untyped] +from src.services.expectation_service import SentinelOneExpectationService +from src.services.trace_service import SentinelOneTraceService +from src.services.utils import SentinelOneConfig + +from .exception import ( + CollectorConfigError, + CollectorProcessingError, + CollectorSetupError, +) +from .expectation_handler import GenericExpectationHandler +from .expectation_manager import GenericExpectationManager + +LOG_PREFIX = "[Collector]" + + +class Collector(CollectorDaemon): # type: ignore[misc] + """Generic Collector using service provider pattern. + + This collector is use-case agnostic and works with any service provider. + """ + + def __init__(self) -> None: + """Initialize the collector. + + Raises: + CollectorConfigError: If collector initialization fails. + + """ + try: + self.config = SentinelOneConfig() + self.config_instance = self.config.load + + super().__init__( + configuration=self.config_instance.to_daemon_config(), + callback=self._process_callback, + collector_type="openaev_sentinelone", + ) + + self.logger.info( # type: ignore[has-type] + f"{LOG_PREFIX} SentinelOne Collector initialized successfully" + ) + + except Exception as err: + import logging + + logging.basicConfig(level=logging.ERROR) + self.logger = logging.getLogger(__name__) + raise CollectorConfigError( + f"Failed to initialize the collector: {err}" + ) from err + + def _setup(self) -> None: + """Set up the collector. + + Initializes SentinelOne services, expectation handler, expectation manager, + and OpenAEV detection helper. Sets up the collector for processing expectations. + + Raises: + CollectorSetupError: If collector setup fails. + + """ + try: + self.logger.info(f"{LOG_PREFIX} Starting collector setup...") + + super()._setup() + + self.logger.debug(f"{LOG_PREFIX} Initializing SentinelOne services...") + + self.sentinelone_service = SentinelOneExpectationService( + config=self.config_instance + ) + + self.trace_service = SentinelOneTraceService(self.config_instance) + + self.expectation_handler = GenericExpectationHandler( + self.sentinelone_service + ) + + self.expectation_manager = GenericExpectationManager( + oaev_api=self.api, + collector_id=self.get_id(), + expectation_handler=self.expectation_handler, + trace_service=self.trace_service, + ) + + supported_signatures = self.sentinelone_service.get_supported_signatures() + self.oaev_detection_helper = OpenAEVDetectionHelper( + logger=self.logger, + relevant_signatures_types=supported_signatures, + ) + + self.logger.info(f"{LOG_PREFIX} Collector setup completed successfully") + self.logger.info( + f"{LOG_PREFIX} Supported signatures: {[sig.value for sig in supported_signatures]}" + ) + + service_info = self.sentinelone_service.get_service_info() + self.logger.debug(f"{LOG_PREFIX} Service info: {service_info}") + + except Exception as err: + self.logger.error(f"{LOG_PREFIX} Collector setup failed: {err}") + raise CollectorSetupError(f"Failed to setup the collector: {err}") from err + + def _process_callback(self) -> None: + """Process the callback for expectation processing. + + Executes a single processing cycle, handling expectations through the + expectation manager and logging results. Handles keyboard interrupts + and system exits gracefully. + + Raises: + CollectorProcessingError: If processing cycle fails. + + """ + try: + self.logger.info(f"{LOG_PREFIX} Starting processing cycle...") + self.logger.debug( + f"{LOG_PREFIX} Processing expectations using SentinelOne services" + ) + + results = self.expectation_manager.process_expectations( + detection_helper=self.oaev_detection_helper + ) + + self.logger.info( + f"{LOG_PREFIX} Processing cycle completed: {results.processed} total, " + f"{results.valid} valid, {results.invalid} invalid, " + f"{results.skipped} skipped" + ) + + except (KeyboardInterrupt, SystemExit): + self.logger.info(f"{LOG_PREFIX} Collector stopping...") + os._exit(0) + except Exception as e: + self.logger.error(f"{LOG_PREFIX} Error during processing cycle: {str(e)}") + raise CollectorProcessingError(f"Processing error: {str(e)}") from e diff --git a/template/src/collector/exception.py b/template/src/collector/exception.py new file mode 100644 index 00000000..df900901 --- /dev/null +++ b/template/src/collector/exception.py @@ -0,0 +1,73 @@ +"""Custom exceptions for the collector.""" + + +class CollectorError(Exception): + """Base exception for the collector.""" + + pass + + +class CollectorConfigError(CollectorError): + """Exception raised when there is an error in the collector configuration.""" + + pass + + +class CollectorSetupError(CollectorError): + """Exception raised when there is an error setting up the collector.""" + + pass + + +class CollectorProcessingError(CollectorError): + """Exception raised when there is an error processing data in the collector.""" + + pass + + +class ExpectationHandlerError(CollectorError): + """Exception raised when there is an error in expectation handling.""" + + pass + + +class ExpectationProcessingError(CollectorError): + """Exception raised when there is an error processing expectations.""" + + pass + + +class ExpectationUpdateError(CollectorError): + """Exception raised when there is an error updating expectations.""" + + pass + + +class BulkUpdateError(ExpectationUpdateError): + """Exception raised when there is an error during bulk update operations.""" + + pass + + +class APIError(CollectorError): + """Exception raised when there is an error with API operations.""" + + pass + + +class TracingError(CollectorError): + """Exception raised when there is an error with tracing operations.""" + + pass + + +class TraceSubmissionError(TracingError): + """Exception raised when there is an error submitting traces.""" + + pass + + +class TraceCreationError(TracingError): + """Exception raised when there is an error creating traces.""" + + pass diff --git a/template/src/collector/expectation_handler.py b/template/src/collector/expectation_handler.py new file mode 100644 index 00000000..7e47aa19 --- /dev/null +++ b/template/src/collector/expectation_handler.py @@ -0,0 +1,204 @@ +"""Generic Expectation Handler.""" + +import logging +from typing import Any + +from pyoaev.apis.inject_expectation.model import ( # type: ignore[import-untyped] + DetectionExpectation, + PreventionExpectation, +) +from pyoaev.helpers import OpenAEVDetectionHelper # type: ignore[import-untyped] +from pyoaev.signatures.types import SignatureTypes # type: ignore[import-untyped] + +from .exception import ExpectationHandlerError +from .expectation_service_provider import ExpectationServiceProvider +from .models import ExpectationResult +from .signature_registry import ExpectationHandlerType, get_registry + +LOG_PREFIX = "[CollectorExpectationHandler]" + + +class GenericExpectationHandler: + """Generic expectation handler that delegates to service providers. + + This handler is completely agnostic to the specific use case and + delegates all processing logic to the injected service provider. + """ + + def __init__(self, service_provider: ExpectationServiceProvider) -> None: + """Initialize the generic handler. + + Args: + service_provider: Service provider implementing business logic. + + """ + self.logger = logging.getLogger(__name__) + self.service_provider = service_provider + + self.logger.debug(f"{LOG_PREFIX} Initializing generic expectation handler") + self._register_with_registry() + self.logger.info( + f"{LOG_PREFIX} Generic expectation handler initialized successfully" + ) + + def _register_with_registry(self) -> None: + """Register handler capabilities with the signature registry. + + Registers detection and prevention handlers with the signature registry + for all supported signature types from the service provider. + + Raises: + Exception: If registration with registry fails. + + """ + try: + registry = get_registry() + supported_signatures = self.service_provider.get_supported_signatures() + + registry.register_handler( + handler_type=ExpectationHandlerType.DETECTION, + handler_func=self.handle_expectation, + signature_types=supported_signatures, + ) + + registry.register_handler( + handler_type=ExpectationHandlerType.PREVENTION, + handler_func=self.handle_expectation, + signature_types=supported_signatures, + ) + + self.logger.info( + f"{LOG_PREFIX} Registered handler for {len(supported_signatures)} signature types: {[sig.value for sig in supported_signatures]}" + ) + + except Exception as e: + self.logger.error( + f"{LOG_PREFIX} Failed to register handler with registry: {e}" + ) + raise + + def handle_expectation( + self, + expectation: Any, + detection_helper: OpenAEVDetectionHelper, + ) -> ExpectationResult: + """Handle an expectation by delegating to the service provider. + + Args: + expectation: The expectation to process. + detection_helper: OpenAEV detection helper instance. + + Returns: + ExpectationResult containing processing results. + + Raises: + Exception: If expectation handling fails. + + """ + expectation_id = ( + str(expectation.inject_expectation_id) + if hasattr(expectation, "inject_expectation_id") + else "unknown" + ) + + try: + if isinstance(expectation, DetectionExpectation): + self.logger.debug( + f"{LOG_PREFIX} Processing detection expectation: {expectation_id}" + ) + result = self.service_provider.handle_detection_expectation( + expectation, detection_helper + ) + elif isinstance(expectation, PreventionExpectation): + self.logger.debug( + f"{LOG_PREFIX} Processing prevention expectation: {expectation_id}" + ) + result = self.service_provider.handle_prevention_expectation( + expectation, detection_helper + ) + else: + self.logger.warning( + f"{LOG_PREFIX} Unsupported expectation type for {expectation_id}: {type(expectation)}" + ) + result = ExpectationResult( + expectation_id=expectation_id, + is_valid=False, + expectation=expectation, + error_message="Unsupported expectation type", + ) + + return result + + except Exception as e: + self.logger.error( + f"{LOG_PREFIX} Error handling expectation {expectation_id}: {e}" + ) + raise + + def handle_batch_expectations( + self, + expectations: list[Any], + detection_helper: OpenAEVDetectionHelper, + ) -> tuple[list[ExpectationResult], int]: + """Handle a batch of expectations by delegating to the service provider. + + Post-processes results to ensure completeness by filling in missing + expectation IDs and expectation objects. + + Args: + expectations: List of expectations to process. + detection_helper: OpenAEV detection helper instance. + + Returns: + Tuple of (results, skipped_count) where: + - results: List of ExpectationResult objects for processed expectations + - skipped_count: Number of expectations skipped due to missing end_date + + Raises: + ExpectationHandlerError: If batch processing fails. + + """ + try: + self.logger.info( + f"{LOG_PREFIX} Starting batch processing of {len(expectations)} expectations" + ) + + results, skipped_count = self.service_provider.handle_batch_expectations( + expectations, detection_helper + ) + + # Post-process results to ensure completeness + self.logger.debug(f"{LOG_PREFIX} Post-processing batch results...") + for i, result in enumerate(results): + if result.expectation is None and i < len(expectations): + result.expectation = expectations[i] + if not result.expectation_id and result.expectation: + result.expectation_id = str( + result.expectation.inject_expectation_id + ) + + valid_count = sum(1 for r in results if r.is_valid) + invalid_count = len(results) - valid_count + + self.logger.info( + f"{LOG_PREFIX} Batch processing completed: {valid_count} valid, {invalid_count} invalid, {skipped_count} skipped" + ) + + return results, skipped_count + + except Exception as e: + self.logger.error(f"{LOG_PREFIX} Batch processing failed: {e}") + raise ExpectationHandlerError(f"Error in batch processing: {e}") from e + + def get_supported_signatures(self) -> list[SignatureTypes]: + """Get supported signature types from service provider. + + Returns: + List of SignatureTypes supported by the service provider. + + """ + signatures = self.service_provider.get_supported_signatures() + self.logger.debug( + f"{LOG_PREFIX} Supported signatures: {[sig.value for sig in signatures]}" + ) + return signatures diff --git a/template/src/collector/expectation_manager.py b/template/src/collector/expectation_manager.py new file mode 100644 index 00000000..3fa752cd --- /dev/null +++ b/template/src/collector/expectation_manager.py @@ -0,0 +1,426 @@ +"""Generic Expectation Manager.""" + +import logging +from typing import Any + +from pyoaev.apis.inject_expectation.model import ( # type: ignore[import-untyped] + DetectionExpectation, + PreventionExpectation, +) +from pyoaev.client import OpenAEV # type: ignore[import-untyped] +from pyoaev.helpers import OpenAEVDetectionHelper # type: ignore[import-untyped] + +from .exception import ( + APIError, + BulkUpdateError, + ExpectationProcessingError, + ExpectationUpdateError, +) +from .expectation_handler import GenericExpectationHandler +from .models import ExpectationResult, ProcessingSummary +from .trace_manager import TraceManager +from .trace_service_provider import TraceServiceProvider + +LOG_PREFIX = "[CollectorExpectationManager]" + + +class GenericExpectationManager: + """Generic expectation manager that works with any service provider. + + This manager is completely agnostic to the specific use case and + delegates all processing logic to the injected service providers. + """ + + def __init__( + self, + oaev_api: OpenAEV, + collector_id: str, + expectation_handler: GenericExpectationHandler, + trace_service: TraceServiceProvider | None = None, + ) -> None: + """Initialize generic expectation manager. + + Args: + oaev_api: OpenAEV API client. + collector_id: ID of the collector. + expectation_handler: Handler for processing expectations. + trace_service: Optional service for creating traces. + + Raises: + ValueError: If required parameters are None or empty. + + """ + if not oaev_api: + raise ValueError("oaev_api cannot be None") + if not collector_id: + raise ValueError("collector_id cannot be empty") + if not expectation_handler: + raise ValueError("expectation_handler cannot be None") + + self.logger = logging.getLogger(__name__) + self.oaev_api = oaev_api + self.collector_id = collector_id + self.expectation_handler = expectation_handler + self.trace_manager = TraceManager( + oaev_api=oaev_api, + collector_id=collector_id, + trace_service=trace_service, + ) + + self.logger.info( + f"{LOG_PREFIX} Expectation manager initialized for collector: {collector_id}" + ) + + def process_expectations( + self, detection_helper: OpenAEVDetectionHelper + ) -> ProcessingSummary: + """Process all expectations using the injected handler. + + Fetches expectations from OpenAEV, processes them through the handler, + updates expectations in OpenAEV, and creates traces. + + Args: + detection_helper: OpenAEV detection helper. + + Returns: + ProcessingSummary containing processing results. + + Raises: + ExpectationProcessingError: If processing fails. + + """ + try: + self.logger.info(f"{LOG_PREFIX} Starting expectation processing cycle") + + self.logger.debug(f"{LOG_PREFIX} Fetching expectations from OpenAEV...") + expectations = self._fetch_expectations() + + if not expectations: + self.logger.warning(f"{LOG_PREFIX} No expectations found to process") + return ProcessingSummary( + processed=0, + valid=0, + invalid=0, + skipped=0, + total_processing_time=None, + ) + + supported_expectations = [ + exp + for exp in expectations + if isinstance(exp, (DetectionExpectation, PreventionExpectation)) + ] + + total_expectations = len(expectations) + supported_count = len(supported_expectations) + skipped_count = total_expectations - supported_count + + self.logger.info( + f"{LOG_PREFIX} Found {total_expectations} total expectations: " + f"{supported_count} supported, {skipped_count} skipped" + ) + + if skipped_count > 0: + self.logger.debug( + f"{LOG_PREFIX} Skipped {skipped_count} unsupported expectation types" + ) + + self.logger.debug( + f"{LOG_PREFIX} Processing expectations through handler..." + ) + results, service_skipped_count = ( + self.expectation_handler.handle_batch_expectations( + supported_expectations, detection_helper + ) + ) + + self.logger.debug(f"{LOG_PREFIX} Updating expectations in OpenAEV...") + self._bulk_update_expectations(results) + + self.logger.debug(f"{LOG_PREFIX} Creating and submitting traces...") + self.trace_manager.create_and_submit_traces(results) + + valid_count = sum(1 for r in results if r.is_valid) + invalid_count = len(results) - valid_count + + total_skipped = skipped_count + service_skipped_count + + summary = ProcessingSummary( + processed=len(results), + valid=valid_count, + invalid=invalid_count, + skipped=total_skipped, + total_processing_time=None, + ) + + self.logger.info( + f"{LOG_PREFIX} Expectation processing: processed {total_expectations} items -> {len(results)} results" + ) + + self.logger.info( + f"{LOG_PREFIX} Processing cycle completed: {valid_count} valid, " + f"{invalid_count} invalid, {total_skipped} skipped ({skipped_count} unsupported types, {service_skipped_count} no end_date)" + ) + + return summary + + except (BulkUpdateError, APIError) as e: + self.logger.error(f"{LOG_PREFIX} API operation failed: {e}") + raise ExpectationProcessingError(f"API error during processing: {e}") from e + except Exception as e: + self.logger.error(f"{LOG_PREFIX} Unexpected error during processing: {e}") + raise ExpectationProcessingError( + f"Unexpected error processing expectations: {e}" + ) from e + + def _bulk_update_expectations(self, results: list[ExpectationResult]) -> None: + """Bulk update expectations in OpenAEV. + + Prepares bulk data from results and attempts to update expectations + using the OpenAEV bulk update API. + + Args: + results: List of ExpectationResult objects to update. + + Raises: + BulkUpdateError: If bulk update fails. + + """ + if not results: + self.logger.debug( + f"{LOG_PREFIX} No results to update, skipping bulk update" + ) + return + + try: + self.logger.debug( + f"{LOG_PREFIX} Preparing bulk data for {len(results)} results..." + ) + bulk_data = self._prepare_bulk_data(results) + + if bulk_data: + self.logger.debug( + f"{LOG_PREFIX} Attempting bulk update of {len(bulk_data)} expectations..." + ) + self._attempt_bulk_update(bulk_data) + else: + self.logger.debug( + f"{LOG_PREFIX} No valid bulk data prepared, skipping update" + ) + + except Exception as e: + self.logger.error(f"{LOG_PREFIX} Bulk update failed: {e}") + raise BulkUpdateError(f"Error in bulk update: {e}") from e + + def _prepare_bulk_data( + self, results: list[ExpectationResult] + ) -> dict[str, dict[str, Any]]: + """Prepare bulk data from results. + + Transforms ExpectationResult objects into dictionary format + required by the OpenAEV bulk update API. + + Args: + results: List of ExpectationResult objects. + + Returns: + Dictionary mapping expectation IDs to update data. + + """ + bulk_data = {} + skipped_count = 0 + + for result in results: + try: + expectation_id = result.expectation_id + if not expectation_id: + skipped_count += 1 + self.logger.debug( + f"{LOG_PREFIX} Skipping result without expectation_id" + ) + continue + + is_valid = result.is_valid + expectation = result.expectation + if expectation: + result_text = self._get_result_text(expectation, is_valid) + bulk_data[expectation_id] = { + "collector_id": self.collector_id, + "result": result_text, + "is_success": is_valid, + } + self.logger.debug( + f"{LOG_PREFIX} Prepared update for expectation {expectation_id}: " + f"result='{result_text}', success={is_valid}" + ) + else: + skipped_count += 1 + self.logger.debug( + f"{LOG_PREFIX} Skipping result {expectation_id} without expectation object" + ) + except Exception as e: + skipped_count += 1 + self.logger.warning(f"{LOG_PREFIX} Error processing result: {e}") + + if skipped_count > 0: + self.logger.debug( + f"{LOG_PREFIX} Skipped {skipped_count} results during bulk data preparation" + ) + return bulk_data + + def _get_result_text( + self, expectation: DetectionExpectation | PreventionExpectation, is_valid: bool + ) -> str: + """Get result text based on expectation type and validity. + + Args: + expectation: The expectation object (Detection or Prevention). + is_valid: Whether the expectation was successfully validated. + + Returns: + Human-readable result text for the expectation. + + """ + try: + base_text = ( + "Detected" + if isinstance(expectation, DetectionExpectation) + else "Prevented" + ) + result_text = base_text if is_valid else f"Not {base_text}" + + self.logger.debug( + f"{LOG_PREFIX} Generated result text: '{result_text}' for {type(expectation).__name__}" + ) + return result_text + except Exception as e: + self.logger.warning(f"{LOG_PREFIX} Error generating result text: {e}") + return "Unknown" + + def _attempt_bulk_update(self, bulk_data: dict[str, dict[str, Any]]) -> None: + """Attempt bulk update with fallback to individual updates. + + Tries to use the bulk update API first, then falls back to individual + updates if the bulk operation fails. + + Args: + bulk_data: Dictionary of expectation updates to apply. + + Raises: + BulkUpdateError: If both bulk and individual updates fail. + + """ + try: + self.logger.debug(f"{LOG_PREFIX} Attempting bulk update via OpenAEV API...") + self.oaev_api.inject_expectation.bulk_update( + inject_expectation_input_by_id=bulk_data + ) + self.logger.info( + f"{LOG_PREFIX} Successfully bulk updated {len(bulk_data)} expectations" + ) + + except Exception as bulk_error: + self.logger.warning( + f"{LOG_PREFIX} Bulk update failed, falling back to individual updates: {bulk_error}" + ) + try: + self._fallback_individual_updates(bulk_data) + except Exception as fallback_error: + raise BulkUpdateError( + f"Both bulk and individual updates failed: {fallback_error}" + ) from fallback_error + + def _fallback_individual_updates( + self, bulk_data: dict[str, dict[str, Any]] + ) -> None: + """Fallback to individual expectation updates. + + Updates expectations one by one when bulk update fails. + + Args: + bulk_data: Dictionary of expectation updates to apply. + + """ + self.logger.info( + f"{LOG_PREFIX} Attempting individual updates for {len(bulk_data)} expectations" + ) + success_count = 0 + error_count = 0 + + for expectation_id, update_data in bulk_data.items(): + try: + self._update_expectation(expectation_id, update_data) + success_count += 1 + except (APIError, ExpectationUpdateError) as e: + error_count += 1 + self.logger.error( + f"{LOG_PREFIX} Failed to update expectation {expectation_id}: {e}" + ) + except Exception as e: + error_count += 1 + self.logger.error( + f"{LOG_PREFIX} Unexpected error updating expectation {expectation_id}: {e}" + ) + + self.logger.info( + f"{LOG_PREFIX} Individual updates completed: {success_count} successful, {error_count} failed" + ) + + def _update_expectation( + self, expectation_id: str, update_data: dict[str, Any] + ) -> None: + """Update a single expectation. + + Args: + expectation_id: ID of the expectation to update. + update_data: Update data to apply to the expectation. + + Raises: + ExpectationUpdateError: If the update fails. + + """ + self.logger.debug( + f"{LOG_PREFIX} Updating individual expectation: {expectation_id}" + ) + + try: + self.oaev_api.inject_expectation.update( + inject_expectation_id=expectation_id, + inject_expectation=update_data, + ) + self.logger.debug( + f"{LOG_PREFIX} Successfully updated expectation {expectation_id}" + ) + + except Exception as individual_error: + raise ExpectationUpdateError( + f"Failed to update expectation {expectation_id}: {individual_error}" + ) from individual_error + + def _fetch_expectations( + self, + ) -> list[DetectionExpectation | PreventionExpectation]: + """Fetch expectations from OpenAEV. + + Returns: + List of expectations. + + """ + self.logger.debug( + f"{LOG_PREFIX} Fetching expectations for collector: {self.collector_id}" + ) + + try: + expectations = ( + self.oaev_api.inject_expectation.expectations_models_for_source( + source_id=self.collector_id + ) + ) + self.logger.debug( + f"{LOG_PREFIX} Fetched {len(expectations)} expectations, reversing order..." + ) + expectations = list(reversed(expectations)) + return expectations + except Exception as e: + self.logger.error(f"{LOG_PREFIX} Error fetching expectations: {e}") + return [] diff --git a/template/src/collector/expectation_service_provider.py b/template/src/collector/expectation_service_provider.py new file mode 100644 index 00000000..627bc4c5 --- /dev/null +++ b/template/src/collector/expectation_service_provider.py @@ -0,0 +1,76 @@ +"""Protocol defining the interface for expectation service providers.""" + +from typing import Any, Protocol + +from pyoaev.apis.inject_expectation.model import ( # type: ignore[import-untyped] + DetectionExpectation, + PreventionExpectation, +) +from pyoaev.helpers import OpenAEVDetectionHelper # type: ignore[import-untyped] +from pyoaev.signatures.types import SignatureTypes # type: ignore[import-untyped] + +from .models import ExpectationResult + + +class ExpectationServiceProvider(Protocol): + """Protocol defining the interface for expectation service providers.""" + + def get_supported_signatures(self) -> list[SignatureTypes]: + """Get list of signature types this provider supports. + + Returns: + List of SignatureTypes that this provider can handle. + + """ + ... + + def handle_detection_expectation( + self, + expectation: DetectionExpectation, + detection_helper: OpenAEVDetectionHelper, + ) -> ExpectationResult: + """Handle a detection expectation. + + Args: + expectation: The detection expectation to process. + detection_helper: OpenAEV detection helper instance. + + Returns: + ExpectationResult containing the processing outcome. + + """ + ... + + def handle_prevention_expectation( + self, + expectation: PreventionExpectation, + detection_helper: OpenAEVDetectionHelper, + ) -> ExpectationResult: + """Handle a prevention expectation. + + Args: + expectation: The prevention expectation to process. + detection_helper: OpenAEV detection helper instance. + + Returns: + ExpectationResult containing the processing outcome. + + """ + ... + + def handle_batch_expectations( + self, expectations: list[Any], detection_helper: OpenAEVDetectionHelper + ) -> tuple[list[ExpectationResult], int]: + """Handle a batch of expectations efficiently. + + Args: + expectations: List of expectations to process in batch. + detection_helper: OpenAEV detection helper instance. + + Returns: + Tuple of (results, skipped_count) where: + - results: List of ExpectationResult objects for processed expectations + - skipped_count: Number of expectations skipped due to missing end_date + + """ + ... diff --git a/template/src/collector/models.py b/template/src/collector/models.py new file mode 100644 index 00000000..86021fe8 --- /dev/null +++ b/template/src/collector/models.py @@ -0,0 +1,168 @@ +"""Pydantic models for collector data structures.""" + +from typing import Any + +from pydantic import BaseModel, Field, field_validator + + +class ExpectationTrace(BaseModel): + """Pydantic model for expectation trace data. + + This model represents the structure of trace data that gets sent to the + OpenAEV API for expectation tracking and validation. + """ + + inject_expectation_trace_expectation: str = Field( + description="The expectation ID this trace is associated with" + ) + inject_expectation_trace_source_id: str = Field( + description="The collector/source ID that generated this trace" + ) + inject_expectation_trace_alert_name: str = Field( + description="Name of the alert that was matched" + ) + inject_expectation_trace_alert_link: str = Field( + description="Link to the alert in the source system" + ) + inject_expectation_trace_date: str = Field( + description="Date when the trace was created (ISO format string)" + ) + + @field_validator("inject_expectation_trace_expectation") + @classmethod + def expectation_must_not_be_empty(cls, v: str) -> str: + """Validate that expectation ID is not empty. + + Args: + v: The expectation ID value to validate. + + Returns: + The trimmed expectation ID. + + Raises: + ValueError: If the expectation ID is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Expectation ID cannot be empty") + return v.strip() + + @field_validator("inject_expectation_trace_source_id") + @classmethod + def source_id_must_not_be_empty(cls, v: str) -> str: + """Validate that source ID is not empty. + + Args: + v: The source ID value to validate. + + Returns: + The trimmed source ID. + + Raises: + ValueError: If the source ID is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Source ID cannot be empty") + return v.strip() + + @field_validator("inject_expectation_trace_alert_name") + @classmethod + def alert_name_must_not_be_empty(cls, v: str) -> str: + """Validate that alert name is not empty. + + Args: + v: The alert name value to validate. + + Returns: + The trimmed alert name. + + Raises: + ValueError: If the alert name is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Alert name cannot be empty") + return v.strip() + + @field_validator("inject_expectation_trace_alert_link") + @classmethod + def alert_link_must_not_be_empty(cls, v: str) -> str: + """Validate that alert link is not empty. + + Args: + v: The alert link value to validate. + + Returns: + The trimmed alert link. + + Raises: + ValueError: If the alert link is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Alert link cannot be empty") + return v.strip() + + @field_validator("inject_expectation_trace_date") + @classmethod + def date_must_not_be_empty(cls, v: str) -> str: + """Validate that date is not empty. + + Args: + v: The date value to validate. + + Returns: + The trimmed date string. + + Raises: + ValueError: If the date is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Trace date cannot be empty") + return v.strip() + + def to_api_dict(self) -> dict[str, str]: + """Convert the model to a dictionary suitable for API submission. + + This method ensures all values are strings as expected by the API, + replacing the manual sanitization logic in the expectation manager. + + Returns: + Dict with all values converted to strings. + + """ + return { + key: str(value) if value is not None else "" + for key, value in self.model_dump().items() + } + + +class ExpectationResult(BaseModel): + """Model for expectation processing results.""" + + expectation_id: str = Field(..., description="ID of the processed expectation") + is_valid: bool = Field(..., description="Whether the expectation was validated") + expectation: Any | None = Field(None, description="The original expectation object") + matched_alerts: list[dict[str, Any]] | None = Field( + None, description="List of alerts that matched this expectation" + ) + error_message: str | None = Field( + None, description="Error message if processing failed" + ) + processing_time: float | None = Field( + None, description="Time taken to process this expectation in seconds" + ) + + +class ProcessingSummary(BaseModel): + """Model for expectation processing summary.""" + + processed: int = Field(..., description="Total number of expectations processed") + valid: int = Field(..., description="Number of valid expectations") + invalid: int = Field(..., description="Number of invalid expectations") + skipped: int = Field(..., description="Number of skipped expectations") + total_processing_time: float | None = Field( + None, description="Total processing time in seconds" + ) diff --git a/template/src/collector/signature_registry.py b/template/src/collector/signature_registry.py new file mode 100644 index 00000000..ec5675b0 --- /dev/null +++ b/template/src/collector/signature_registry.py @@ -0,0 +1,159 @@ +"""Signature Registry for dynamic expectation handling.""" + +from enum import Enum +from typing import Any, Callable + +from pyoaev.signatures.types import SignatureTypes # type: ignore[import-untyped] + +from .models import ExpectationResult + + +class ExpectationHandlerType(Enum): + """Types of expectation handlers.""" + + DETECTION = "detection" + PREVENTION = "prevention" + + +class SignatureRegistry: + """Simple registry for managing signature subscriptions and expectation handlers. + + This registry allows components to dynamically register: + - Which signature types they're interested in + - How to handle different types of expectations + + Keeps it simple by using basic data structures and clear interfaces. + """ + + def __init__(self) -> None: + """Initialize the registry. + + Creates empty data structures for managing signature subscriptions + and expectation handlers. + """ + self._subscribed_signatures: set[SignatureTypes] = set() + self._handlers: dict[ + ExpectationHandlerType, Callable[[Any, Any], ExpectationResult] + ] = {} + self._handler_signatures: dict[ExpectationHandlerType, set[SignatureTypes]] = {} + + def subscribe_to_signatures(self, signature_types: list[SignatureTypes]) -> None: + """Subscribe to specific signature types. + + Args: + signature_types: List of signature types to subscribe to. + + """ + self._subscribed_signatures.update(signature_types) + + def register_handler( + self, + handler_type: ExpectationHandlerType, + handler_func: Callable[[Any, Any], ExpectationResult], + signature_types: list[SignatureTypes], + ) -> None: + """Register an expectation handler for specific signature types. + + Args: + handler_type: Type of handler (detection/prevention). + handler_func: Function to handle expectations. + signature_types: Signature types this handler supports. + + """ + self._handlers[handler_type] = handler_func + self._handler_signatures[handler_type] = set(signature_types) + + self.subscribe_to_signatures(signature_types) + + def get_subscribed_signatures(self) -> list[SignatureTypes]: + """Get all subscribed signature types. + + Returns: + List of subscribed signature types. + + """ + return list(self._subscribed_signatures) + + def has_handler_for_signatures( + self, + handler_type: ExpectationHandlerType, + signature_types: list[SignatureTypes], + ) -> bool: + """Check if a handler supports the given signature types. + + Args: + handler_type: Type of handler to check. + signature_types: Signature types to check. + + Returns: + True if handler supports any of the signature types. + + """ + if handler_type not in self._handler_signatures: + return False + + handler_sigs = self._handler_signatures[handler_type] + return any(sig in handler_sigs for sig in signature_types) + + def get_handler( + self, handler_type: ExpectationHandlerType + ) -> Callable[[Any, Any], ExpectationResult]: + """Get handler function for the given type. + + Args: + handler_type: Type of handler to retrieve. + + Returns: + Handler function. + + Raises: + KeyError: If no handler registered for the type. + + """ + if handler_type not in self._handlers: + raise KeyError(f"No handler registered for type: {handler_type}") + return self._handlers[handler_type] + + def is_signature_supported(self, signature_type: SignatureTypes) -> bool: + """Check if a signature type is supported by any registered handler. + + Args: + signature_type: Signature type to check. + + Returns: + True if supported. + + """ + return signature_type in self._subscribed_signatures + + def get_handler_types(self) -> list[ExpectationHandlerType]: + """Get all registered handler types. + + Returns: + List of registered handler types. + + """ + return list(self._handlers.keys()) + + def clear(self) -> None: + """Clear all registrations. + + Removes all signature subscriptions and handler registrations. + Useful for testing and cleanup scenarios. + """ + self._subscribed_signatures.clear() + self._handlers.clear() + self._handler_signatures.clear() + + +_registry = SignatureRegistry() + + +def get_registry() -> SignatureRegistry: + """Get the global signature registry instance. + + Returns: + The global registry instance. + + """ + return _registry diff --git a/template/src/collector/trace_manager.py b/template/src/collector/trace_manager.py new file mode 100644 index 00000000..983859bf --- /dev/null +++ b/template/src/collector/trace_manager.py @@ -0,0 +1,201 @@ +"""Trace Manager for handling expectation traces. + +This module provides the TraceManager class which handles all trace-related operations +for expectation processing. It separates trace concerns from the main expectation +""" + +import logging +from typing import Any + +from pyoaev.client import OpenAEV # type: ignore[import-untyped] + +from .exception import TraceCreationError, TraceSubmissionError, TracingError +from .models import ExpectationResult +from .trace_service_provider import TraceServiceProvider + +LOG_PREFIX = "[CollectorTraceManager]" + + +class TraceManager: + """Manages trace creation and submission for expectations. + + This manager handles all trace-related operations, including creating traces + from expectation results and submitting them to the OpenAEV API. + """ + + def __init__( + self, + oaev_api: OpenAEV, + collector_id: str, + trace_service: TraceServiceProvider | None = None, + ) -> None: + """Initialize trace manager. + + Args: + oaev_api: OpenAEV API client. + collector_id: ID of the collector. + trace_service: Service for creating traces from results. + + """ + self.logger = logging.getLogger(__name__) + self.oaev_api = oaev_api + self.collector_id = collector_id + self.trace_service = trace_service + + self.logger.info( + f"{LOG_PREFIX} Trace manager initialized for collector: {collector_id}" + ) + if trace_service: + self.logger.debug( + f"{LOG_PREFIX} Trace service available for trace creation" + ) + else: + self.logger.debug( + f"{LOG_PREFIX} No trace service provided - traces will be skipped" + ) + + def create_and_submit_traces(self, results: list[ExpectationResult]) -> None: + """Create and submit traces from expectation results. + + Creates traces from the provided expectation results using the trace service + and submits them to the OpenAEV API. + + Args: + results: List of ExpectationResult objects. + + Raises: + TracingError: If trace creation or submission fails. + + """ + try: + if not self.trace_service: + self.logger.debug( + f"{LOG_PREFIX} No trace service provided, skipping trace creation" + ) + return + + self.logger.debug( + f"{LOG_PREFIX} Creating traces from {len(results)} expectation results..." + ) + traces = self.trace_service.create_traces_from_results( + results, self.collector_id + ) + + if not traces: + self.logger.info(f"{LOG_PREFIX} No traces created from results") + return + + self.logger.info( + f"{LOG_PREFIX} Created {len(traces)} traces, submitting to OpenAEV..." + ) + self._submit_traces(traces) + + except Exception as e: + self.logger.error( + f"{LOG_PREFIX} Error creating and submitting traces: {e} (Context: results_count={len(results)}, collector_id={self.collector_id})" + ) + raise TracingError(f"Error creating and submitting traces: {e}") from e + + def _submit_traces(self, traces: list[Any]) -> None: + """Submit traces to the OpenAEV API. + + Converts traces to API format and submits them using bulk creation. + Falls back to individual creation if bulk submission fails. + + Args: + traces: List of trace objects to submit. + + Raises: + TraceSubmissionError: If trace submission fails. + + """ + try: + self.logger.debug(f"{LOG_PREFIX} Converting traces to API format...") + trace_dicts = [trace.to_api_dict() for trace in traces] + + if not trace_dicts: + self.logger.warning( + f"{LOG_PREFIX} No trace dictionaries generated from traces" + ) + return + + self.logger.debug( + f"{LOG_PREFIX} Submitting {len(trace_dicts)} trace dictionaries to OpenAEV" + ) + self.logger.debug( + f"{LOG_PREFIX} Trace data preview: {trace_dicts[:2] if len(trace_dicts) > 2 else trace_dicts}" + ) + + response = self.oaev_api.inject_expectation_trace.bulk_create( + payload={"expectation_traces": trace_dicts} + ) + + self.logger.info( + f"{LOG_PREFIX} Successfully created {len(trace_dicts)} expectation traces" + ) + self.logger.debug(f"{LOG_PREFIX} OpenAEV response: {response}") + + except Exception as e: + self.logger.error(f"{LOG_PREFIX} Bulk trace submission failed: {e}") + try: + self.logger.info( + f"{LOG_PREFIX} Attempting individual trace creation as fallback..." + ) + self._fallback_individual_trace_creation(traces) + except TraceCreationError as fallback_error: + self.logger.error( + f"{LOG_PREFIX} Fallback trace creation also failed: {fallback_error}" + ) + raise TraceSubmissionError(f"Error submitting traces: {e}") from e + + def _fallback_individual_trace_creation(self, traces: list[Any]) -> None: + """Fallback method to create traces individually if bulk creation fails. + + Creates traces one by one when bulk creation fails, providing + resilience for trace submission. + + Args: + traces: List of trace objects to create individually. + + Raises: + TraceCreationError: If all individual trace creations fail. + + """ + try: + self.logger.info( + f"{LOG_PREFIX} Creating {len(traces)} traces individually as fallback" + ) + success_count = 0 + error_count = 0 + + for i, trace in enumerate(traces, 1): + try: + self.logger.debug( + f"{LOG_PREFIX} Creating individual trace {i}/{len(traces)}" + ) + r = self.oaev_api.inject_expectation_trace.create( + trace.to_api_dict() + ) + success_count += 1 + self.logger.debug( + f"{LOG_PREFIX} Individual trace {i} created successfully" + ) + self.logger.debug(f"{LOG_PREFIX} single Response: {r}") + except Exception as individual_error: + error_count += 1 + self.logger.error( + f"{LOG_PREFIX} Failed to create individual trace {i}: {individual_error}" + ) + + self.logger.info( + f"{LOG_PREFIX} Individual trace creation completed: {success_count} successful, {error_count} failed" + ) + + if success_count == 0: + raise TraceCreationError("All individual trace creations failed") + + except Exception as e: + self.logger.error( + f"{LOG_PREFIX} Error in fallback trace creation: {e} (Context: traces_count={len(traces)}, success_count={success_count})" + ) + raise TraceCreationError(f"Error in fallback trace creation: {e}") from e diff --git a/template/src/collector/trace_service_provider.py b/template/src/collector/trace_service_provider.py new file mode 100644 index 00000000..9c4abe4f --- /dev/null +++ b/template/src/collector/trace_service_provider.py @@ -0,0 +1,24 @@ +"""Protocol for trace creation services.""" + +from typing import Protocol + +from .models import ExpectationResult, ExpectationTrace + + +class TraceServiceProvider(Protocol): + """Protocol for trace creation services.""" + + def create_traces_from_results( + self, results: list[ExpectationResult], collector_id: str + ) -> list[ExpectationTrace]: + """Create trace data from processing results. + + Args: + results: List of ExpectationResult objects to create traces from. + collector_id: ID of the collector creating the traces. + + Returns: + List of ExpectationTrace objects for successful expectations. + + """ + ... diff --git a/template/src/config.yml.sample b/template/src/config.yml.sample new file mode 100644 index 00000000..2a162707 --- /dev/null +++ b/template/src/config.yml.sample @@ -0,0 +1,10 @@ +openaev: + url: "http://change.me" + token: "ChangeMe" + +collector: + id: "ChangeMe" + +sentinelone: + base_url: "http://change.me" + api_key: "ChangeMe" diff --git a/template/src/img/sentinelone-logo.png b/template/src/img/sentinelone-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..ddd6d518dc23eb95c33d0c5c842671b1cd4669df GIT binary patch literal 63856 zcmZsE2_Tef+y8W0Po-T`Y45ZgLMxp_TC_=&WfEgbN9rI&s99=Gowkv7X|tqcHzb6f zk&2Qe`#u~IF-*ja<*5I4-_J8c@Ap5?_xjR2b1&ERyMEVlFI_a-u~}v0w2?F#O=atr zjk{>HAzd`uu$3c*!QZ6C>@9=;H^_b0=JkWhvKF6$Uw%EZ#oC=l8#NLAZxHS7{V6nB zx9Zl7>nx7nmsDk6kF-8<{)CF^4E^>;uEE236JsKnoRT{a2MxY4(**#t zjgG2+@Z2{hlkajVHaj!3l&fR#r*M0S=eBUu{DmO~(O%J=oVmqo<|WSUxYEG${QT!H zG#bCD>s40Z_JVu*0o=|Xya)*YN9j9{KBwkg4jj%j38kXOX`z z(Rm{vCJvdUXz%_#^9=Fe{@>T!JTD-Y_cyodmit9w`AB8+t6nE(h)0~h(m0rGuXXvY zQnj7^-Idx)e^GvF$g$lfVVWsF4GT2(CPw#v%&VKgnL1Q47Jhq{rtFvguItAeNrp~P zj=XlFiDc-c{$IyiN`_8W{@Ol~DN&u>b-{ofmTkbU*LLfaJ-z6GHxtoiq*mQ1~0NS8w=$1$$e%}Hl<3#A&D8{rg}f8@iWtt=66{sRPr|KaNOqh@!a9{Gbtrs^>$nk7+|ox zyHIk6u)k6>uWQ&SQE0Gq|BWC1r!G;NoN_*-w6|k1k+J-SDO2Cwto0gqNcG-iuWts4 z+^RPVek?atbg*jpJ$q>Z!^<_SE?c{7glL58)yh=f5BEMw`26#+Hj)_{UFEl+g?-8L zHkah2?~2d%uUag#fWL3-RE+M(;@s^9%S~K^O3mBGbLru7*#*&$1(fJxk|%Nktsi+M za5oGM6kMrE-9mUt`{bR?`IK@DQ#-tgr9)Zwt_xk#y<}`xrUGq2CNgg#Za(Yezni{h zjOUIGllLgPZ2EjB%W=3sUv*YGV>>~ifZt;+AaXzC+dR@Wl!SMU5Y;vj0df|T{jGx< zyxD@>9R_+Pt}=;vToOBZD9h%$&?Mb2T5Ln-MtaTbG1x}piN9+%o%4mMpE^e#yhZe1 ztxrv4in!wVET*XSA*H+v@8fUex?ChW>Be6Z=5?1(0vxe@3inOTcO!c-z0V43Oa^Cx2&g|zpCz(wv6Zf*hoF6CpQWPRua{pwU?&BKo+1>5lt4^B|NgEXE*gK^+LtJ`YJE3()UHp+`Dnt5PX0R`%M25qt3K0Dbr~1n#9S^MyBeXt21viI;Dvk z?rBz-;Oy0@8RFZgO*80yMGN7}n+rM33Y(v;em(=HQZEp$Ob-=TXvnEy&TXr#>pa9B zj6N(92({ot&R!yEb4TB0(m-o=RbC`Y=wFzc)n@6TwNBBsPFuf*%P-&JBuTwWgvOK( z_imrrDIgZ1&O3|dLFe-ZbiQ|SOokmhzGL#0#MgGeWrfyyckLO^Z6e~|5-NG}Ygy9I zv*wV7x5|I>4ecET9CewMc1#V-k?VovdbWx1kzcE)q_FCiUYtC`h|Oqr(-t8?a5T|N z2i;s-`Z#{#HTtAa;xnDSiB5(9p~8ae@vA|nT_&>AXv!R4@W4<{rqgZO>nC&7MYo0o zvZcXpiuu4mbP+HD1wsc9$|x%-WdXMFlQ^#L^VN_B)eQh^^j)o06L~*$6hu{+{kW9# zsj1aptGjljXvB5X3`Sy&xm49m8cm$Cp$2C*3v~5rD9bs^&jvv9*JKdK7B^P~Z24@= zSGawKr2>^rHFq+(yr#lY02_a+26w}^{m>%;YFCg@EsBmlXtv3W$Z-k$@!HXz!spf-@41A zyWUNJVZLkbXN>l?|LTkjIy{l(ZcHyZoWZyOgEG3#84Juo$$3g3s4K*Xtp^y>j;M2r zuhrNL2$~ipXN%K3zbL`ou-^vE=>b5pugonRWNS$HuPTA7rJC!-_DQ2*jJs$d(sdjzJdmTUU8l*EUrWHc@Qpb2q8g*ATh3 z(j@u$U+A1>(+nV!v|4XOCQZc7ezG^rnm(DME?R||F|Af(CrD#%7BRP(5RqJM`0JVg zuESk14d}>wUKh}LiiYSoo@nZB%n-{-KYtDQnPvS**9^Q@?G{nxX<@5CxB_P4{hK@@ z9F)A)i}D^<@43V!eZPDI>_(Lpt;$-Ar|S?cw{mmsBhBmTnLc|de7QDTv)gTyh^-;I z4!G35UJ0(OnM2Cf_Vk+@!ZBFMfP=>%*l_vFBE^RSh$osBQ{0WFuVF$??Q;_`BUoCZ zaI42hm?pj!H>s$i(Lg)ra65u9d9hf?AX$KK{%kVmOHRJcnzC`6&@JdSVT(XGgq&L; zM?R!Fr}tr&Fty|Jsg2mGUYu<8CW<(>fx5!G%453P0XqR~Vd+5L;jOvDb-62EOsK?p z8GtIy+Y)TrjObXC;13?cJd=W@fh0k@#Q!c6c_wi8kHEcG3ABZHoDmW7R^n93NhWC* zd|4x<>}<`hrtq%0BxV7V!AeZ>6G=`HZMiPIEYJ=l+Yu`4G-yY_c9DUJD<9$^Xy8}T zYIRW9qC|Dk1x1US!Ukyl-QKiP@41Mq&^y^!Xr@HTJSwh^JF)BphhGXDgx&yq zJE&+yOJUx%?Jw(E5H(IVUEn%)9A_s`glR^}jtnnr46< z)D*uB_6Tz3Z5h#B!QhmDHwO5Q%ui>tbRPqtDm1`(L|bs*0H#T~Qkix53rIR7njk)DII-R}_bF>G8JD#)$>`)*GIzolNuM_|(HB(Te~hD0B?8~fH1dZ_!(ov)$g>c@3gB<9nYIJ8@V}t`-!9N(V-BZ zMbhQ(kcWo1_h;x{#(*y53~Gj8;qUrp14E2fhbNOb)ptXZwn+b)FaMx{AOu4HRh2bH zVa6)O5GT2sboBj2$$g_Ecp~u>@I+|bzGiKR>cP!45$l_X1bJi;4n+82lF6L?#%TT! z^$@kGPbmq2*l&?Z9RJ|xiOAgmZ`2E<&nYdd+LAuW@He~rTn=J_K@@5dY}=T-s7VK~ zU9Z;IOyqe}u4#!5IDiU9HjWpNe%`89cLNrXej=m9sq)Dsq%<>wQCM0DaR(Z`QFNEq zd`i`(gjZCJP30*}qeryHc;kSLdg}v<+5j?kgQ*nh{tf!V>N^MzM%O8hw9oMRRUx$| zVt5IMv$0zHBFcSh#%Y^~mh--BlJFd%g3#`Ztr-OUS|I(Gn)k{*($42BPtq9}&eDkU z6W4}>B6v{f2E$gZM!pntP9f)EAx9%w-a(vd&(_8 zc`b<1OE^5NHp65XF?P1dQY^Ti7>M&p0x_4KjU)s1n>mmpxZe8ty*CtJ0^1uy=F`z< zPvV$=+!v5f`ZE+>!@KM)xDAsi*A*((NEBn!xa#M(@RSZ@tnGCiZ@;k+9U`HEQ)jiQfTKga z30TZy3mF}RYrA)M?}Agt$XILsgJff^jUv}05nLbH^`RfZ5Dh1_iM_skPb5>17bk%D zyeLzJc#Oe?=nxx)xzs_}?_*Z^_u0FeV-iL7Unh`GSE#IzQ>4(siz9Xc|8*Ne$U}KG=#o7P-|eZlrgpx`AUH!^r1rPB@Zo8u0*IT~q~AArU2r!DHkX z1E{?q+NGi-K(uLwSA>$FfL!>MsSA6)vq`9!4F+=Yhr|*{6vGdhA}dX40ZmN=M=BIz zPiVGGxLpjqz;Q4}wtGl%DpTA3uCBMBL+7=)2Pn*!l$nam85g~qfUIc24-<` z2LjK=njI3}DMUUfrW5@~VZw*m<^p*#LR_fWDG`YB@LKRllJRwcB2bo1p}AxpDR|?ycXisRk^v#*b!>F)3{*ry zF)V5FbP`7VIH>!i-3G_hA9BU1i}wEp&Z1MG&9f8l!VCfJi}xlc$xarB=#nYssFzAAElNlODaVq=J*KUxZa8Ev|MR-*8c+>x8mY5}?Rvhk_Y0YeKh4@EK@Pad zH`H}Bj(Cl(VlNyZa7AG#3L6k72(PlG;@&SsoCRxJ^6#acM->RN$kK+xIr=)txGcCQ z)J6LM@Q@{vPWfdM0Wq9{`aR-JC7^JMam?HN&D&ch46q{bzT|`pGa_%y+EIXGG3=uu z@id1f$(Y@gV64Mgjw%Xz-qr1p^c_)POVPNVUtKf!9;Ja=S^=G&@}SUVHU=2ynZlk7 zRIBGB6ew)U7X=-d`5j_ek{L)p)F2=!@Iq6Yn?>?ui6K#fu{F+{v2g8OWq^i8p}jdH z3E7Zh%q3>HR=__FJUUA=Uu`xnQeav5J*=I;mEUoEO!U1>s=X-dauA?m5d;+DPiO?<~;$kY5yvF@>q zq!UzGJ@21~D477l@Yu>!Rc>Snq0^mB224{jAD8yF6;-KiP>?=2HHpA=po&AqY8FtH zNwdWkAaP78RfCTe27vfR;i-YN0H9QTCMfWgJ(@Qp;aF?cUk1Pt%r%RQpc1!k*n3w@ zIuk_*C7qJxXRv%TMM$~qL<+jvCz$BlPat)V{glINp_-6+g$Tc~p8Z+mUQ02ohEtv*7b0@#lji7OiXY2a`IaanBcR)u|l z@uk*gYjn9~6C<#Pg88^*13R3*S4w*IuS6Yxfzcd_95DRt0rsGPXcKeIhvmskYfnNVK#2hp)A;J5%-ls5N)ktf%P! z8bfY=EGWzv!T^m<_HB-Bm{R--_CeZRCiiOV6wl0)^Pq=u=S|`oPVfFWbSH(JyJ0A` z{$J$e$_!3$aP^J<973JRF&C0IpHoUzk&7ep^^Q-CjFR_w>H_+ot%2bi+yCpg5hm(7 z1Uu~d8{AKb8h)k;NB2MvcO;m$t^4<*QenxIR3T0UAnRDi%> zWqAF6Xvn$gR z_c9o{VTz%Z$KQ%!th+8f_P_L8_94+dp`zQW1Etyg`?lT$s=142**|)=K4raP+Rw?j zKp~^UiT9)}pS4_SyNw|=w!PJBpF2Mvo?Vo+c^ZofkQ*MzrjE-i-kZ1_0BmceYsDq~ zvD@v6YRQ(8w`L_0J5T+m-!vLpo)O^v2tAZ>b{dO1{#dmO zG12}3FmiUVDYG)L*8u$;83VDA6sB@nO)Gh`cjTkUKf59y3A7(!Po0T!^8ZK=i1^;~ zi9fe?M{EADfKXuX7fYSJ_5HS@hti9lW&4bU*f%3Cqi<*}d#m%rr!xK;@Y9r+hvU57 z@slf$AN4=xAJw=zyf#}4S%j!fnHrb9k;U=;3*sT|%zE`YU{nZtsLA-x?LLP>4AM?B zu5uFNT%6U$R%RmE-Wc|N1!0XBT62B#eH9X(sZLd+B0B?^&h(0md9AftF4ZWs<0qC* z;%NDz=PXm*5+@S-RI8Ve&KRliUg?6wi{90*;awf54#fDPXDCa9e0Pv)^)~DTkv-@; zkF9t`FM9>AuHL%;z+JIqFsxH~hrr`7lx~d0Lw{P!aWlg%2V2nO#Ir8{swaH~-*c^T zu-WQEAGa5zj_1VOLXQbklk6r)Al;+vW`^_diu5wiBl23G#$KJg;X#Xm{_OvC`vOnd zfA!?79b&p<{ca{3Kd?5vLwlNDfq#d72Q{@{`8TezOpo#u&<)>JEYF9 z|4b`&Pd|nq(k%YafG`1vLI?_qcRwe;?ZH)@SU%&xdWo`@nas0#(*@ zitZ$PQj22}oHmyxY@GE`{m>ATf@W2YmOuIUvR1>yMW5AyB#vvOQFR5KXcMP`@~87$ z%Wd-HD#2sJ{9&19?5L6*iU-Cd^ieEy6>?-z`cAp!@$~X|RI4|e*Jq!bQ9wON2~H1c z^>$DQwkf`q;NViWEpc()@WPiT;~JVylrM4XJO+<*O_TKnOgiVbqM{4sj3vAEiZACp z+kep@$RdBqv_&#o119yLv6pLz6lLMESe+;pr&m_b_rbf9fYqw3*`rEKRRfzt(Su1u zQC9%}buxj~QnI%;<`>!7OajBoce!qJcgq7uyU1O+P zt7Ck5lul$$`EHbN|B*~C6Fxh)Y07r+UuN)pT!bmVLbJOar;d0)j225y!bhS!WX5V# z%K4D1#mu0EtN!HC3&=)WQ(g2%4%ePy5Pu;Ox!`&18jeB7s6;>h?76|g!! zEDzI(@QDzsJ>PlJVDF+st4s>iev{d5o~N=1WFH34K{BIsF&1Uxgf-W|Qd$-&3+L_W zHXu}ata&~BUMMu2+Vkvc3iXg0P&kHo6BqM}LLEz~{=s{w?*C#XfXsKr2VaYfhHXvo zOsm}%fzp93?MxDCp5g3Vtpwwi{9#_d=biT~)0Ww8V^R+(98|+ctWfqyC*0((8xisk zqOBJVo^F_C4(pLlhweLt3vv?*;aMR`umBjV{no8TE^Tlu2*)es4`pHc+d3Y&0nkcd zxlrq|;>p`&sY*8oBXJGF;&k3=Q#d)n$Yp1z*eLNJNon=*nlf9H?B{nQx@=9D@OY=d zA1%0Qu3T)mWw5@lS2!kNL(4~hy_=02k{|6{3Ayb9h+5s6sArnHttICVq>N$=RL3*r zK{{b7f1QbtKXLh}<#N4yfAHA(SBDjw$2G9`RCI!@*iG>4`49eBaxg;|HpT#-NK8BJ zT6&i!RmQJv$!(G?%sc-Kc5kiC&9`|Lv}uYpu+^vtc>cOSDgvY3hE$>KslJP8MS;vh zT3->M|E6Bc& z*KK(5a9Q5@?4!FS$&WLq+KeCO2SaPP+jMuDgL8P-o|vvosCOgZ5H617xqiOIkv%5L zB;wzCmdrfl4DbXNvG{Z;rt);IQ4u^VKNwjeqDrcToi50fZ3>E?u5>Z6F(>Gax49ZU zqU|03{)-T5{VXCSBBCppX=%xgiHM09bk0v8SkV zsrfNaHD0y6gI#OoJ{wuyQLpXT-UV-N8%G*D{j>CQ{NuRsHzs z?=IW!8@0~XtxPT;M)v8l`sihk}#J&yV;eP?USwB3LC$#tz57%^2tD!ZQeNb`_! z`Sm7Zx0^J|=6Yk~McBYmV$DWdXL5^>e%(FLe=6=CmTmsl?fSOZCwV>sj(6N#e$DgG zIK3yvy+^y}ZGn%Akw`knhp@I)~da<(r>q4>4^O~ z^K~+JyI<67o`CygwndZQJn(d?%cgsI2e8`q>7XqbUcdKbh*#}5$!M5{-c8O-foC$$ z$Fi*oG8p709knAMER+Qk2eoyQ=@&&yyWc|5>(JBWw<&KoO>xnBvUJluZ)a<|HNvD= z;EGn_z|iuno~pVS+3Y0f#;s??f{oRGY<-9}wfH0&U5<;W@FvQ*$J%jxZE)=)EhLzoU!O|5F52tC`+a9pwYQ>XJCkuX27p>M1IgyIH zpcCwKa9;TZq&VR-pHf`=RC}3NntjUOvMu=oBI1~M^qAN>Mn-qRiW};Whx#%crD_qT|?M>TL%G=5zA&@Td^NC4G&TnR{g;Bru{{9NJ#w)U;cNPD#jyyi!9 zAXBbS%Ll_ggiylR15fz@;nNIQam0C$BOOW7^8xc9z5MLv)|ZJ!)lih`T}(jz^j(Nm zM#HXV{Mh_}Utm{OBv~4T5t<*@m;d>rqdy)eot8r&HcX0!)> zc+cumxf#0Ku@%UIkP63^pOFRWA3yh_XlGJmg{ExV^HLb5GZR0}Y4D2$x6^{#O$MX! zLtWO{e4PArXb~r_&d%H>Rx8hIx5U>Y!LrS0n&*5bY>gFWYIB2GX?NKkX5wd_PwI88By~}-x@Zj< zaF_9Qf>aI;F6UzrM=&{p=CF{0xrQXZFT5uhhu1LBNfJ8UDOiyoM+ib{4A{Tnb z%cy*y?3669&+&&3tX#FEX%WwYrF$eHP(N#ny~fBN%0Sx(lWS7o$xN=9xx9T_N(~Y~ zJ%g;RLI^2~?{i*Tf$SQL(>X1mr6j~h_mq~PF;!c(C%HDFYpx1$J?5FZXbQs?s5M68 z@TCWgW>sA~MI~jtr(!5P&%i22IWr21io6p>tZD3m4%k(8BGYcGpRFqEWO~E(W_Z*Z zu1D5tG{cWk$FpXH)TTyJ6{o-)$8Ti_r9kbD-&88OeyTh#d!fzKlQ~Em-1$s*q>W@? zCEJq9DQASmPKToeLWMx=$ONg3BMTrLrLd8%M`j0s3UPE}bz-R+r^EMHo@HA-oeMQA zc#dkNsEeMUAM3SJ#zu5$Q5k=d3>_qR$yNAj{SxT`XJK}@!uqC^t5eZ-C*W|p8PIy=?QjIuk-;cNKfXHXUZtiV;N$qI>^L?3PyQ!9%~;q2aoJkDO*1AHHe(HrebfxY$n;=j zG^8}AsO_f}v#We|q9xtl9Rz(}H7W!nBX_JW^O7;gcZ$d4$wR+&^cYn?F7@tUt=g9I zH3DtJquaqFUPZT4Wm%&i4s`J-GTC;lDDzD8cR?{FxVyxvp(k_O6B~{9OMjS185L>V zKTg+Ze0m2mSTV#nr9Gm{-u46p5zk|uFFm(6nj!uV)}nNKuQO2%wV4t|OnkFsb!rUW z_E$yc{dkj5(_3*enQqkXd+ZCT5kd=|qgn*g2>jTiO?+AYE$iu*?{>)Dd~sfV@55ti zW%OQa9a|{vt1dY$ZHODnnpy(Gf3$9QegXqI;>cR}7LOL6r6ZgO*@5sbos;9;%MWe- zZ%~4#{60>DfNb4m_w^lfRbj0w?S~^DZFbw1(nX4eMQ72gSJ7EWEbwC}y_z_+&*gR_ ze9JBIzNTY#nH$uelWY$sJ7&XifNc}F4z@f??zMCKwld)TOOZnG(j`T@nM5dD{We3toh6;RnTAoFJ)B~J5b{Cs zR(m0lFKTA-!G#amR}DG=?3dX-~p8I%HBP*VsRHlNw9zjqm|?#?|ugf*417u4`zcfRUw zP{YWmTKA!zLoY9#5jik0kr!%1HKvG7jOGY6l6xE`I zkBrH2{OSqMW%pmvIN-<7{|iEb9IN;u=arpy;WnB{m1<6r{-&)fcS*^mc;WNN zSe%7G3^MY~cy7$gvC}U<$Pn|!c#FF9`bD$?!h-{$8Ks9Zia4Grtm^%8idH>^ip z!w)%6r=h#~WpT>fAATEY|S2fmg-&onms0)7MK=(Q66nHDML;7(||E z1RMQ7=-)z1mxY}9txKT_df*Cdp(d9{{C?O+VIO{pdCBiq^_iPx((mL=VW8z<(mc|x06Yk`>3BMn1-izsiqecYmbm#p@F_Zev zl{;%!WXsguImig@&?2q&aRS9;D9~VDBSRQLu4;GbweHgz?GGND(2$8XN|^ko;g1nq zpg5nZfdm#V-&VhUVM^SN959q!Sn%qaP_H|o*b%~zG>H#3fY(?wwXiem(9%XeX$&D8xcH*(nub)PWa(&2c%dGFC;;|Rl z2OSJPh+0dRvTBbRv-$Kmtzg z`vE8CLnWYaV09E8de4Dr%WfiNI}={ND~RPjCKm5_INpYSM0vYk5L_vJwF3-fs}0OE zKE(O=PP;oc&~NcA46(Dt93HM|B0lp#zGXG|7XRkwtLYYPaImOgp6tj~y*zpDDD3*_ z`!?g)Om7=dtVW7B^^pQ#ZR$tnaQx1HZAg_aLkz=lO9qz?XGy zssP&DHy|T3!#g2hTbUoV6{VgTh(hCB2#p(73v<_L7}$?>{DcCCqU1m)j_$gke`!-%V>aOrU#tj!-Lk@27n9@RP%_^c zWi4%?_HPQGYdnR3RCN#J@5~0()rJ`*A)6)t&UF^Rb}9eV-w{M$jSFfoTVdjn!QyPC zqTzdKH5$?h(bznqCLi-F z32(6OJrHe7i>`I|iTO5-_pd<6SudG0LL<-5GjBb?cx8uGg@I;=fCdL}+M_XNH{nLV%x@r>M>G-l75k3oUMhiOLC>?Ff`!6?oR($0 zBhT^4FPYhN<2<=3oM;n>cGU&=Sk!JJcRhx>BbdS*C+Ss{4trq#m2g%;53-F-G4zE8c;gh5NM znU8>7-~e$U=T0-)c0}&Ao_nbb&Dl7Cob%^hu(iPSO6dmo%8Q~?wm0XKBS>5wNBKTGONM=KH7D&x2#c2Q zz>5jkaVG%0ivZq#F&m25(LlrezdIn-&|9Q%SnCpsIk45_4?QGj*gzU0FY~SG6~6DE zdIeTDIrZ$5$*uX%=F9jK!F@`#oM)?{uR(4DQ-*OL?G%SAX@x+vVuu_({1|hQz*PET z@YY_(yNNlHGiNrED?8qCrwTbft+PP<+V%w6Er}5_Wu7jY@@aC)TI}=%GXAx$nsvaU zvhI+U8-35h5w4%{D9ycPOXn!N&Olw;qpsn{1^t?K$z;T#&_ibHPU*^64gxilDWQsp z&DZ`?71nPe)~{O2brZ>;JEfC&^Ic|k+I@fc7!hvm=Pcx$VKbVWwzm>c@d!|55KYIF z}hPj*|}-)*F_+XU1t>`QqPQ*MG#Y&N17B0-3i_chd!Ni_UYVnHNPJ1s?1K4Eb_ zA;o!lrU<^Xs3KTfp1D(6zt$dAyXGIhPa+Y*srA&pZid*y1~@7u8p7oiCKKop;ePPQ zh8-P!ee02Brn!;|KouA`Sgl8Gqhwtgf8XY-uh+k1+{PKcByPCHw(_M=QUSdS|8seW z6Df>b@mU;sAVYbF45c19&U&pBgw{>Zw0rDlXhFr6nBe8Z)~<=!2}}sb^@dizk&T4X z^Y5p%&!gj=0C~|Ozr)yn%a4ul`jW%?O~(KHY%=fAoA>!PVe)rXD=-AmK+UDh-AX2^ zLQHfR;_;GJdv%ePzWvx2b%RZfs)dt1TJIodkXX0FWKRK5ixN7a^H8A(=MKcmlJejjoM_+~*Ba z9We!ZB(u!BF>)P>LZBRdyc>yfS{UG&wlK~HH$P=y^Eg&VL1QOsz6Ukm<4>UG@1HdR zvpI=@gxm?)!Ah3Yh58OuFx!5K0aLKFqP#!tzHdmQ4eSA(-2?l}uiQYBUs^H$Dh_k! zz9oVwl_^dbEWh$6$jxX=>YsdYy*5h!p$HaNrpv3F{$gaD#DAN& zB4epL@T(SOWx_3#bt*P64%&aGbnv*-E)Y=&|3_4X94{%-OErUv9T~kww#Yp_s%_4{ z7rI$8Xhg=Yjy|_r;+Iwc_-b_}6jEV4K_}>>KKg@7=2pNQ8@E) zcxQu77LSY888AXYgsB53BWI;kCfH)SA(Y{(4~ z?fof_jJq?`Aq+FTF?}-d;PuMQn+MF}CGbP_QCJ(qd+_j~d=Pv!u?~py#h)n9_f6Q` zPYmYLmaCjzC1i|}Zv!{070qY9-yDwc+_=vioUoDGEH$(=L-jLiA#wy{tONi}RA}TSUqo@sq?`(6X_@L>MLEqeQfjoxwV+-6N3 z*pbFh9R(wksTnPq!0Sf2;5iI=lBfnH>uw}@1F8`>N2Ne&e2tkuv@Z-oH^3%8K3=QG z-uX)98>4};QW_ZonjjLihms1~lDQ>N5xQv2_Fgu|8|?nP5p2=#--cG5Q7=ZfG$_#t zTS{id4W~`r!m$qm0oSipN31;_+cSM#>&o8;&SDf3f(+U?ejiZ$I#}8H6ch#rI`H;% zm=kQ0oGZUfMG54F<&|)A3Vh+}$vhzSQ`=#I*g{H#iAOEK)@ZNP9fJKnk|t@V-2s6I z9`rN0-Sl)nlh*!&M5yV=CnzpHwL>zY+2HBQM3DF0iGXQ_H* zH!$2g>M#i+EWKT=(L<_#OrJAwni?pxqy?>q`pK@#=atZb)3@qf1~ZHlxVo95Q-$Vvs4CvMlV_Hy5A^$50VFSvFGo#oa5JwD$ zg?6O}K$2gG7?V~rTUs>%aVG8q+%-CN0Ouwiv4y6nE~}hJbR2aMN;fqbtv5Q(8q{us1j9Wg0mJFXbRt_g z@Ce5?4G}EcPm{v*gLdcZpe^FnKfIU8){%=oMpvnmGhRoPXIv5S34_O<0+%vhMx&`- z6ZM8l|3LJOeDP?-4oOE}&U~ep00)ke3%DGtOtWXM}G#0l?W0vVGA<%u+Ajc3kYATey78NYRzRpRvX~DDe`4N&v{djmFjA$bt$v zc-dPba&0BLum<+7;4C1#>!k%(*Iv33?1}IENZS3y$rOq>|6}K;PfI^7?6;TslwlQX{argHHQ~OFGxFZkv zWTOL1dB2gS3*kx>R@0(O;8K#v6=d>!5nL&7yRrf~Ygh`xc5dG0V~1c!oAe>E5m(D_ z)ImtH}IS%DYdLa<=PIK#U}#^A=Lg(^#oPzDHQyP}TJ z_<&1UdANECM+XC*{7#w*ilG_(^$cvQS5c(Z_YXjhMm*Q)&O=+F`?YNe7hw$;n#N-k z&ZM13Zx2qM%oFZ7xq{;%1i!BEii0}E!{Lh07;g}3xpyISacdrSKfY{goQ9s2~biCAdI1PHJ~Jy<{VSZZ=S8tJ*FdMIf{&oE=jAOQE`~D085}PDDXH9Q+)6 zPdTGPj$;@s^eg%^%}COcoispsK8-fz#jXzCsas;h0o*)MY36WnZ3J$vX3+Cx-QQ^g zUeb3%cSTU7HVsu};kJ+EPX<_-^l#4c zoH)_m@Uzkx4uluEyol6(mPDPQ5u{!eDi|(=m-67ID3~OJAbU@xoku2$)!-^FQi-e= zq}?98&>xjpmT*FWdP$-n=?CS-G}?-{kQtZ3)+Qw&;Na-PI`MI(=$GNB6X>tKy)Jhc z(17wNu@+vl!OrVjsBO0;R~yuqE{_ zNSlq61DSZRl%t?fczwrp&L?Fur50Wb2Q{Q9BA^`^W*5M9?a8m>#QTk*=+Tb_Rhv!9 zpx!Hw6v>N&&!1AXKukFe(IJ&2h|Z%cd_)nXWNGw=6jE`JtsAVcGEv?Tw2}tCfWH{{ zNUBBB3p(SOt*)>j>6R?8@b8zA0U3JQ-Q{}0(P1SL>Cy5Jzbju|Oak9Cv9?p3JhXbZBL9YT#fpUeqFen6w{9Zf_9aT;sS`PA_ep&R z9fyQQFTR6~D~3o_BIk8%bkA?kts_9Uc zpZLGQ$Y^NO>h^RHtGYi(GD4Nm`k*Cmry@%RPf<<;DSdn1WzoUoD00c6)H|?68e^2E zcl>BO+zB<9DjLI>6_?UjlI%zu{b!nz$jvC|uEfzWjrQVVe-y7luI#Fd>;Y_0?1O=> z8Vo?QQ2=e!BFcyyRI8&)dsUJ2Y4`8I1v;>*gUWiO%4bs=65xs)9B^OXZ>s5Vok?nn6w(fa<7jyCR6NMrA3xTl0m9i!3u zO>R#K$uFU{*zHLmtL>kU}zN9;oL_c0>K20@41q(Eq4|WReB^@GnYuGi}KxXopd440$}5 zd!*ug_%*5s0ot7m>T5wMBJ36nm*1sqUK)l1GQt=OrZjuhjik^oGbkxToP$G2#Fvhg z0->sEN@|y|Zz8&=d9)8YCS`Lqc* zzP&-2P|y~*Hcm$YaolTUJW1I!i*!YLHf<9AQm5ie+j=M>nTPBKU139+m0>3^q>y6B zc(S<&R18X6fF-sf<3<)E^ot9XLXiX$Li#=$ZMzQDtP`B@gO)S!^2%83W@)kM35}dJ z-A$6VbCfJhFx6@ArtIkjZKf-ZS0PSM#^eols*E3KM}}Nk$bu@pLJQ#WBy)v3Fz_4{ z8ecW|6u%Hld6+G^ve4Be zFz`8*+OOO#^~E$O8cz9sn#J31!?8vnR$;&eaWG}iEjaiCrVk>$Wl=FF1jiWCMOh?o z8=gV+Hb^YW*+M%!8O;oF+6R(7XhHh|0=b9qt!dap%bs8eHUB`0%zt)>v@sq><#<Ci_jYX_ zv*utV)9450_sldA+oCu!orM-U7=I$~_G1Jo|F>@+X5A~;49x6&X*->*qkz?t^a9o0h%Iq2^n)u|;=R!Pgq)5cyHdZb%@^_klZjpclW zi(F4#6)!C4wPpuyk^Xq!w;Y^9%^Fe_aMf(v*;`!4iHgL#D1L$LO$OMe6&{D;L(s{C0B1vo_ok-sQ=DTFOVsOGJB*>4WUuB5R zsf%6>9moY9XyhttBm-GE|G;s$-ged_uYXD_u98<_Wsu1r8lHn6cQ^_pkkSx~5V}{! zaX934>1Mx!0(%w{NREgyf~~k!vnFzA)ApH5h!Y81#~U2eYPhgf`dEJP%?)ihUQMI< zem&UUV~!&*yiSN6;eKd)Fx;fDzh1l(E`{TDSK%9|`2tuo%r&tK)Gs099}unA9dcH1 zrB_)0C5%DK0xF;ktB);8jn$5GfV(xZrKww}8&~-6V40k?8=(Re=eigreyh=x)=c|N zre#N79jNlPvThW*xfjZR>9~XU;tLU`)^*I&)O~0O%i=*d_63YiWxEFC0|qfuyr}Xb%b4)uY%e^{tMuy zSJ%o5M3F{R|E1m*nU5PajL?-RW%X(>kkqgmpz9y&B!j{N;4aLn{~;9l`!P96l~sL>46s2X0ATmd)fK}ufoP1JK3 zK#Ffq3j2o{4CDqCqY23l2U5o(dbxD7ku ze#V?4j*T>{E)pq!A-W0AaHXxc$Z=s{UlC-Fs z2E#7Fn{Y8@A;)*kX<;p?b7R;aJMT3F-CKX8;&YOKC-r?O*0gNjqsq0!YnTVny}AFn zus2_TBHNgBuU?VtFuu*z&#Il=IaT7#y$UR{t8#Y(Zjmn0;xAZa$q&LN^Kx0Oz*VI<@8q37q}|t7I25_+4J5U zLRT2dkm_HryFOBcK|$=846!?5kIQ$U_!zt0_}`@nS_Vnca&pfGxRi}w{gV;Ju6PBv zoyjX25&;0heAtyEFQ!Zi{euH&nPG%z?{(>D5~^@t@VB=xqchn`^KWYZV1aP%L}^<6e*hN=bDIO%|T{`oYZPvu?cA& z7hoym%URUo?_I-4D&8$1w7JXRZX=Y(L7AGog%yAd|5^jN4l+VC`nszr3X2Y&LpRR* z!`$JntVQ@fUjDW}7USNBrI%ng2iu_}(I_F1?@cDjW;=p<-w#Qp@r^$jC#3(*SU7!* z-pex+r`_+Fr24P)$a>9VPF2#Enw#`*I({E1SarsHR^8~o{xRgoE&Z&%<(~|rr;8Vj z&M}%j&gb`gHvX2r{!Bd$uLG@7@BDk2PWq;=^;}g}yM0MJb^UMG*Vm-2FTC!5h#hXR zuR9jaNM+gAgmWU@A9Jc$SNgob1-IOu1>+qiY8xi{bj4qVb4o`tD8oh{OX|#J*aeVN zkm}!RxnUN~@Q;|4!XK6?X=!OE`W6l|7+kHjSrordy8oPh&@X-GP|4V}@o#C4w<9>c zMeqA^nooKBc=y1yW4vbj)GnPVU86aMD|^PG)fVxGCgQ^bDf`>!$PyLsCmFj9I(!U* zjSL2}t}K@fD(c$u?M`pL%Mv^aJ)!(_Yfr~$c<=0;Sv{L(eGCr#+qE;+%k3&mY?72) z+WUUkD0~-HjWi_|*qhk$p{?ubE=iX(!CKSdN>VL(}I5aN*I#tGR(wrrOWEIjG zeI$^fWV2@eow$33EsulKku#QF4@(=Kta2aGKfGcjrYB zXh3E@IaqE&0ozrlH6a&$czfGk}5jU?!o$gat>k-Vv!aVTG zE=A=$5Bx;+8`5UV1ijJYluAgMkp}G>y=dWWb zoL5&k*~3MxM^Pu)5#rP%HwyIBdS)(%yL%=*{P;k4;qP34&;BwDAGo~kkFO@5TjLJ6 z^)5xD**Q|J=LtEQVihDb06}x7*tyav5Q$;Q+0i`fFNTw2r7atwEJJm|!JQtRLo&7_($dS>dPVeEhO zz;)r^zZqJg5OZS2!K8hiEBEThwxWd&5-5KK(D|q2$esu`85&s8>-WYyVxWy zXsYo1reRc&~UbN+HqtAL;Qj*8dht?`bSS<6;#fEZ6T7i!(RlE z{D}t3_>(A^*`+m_SB)S_8&{I>c_QbF&J<+>jcn;lUUSFcP$U=Hxa{s&`9&4V2mvyW z#ZF)}1`vGJ0zeFoE+ew9kX=LD$4MKd(L9Bm(u!k>9XrHdSg|zPr`<{@Kp;2rllbfd zX@&4H?kQHknZh=bnu}viuS|l-n`X;Q!&F9!(k4P)J{XY%jh|iNtPl_A@)p&HpL_HP zp0*O>=v%|0TtI$03X2@F8bS1>e zWyGA}qB(DnU=h!0h{X4NIF9nVj>BsY;S?>`xzyN;`-9l$+K8F(f5#ZKfPS9n9`Mnf zA`TOQlD24PNo>+8uo+5m+68RajE{0ASUt_~gB3-JI7Hx7R$+zku2=Qu}VA;qG<6H$W7C ziv-lBmz0xKKvDoL_FcUaomko70FY`DI!vv#3Ua}YUV%@(Ipe4CR9F*ODYwf$D9s-6 z!=g^H;DPiO8pxsw&!2|5DO#>o84W=W;vw2i$KC<*F68V}xDeXAW&KP7P1-$U3!9NB z4Fk^2P}m&O@+hVhSwElKa-bj7u*~a?VG)CdaKXXrr%uX>IP}&k#SkvJ%76Z}7yu!a z1RNF=!XU%LmGXu@P|rCP|v-{>lMX)W4eKdM6O`M{K`7Yt1Dzuk^jQ4 zwKvr3CdR}ci}nxHB>XcZ{zs)^99Zxh-%TB;e86=w8K==U?iBw7!K>+<2KWbn3GmrdWHDqEj=dfD-b0nJtPiZ*lB(Cy_YrK-@Wg0N z#T0y_21VQ6D}ZKK4-*|*y9U&U1odujX+<$G6iL^}EFS?!m*}@vsdXXatjmo}YWbTS z9cE}^3xs#VH_s5zXPcfQ<_xd{pkE(|l><8Q2%k@potyrDde{1+l{uy7`=Y3%=U6-o*b4suqUaw-*=T^7`%5c|UZ4RM?NSClFw8Jz!2hCkip-OWh)Dc_C@_u? zkH^}aBbE+Bg)GG+=BmK$g@3<60T%7m5{b{s|I>d<5NL4L|D)^67D78Y1B8m&GD2PB^5Q0_=0ym-*P!U07 zQ%HhPfv`m&5EhZ&%(*uqF7H3#qq+B;^LwiV2aw zC9n#J$CBtMGTqlh1)smGi{n}btCw%qg+psx1Qw%5fOf{v{=8~N^8#~C21H2@&jhhP zgdau-mROmC=Wg9laoFY^5k7*GepTU1l@l@Voc}VIx>q|mQTTl;5Qy_qQ-p2Z1(C}D!7=6njI%BnUY$Ubd>S{$LBX8d zPFX~(vWqz2f#qhskZYNYuAR@PmqFZ?CukRl4D2#uS0?du;sIvvt4RTh;DRafBHx|O zolaSCta2Y1Z~m>J0U7VY8_u)Uwe~11>@YG-jEd!5(jgu2f^vAI=sK!IO2v#!$oP}6 z=gE%po9}AHMeC{aA+A;qe`h`=0tW>tVBl+avv;swA~;6~8W=`behmPSvq)$4!H5zTfvy*&gABI?tDO(xpC7& z-oEQf6IQr|-h_jmymI#KTOle0IM|Z1MA42d53RH`1{UQ~0`+$te?#EoPDKM-7ry5a zyQ`xJRDJguYF3wlcJ82Q5+N;D)#=0}>D`XH5F?{0Kh>8@`+78aE+N#Gbot+qBzbo0 zIYZ&!L;>HF73qXFCQ*FHl+VRE!{42S$5l0ky;&o8Z`%Y~Kk$YNC}^>gJh@f!G)mrsrTD<0TyM{q^)Na!4~h14nJ#1IPpYj*W31iyYG>>dVXQ0`MY<1+ejS} zT-!I$kW*($CmxqQsqg9^aubk7MIec-2Z}wt-<)OSdW!uBvVnOlNhCCg2Hw?SemvpG zB|rDS=ew))cbDElHRQLo$~{%A@M0<9NfS$7y?eX$d6bYgU+e3-PRiFnaoPy6BJ+MD zyp{-{0800st?JLIP%eevM83ZD&B>62`XeONBwse#^rMV9@1o^8{D{lY04ppcU1pj0 zLW9mKQP<58lJKi-HmwIe2(=oQQI;v;w`_0d`{$ZAqreLW7}3NDOwnKLxy}AsWODtx z6U>L?lJ3EpWmUEJJ|yX7$${@8i4_Nlo^uBC%6Z@XccH-fi*$(ONs(u`((ei0N^9OT zR3_xn+cvb%{^cm2U|ekRzS(W6<@BM35;Dq!*WV}d|1>vyC3-HbnxG7qT-EQ9Ax-8n zUDe;aki^N$b?=$X2SASl?MN$1kvxmnoM8Js|LPzkq21Q{0^$M~!GSdogCz!M*hh7a zKF|?pK?zdX10(E;GOww#ignRXO=L1j#=lv7uZBLPN6W6{=w5>Pj`aY*pHY)mo^H&e zs)T!MHp`tlNHGM&w0e0jYk$#jfJck?%UES&6n8RNQIad*R62 zv9pn~WPJO77#k8Rn_Y|y-X(R8YJu6?5G@fzA`{>(Ax+G5MUh$oEwVRx4~f_^6P*xb z`Ln%r+k2d51u4L`-gqlI5QlQ7zbl#b9$!jUpB@^(_l_rV(t-hEBhpjTNjXh(Tc?2A&caZK?Kbh(U)-91I>@H~)%b3~UrKnU_1B9Y&ubsRo( zAngqYr*^>4;%I48Wg63NR`ePMxV{y+@X_H+CVufA_oq7`F^J~nQ;gnsyKGz&^_e>6 zBHx;%(+y8zm46#3j;|qGLY9;$%%m%&1dpa02fKD?3;rP=(VpA489jEyDm(s{-Q{B1 z^+V1SDib`kIvTY7^~)Oa>ZXPMU|%zz@%ZGoZ4TQ6_Yg4;5;6adRX(pHLJU3`HStv= zxO|A#;vrFmO1r98P2Z-kf2#m!gD50BP1;KHr&xNw-g8iQM6K&IO~W86Lhkb*HBdJX zXru8r;Cz|WDG&Oho5D9QVoWjHLcK(r_Fe5$GxK z8}ILa+v5jA>qjMmBwNxLKko}^D&OEeA+wRH9*j}P>+)$*F!&4%*W($Al&B3pgJun8 zzN0<-jhc^D%eWnQwl#XT2A8F-${qHtIV3YtVH$7QBDt>qx#QG9CxK3Z$_zver5C`$ zEL+|~f2LodOUE_#ZyOOdnj|sby@wtCNiU*~0GAM1#@ntU>ts4|!p~?T%(kk2?kF`3 zeqS!YsFZcK^F8MMTT3$DmxYiMtV`gFnh79u`n9m5_+bO*yP6use{8%eoQ z3YKK7=F#9Q|217v$%FwcU^d!_%H>_)|B(<8efM zwD{_+Jw=0a(L9(Y=c~895wP)bBuC?*+c(DIRl$9SMK9-KVB}y}Bce@-^2oQJq<$!K$O*s)?!eCMrJ+HrXsA_LS1e9)};SrRs5NS8u!%L1Ayx zJmgV|R)~h~j0tPBC0PBCv;lTk#{cOZ2(l~7Oa&e))h+CaABz51@e<>dj@EAQ9Y8Z7M^p^GBeRWlRztiT%jgr&)E^)cP1w~}u*zun3> zgjEPZ_Ij(m0~_AV1?~Y98KYNlecxu5^xJe;=pfxaa{-e|;EXm3U)_G1p#`kF z4Ez05q((Gmb**KDlT|QG)jCav*44|NYtb9MRW|pnt`?c}hQ`YRN4fgVK#cl3@Pl2c zkcX*5*Y!Hd1U|kj*G3|!@6R?-WM>A>?UL+`iV_Y6+o6h%g%8@E37L5Hd9k91-!vHU za_Hq0ydx_&daE~s|G`?C%;%wk&e*O)G;0XtzkM0v^VZc@8B+V!)vDPRy)rvmC4{)r z(E_EaUJlx_Gw+23dpjLA#9B=osi8?xC#5dBzu6aoZCAlVa4~wj7p#|i)V*0o z!-iKho1Wvo^d2=fnFs55QXp<*p68dKt83r^FKH*6db(80w+Ko9me%zAVTX18A0qt+ z%>S>fca#hA8Y$0mK_b&7GWEif*hG2%)+p*I%GX+l+q$h#S|ns_LG0fvx}pPv9N1e1 zkt*-C-G`v8&~J1S1H1{7JnrOH$nQ@w2ivHy;6#8he@gTgS*75wp4W8Q$a}@m8qaGo zTNmO4mmbe3Q0r!!&sHm?7G1Bmqa>H4y*iQv(GS&=4mJeHZ=w>-s%GefSX@TO*p&5R z-ji4B(8Z#4)YJ`)L2}P_C%WH;>+rjSdF8@T;nOm~=4_cYx};uk znKln_e)VDH`zt3(VG$C)@@o<{*xE1{qK4~pSJ4i1V0iSFqg}-XIuxI%?bx4PTE}9h z_NV{Nxz85yUpC*>euFzR?9N+SZCmnMU9o?Np5D#PKAOT9UQ7vkUKZ9D6|TJ4$Gxa8 zH!faWY-itP^(o}yKaBz$<_(4cDE)9_C+ug;_MM=3?Kk`7xhm3y^NQXPSg~9C%&!qb z%fdd(mfgQjx0NmplKqWt2$Mf?Cok`QirA0d#W%R7NNuMjnw8B_GB*!0{<~0nfCP3S zngS>`ddKVaZ<+@;M8U?*ik%f1D#E!Zu9Ryr;I|Uo?!eZyyDBo;=`TwL4K)Nqk%KR6b>791N1_A(T0& z;0xOkU4HiTFoRXNtR{t@4V!n@r$W}XctY#wx4nbijXb;*W!FP@4~ZT%T5HIoN8 zu?V(Hx%aX;f4pTiIN^dS{djuwA$ps|Aq`98eA_8=wE=4%T9?hKU~bEBly6hCk~aci z6DT8FrZg1HZ~Mha@SHANW23iu%OhJ;eHStDgAEPe3jH}I!ixM&wC1-yW$+8j@~7#| zn|}@!JcV7mP-K1Qsm)zvcdR>XjK0at_oAb`qoAd{SW3UYof3sQ8Q;7Rlh3Lmrowg z5&7_DH}`i|?BD-E@T&VM?otK|9R*9-bomHv@FldXJqV_@GSj1(9OtADZ!@vy!h~#2 zC!-sh;;eNAW=^))evA>7J4 ztgd&acT=ZpQXRju4K}rQs`V!d6+sqsd zIOs$AoGGSfV&s}7juu!01jsg6b{!E8k$4$MfVNnpi7u#!{V0COmj&_oE7pIO>`(KfRzGjP|AmbA_P&Uk4!=Os0rzn@;P z^!k!u88^eX_`=RuI8|Yb+iZ6Q1~^F`hupZdxjLgcO?@p#v(5U45Q&<*i0=d_tXxja z@z@S_aK1F_87lO!?6zSxFLQpDDe3M_u-$<@+3+(^S?roL@n57JuyP$;<4rLmJE@ar z#Wce{v+kk7F@{1rTye>J5=EA`hQ-s73?ft#?5m$({N&sZB4yUani(%!c2;GO?Swez zLubH{vz2mm9=LYLmm#o`)_zP(kTO(X4h>PVQDazLPF;T=XM14V8j==8S(q$WuJ1`< ztM0LTFMX*PDvUaU>)rTBlfBn^79cHzx0Lg3p~$;*H?=OzlFY<(XLaQ+*u%U34_e9% z*xj&>mu1*C_md1^(@gpNk(`%;f_`L<`++rEL=P#jYQSa`*v5D!ui0TIuVS105B-?q zg<0a~S>l7RChh$LZ&!8^teu{OZdxQb79?W4{%by%sDiS~2?E?WSf5Od zNrvrpZHMgixkEZ-hx+;z*g*L7Z+}RRVJnqz-o^@?-ee9SW6!QY$0KkMT%Rk|IX{7H zjQ15SM#ExG+>0|r>vp8tmNS!N%U9{k2ldoFBj~oAaE=bcf3^I)}Ze1kC zGVrF&OJqS$@Wf0|-4nYII}`3g_l~T{6;%1QstmzEfd2{7{{H0#09RI)OL8`BKvqC< zf6`2nTigy!1ip6b*{EK_Ae=wk^rDabyFhynD9+#SvtbTGR-^1K(> zocQBQb2h^g#tt|(fV!Av(Hq!_z~P>v6Amo$sfKp}O9EZM8;+thWD9zP{q{OgOR(i-%Oc5xx0h*rv`R_r5B|a$ca8w{& zCR~VJ_>`3G6_vv3!&8|@>>z>az<1}218z%J?4&S5Ct-$8_d?07U%`IVT1!2M{jhnWtfg3MPr!fNaH7sk5NU!vU-aZwE34 zEPOt=|ILfz9r$E-14Q9HXRNxOF>^z}%&8?U!*z)vcpfVT0ICLa(DKcNWDhCgQ(t2^ zyma0Yjy1r`UjRw)l4i0igsjLP19~$st@(pgkh#7e_5)}xvW}B&lkzrW%TI5VH7L~~ z=B)gM6TU!(JmVU(BScT{dJ$~K4*+$;Z#S9DXg`J9T$z@;-mhXfIs$f}SH=NxdIyeK zZnqcf+7u3V@#|INS!_NASiRz0wA z=cvx57Hef%q1*!-aFGG%rMSU?hpx-rUxmRdC?*@g2FZ9I-Znx4^bRELz0?z#03n-I z1In`A2@iK$4SS<_L@NB)w?aMlP`h7zb{zK6-VuDyzU@>Tzgsp;;MA>A~R_{}`LvukgCOkMB zKezn_eSRYn4zi&FfzEb)GlKwWh~zOncNY9UDq3=4QT8@sVo`S}OH4>(Wn&iK4|YT? z;gFL!8(uuG?c)SZFt~v7;T0+t>O^|vtOJntO^u#(j)aUcS<+}N8A%KPeIa!s*&m>Hb9*2p z&`o0yKJH#Qkpr|EO+1+cDAl2-jRleBV8F?ePpu^l#1haGXc|fKyq*Giv-Pq*HD+_Gmh$OjjsTx-1v;uN7pxB#d_F`XtL3B{(+3*T)&*8XdCG&B$;^ex zz&j!ETn1=;w$%@unTXd@g7}^qAc{_P!BjrxQE)TX-$a#gD#kOgD#P|=0meMNl^5-r z3tG1lw*c^NC5oGV2{z-k|J1PvgB~XR|UO`Ld%sHI}c%l$=czmIx z##)j|h7p}7>t4`6#kIs>3@ESz!CeTyTt$S1Q$lbk)~3~z#6rDMCm~J0z%*^Xo5bvu zLO64Yi$2e_OhRwCv?fKYAb?*mygM|x8xkyD@@o(a*xnw9b9(FQ(V%AFaFZn;kKtZ3 z;lkzxkk;LchW^6GJjS3w3nd?GCi0nwa*q+PeI+gdA4u}Q(enV!a7F}491{bH-f%2t zR_jJraKQc(rhgGfO5Q#jlJ`E<_9}BQ0RRAI1+T+c50RS|D^a^kSZ{lCO-i}M(_JVf zXauYML=;gXSVSKd+c_zbs0rTPurY^SM|ztuP1qg@Te=)@%d6gQ%N>&R81EM(`&{n6 zezm)XC<)F;#Gy%F(I(wi(bsrim1{|0{jomR)kMBpGga8vj0Y4jKF&V`RJbe5j=3bj{dB)OElm$&zLhB;P?xfHge&>u+PyHZ4 z5WKB|i>FKl>b1KC?PtM02R=P(1Pd)z8SG|5h>wU7nB=IzNq#hVmTuT(yXI70H?VNb z4e-vL7LCTCG3bdHI{$HgFeNpl)h%iddaxN5k@Z2dsmlsclF?#0OeKfdV`O zL3EU{@b#&XYQ3A82lnvjkI%OL4P z6P|#{eM^owN#jkhLVAg~IcDFK-Qlx%EokZmBQTJGUmY@$downK-LZG!WS0}!@w#pT zL~rquN32MVY^&RLtS7@lC~Bm>({44@I|zM)*%sYjF?mIFJ(SnYm;z|~JW|M1T)|w- zdPG9(BI%?%P4@^+v+UrV4n*6)E+jSP+QXvM@h0kM-}AAgp!qPI zJ#aIob*iU^Y(Qicb$>(mWiXJ>#)6SPG_eeOG_wg|Ai*g+l-H-70UQOQ4^I5n>ZNRl zV4yXN=OCu8(p#gxrTHHrCJS=JQtkAnUq&D}A!#2v4;EiFBpE%zZKW`ZCefJE*1|d5 zqh!1Be@ejTyYBZGg3PqixC>3>U0THCr;(r^HE%t*#>dZpuWRXtm02D4w}u~5fby78 zo=K;y%5DZmldSLY{R}52*yL%{2vK&27S?yQN2{`q&U-A}=8ati+9~1PXLM9R1*T%( z;ErN3**-mFszsM%&mXND`~$S%2Z&)tFOxOMek{6FOWAlhTh`7%8)s%tGAq5B*fg93 zPLC8Jyz+&nI0R9Lwf1mZLvmDdHLT&%6oi(ob7apdonv6IlR34ukFLp#o_HxMwY;0m z1z0Q-4`3!t?=H);18T-^95m_%Ah&46Z=|friOvI?N;PzaSVfYgD?NED$AQMHR=_ZP zG%*2wYfP&(Nb33IP&yi7Sv;)0Hak2PUc}P#Fiq?yOCDNF3P=W}S13IBG1uc4yt23pi9!(D32B|Q9w2mI#FS;tf1X> zLnSfiW3FG@UJ*q+6loOF>jgP*ewTP3+DA<9`P%T}eqDw&iLIOMS;GFg7Ypbr=ScXlOz(yF9);EW5^Rj=RCL1C2ZAm81T4MJ!^U3-OYpSMl1EaLb zlK+^|?BEuCV)5LS>S z(kn--i(NX5N&mfEhlj}Lbgb;4`h7P;JiE!(!STPLA}sASDG-h&mPyZGoSf`u-yER) z_f_z4Iw_$V*+1o!>zvADH8GYs{~f%AZ6I4F$Jb%+G2q@#*F6 zIs=L4Q9+y5O{{OqntDz3z^OKD@Tu_VH5U;H5gYB+_zoe><^gC3xYD-x5MPHv2Ud73 zU~#BYSFNd}WA#fD`;AL@XM+HEP&K3fzl!Pdsq2$k*a7_^J+n zM2TF?g4c@kqTE|`^u9vb8Q1_t&bq_j7zr{4wES+eGCyGC23T>3cW%-3&PIv@9NJ3n25AsuRJckKLw<5i zWr0Q=uwacW6hh~*5GvOAz?B5?{BSQ(gZO|}>|jy%Y_j2$h7ZV~a_=L&zw{3yfzl#( zpYbJ)21V6Kk{CNK7>=*DKy^QoRDf0{K6{l?zxccsJm*#lA)%3=M!V0=-c)F*&v!j- zEI2OvNNYd9xw&8UVfxJV=eeX(fjHP0=?}B}agp+u{>`v>BO9WuvipMM zA5djl1aM!mV-U}PA&l17YMON7431zTDX*<6GRRt3+!u=rIQi=S^EKxCnTCb!)ozHS zZ$iBOGhrLInz+Q*vn;xf8d0M&M>EP7oexz)?@<$KfnCIll#zjdfG~e_G*_jFql^#r zWRB6ADso?r7}iDTsG>n5SC;sy-F_cPwJh}sECG21@P4nd|1vmW5n~IkkqK8Ym^`7_ zE_PQZ+TCN#djLnDPi-+4tPh;}u--0K3uFidiNS)c4#<@fzhGyMJIPc=1`+>D$)KbXb8o;`=0L2edcp zbDPN?n8QoZVWw zw_x9a%^t}3KGk;Zt}(=ly)K8}^OowUOi4M=hZ z=M80WV7}|$$_=1ESSr?F`D6&+71jx|D~k;N^AOVXIexGV`RVXOlaZZ=@?flW8xF&y7Fw9d$B*GI;TdX1+VC%7;kjCV#L6?^pT_($>VT$8 zO4GQ>5^KY^Ok~(mu}bN-3Kg*Mm~5n7FeQ|a)H?5qHp8jyof!C_^kTSIWUXVyAHa~V z**P0;akR?%GC`dSeXjzCudiq#ATXtX9P!h4;wuEKWmI|RIBAi=&ILdiPm4ozG~i*` zgV~-MOuG4PL7wpXxX$ylRUd4>w5=lJ6|p^&RPk-R18*o*G-tCUmrtu*gF&`9@mbO-pcEy?OV1AK_@kvZ=I4IbVXjH%Iz zH|MqQF%;I832L!jtjQetZKIQ9mDSVioWtSh>#~!o@lO$zc*6y15xvqt>YO4iXK;D7 zJQ?>gtOQSsHw>8H^$14#KNIea=I5*RQsvc0m^K}bD9_I z6X8hiI1+r|NY72=nwXn=MeWUdxz4ih+F7}&55SF;P0EF=YRK5H59@;h*@r_}FQe$u zT5i_ZT6Djj+L*
H&s2@S3KVz0{bmGEKS7Q4Mc|7Nm0#q@>rLe1eeZ`ZaLo$!Gu zho80V&_=3L-Q(5*`+;2nP5xt-LREag+L?D6`1|YIcYI{u=s;@6oc1u=m#{mN#6@Up za_{5?j!IOw3*h4yyJLm{S-G$XHB!!kA1qaxns7PrfqdHi=N!JS`LfV_zrcIjfOl_$ z*F!=#QZ{w6WFp3&;S4C7?ApWFZwz)jFK{>x1|QS+4hPVG)P#dg!@-8*VDzo->vGuS zC5`B`qJLO&2r!(DWTj-Wqt(i9sP$b>I zY!h`%o42o|$RG#ao>pwp+PXLnvfF}vPw-y&z>3veC)IP)!jZA z20z~g(cL(5F12l?aGGq^WJxA`p!-4c)*(3i0u}TYN6P9>u>HjO{*}JKuLCM^C|B8R zylpnQjkgFoc?OcUJcjm}KDSm^7_WmLPr(n4adnv09qdHhdkF2ap64l6Q+NBz5yL#h zFnqvMxT8(v*R_yue|0yBI8L&vdzR|kA>hapc_TqW)&p~=oXB$cz!GdnY)z%~4dhBK z`<^EPa`B_-dB&+cQLOQYYwY1|K5grMdkZq}=GFf7z+#WIeLLf#`Y8N3wE!Oi zANY`soHkEg13x1M=1t(0j~tBf#_r|8SOHMPy*mnMQ{gJgI!KYoY|TGP1Kp6K%aS{I~$29OWSK%;u(lEJ=K6k)}jLU685&lf!Pc^S!_LziqBUsgq>U5PSb{89c!b{w(sp z_LsWX>hH*rH$=vzm*K6ThH`0&+C5~OrCna;?-T7qSue(4=-w**?=Y(zr?x#7-M!(0 zPxP%QC*`=zRqi$$6x0#N)3U@p_f9_+1YeeZKO>k5RnwOf8C``r!{ld!v{Q^zM{q7F zsMmQ-U->I5+HBxeycfG$!FYbFJ3jY)X#P70x!ikfsHiLqH8f1V*kEwbbFBesWYFg> zFzDKlCxX1A?v62E3Fb9>?zVPIKQTDVaQFe`?nj0~%`7dcNY(Vk%1(zh?r@hxvEDJ} z!>#<{*XB+k)E+Cxyv$QXyQ@&UE<|$ahUzAV)^S6HDo&ESLS|qM>Pi@S75;T_&;Nn! zci=TLk+PwJdgVBfmilko#;QfbxJd$>!9>Z<9R5}W2AUP|5pTN1Uo>#de+9*FUVZM_ z=UW?|G^Pr+$R&;gQNP!it2Y6@yvM_X3UJ!u6+&2A&(YC$SZlx~GWA%T`bu`9`lv-> zuzgZ-e+s`q`ro$F^r(rFBN@NfB<^8@+N^Tm?c~ws-J0?3{8@@|XJk$Li8sg$CXS`fQ8 zb>32Yxxh0fg-oK{o2DkHzp-7ywK}%~VCi2BjvzN~lDUG8Z9{AF)8YV_{9y!`L7__N zn-GhJpF7{|ZuQrxqs7y~R<5A#rf4I932#y;ugU{&Pe6M=KMkHCfqp$Kl>Q1&NbGE^ z>Kl~+_Bwg+B>2&>Vu5=DwLT`67iZ_toD{GWH*}Ma^7lW!4+qP>{m!uicVR845VNi8!-H&rVAM4e(wBi3ZXbyG5!VtX1eGsJB|2 zSII&_ZGckiXBpWXk-^GP-pM{xn+UY7M(JIJqeGRYS!(eih+v|!&vqx&o*+m+Yq|-L z-gfH~K>7@oiJ~OMsm)Dtjks4Xs{2>-!{8#TMquz_+|yZ4ukr(xeC`i{Ix1Q6~o$>FpnwoUUpicpDI<4|K#r^ zK6b|$d~A+_(t|0a%~b7a>`q}1$S!n8Kn75M3Haq94yF+N+E_~_gZbT?7ThYnhWE`@ zpwH)IESeU>N~Li z@huG**Dq$vX0e9@;DS<%`xiizfIPCC^(6;xpq2deO*Rk2KXpV;6WVdrFY)G|hRG|!lTLY013*n#2!Jv*fV%Z z0;7QN5GvJbElQDms&yJjVTlh~4^o)&!^4uhmm8on9QH|A9|0EmVbCJu$Eal2cr)OP zL^dbKsoaHa1{xiY%J5C{J%P%Y4$6p{%YYFr62UoaZ4m*ec8ri!{wAv|DdGoKpw_SO z%>aXuh(Wx`+9xvXLiZPBi4b|z#Whbhg0DaoE@PxEWAwQC+{_s?=+s^w4C(k&2F&n*Nv zKLI$4o}F8@ylot$eNIy3a~G#EV7{N@(Ysljs}p@^o-P+WcI0N1cC|!JYjzdR72}wz zPzgtpd=pWaS)c^L<~Jbusk_Z^fTY0~z_#a}sY_a#rmLjJ0u!qZh=c2=T4wxdjX&90|wn)d&VgU?1yJau84 zgH)U1x(ek$yeaRzwk;U!1mnJ86=wAqtf`^eJ3?ju9DR1L2 zt`fqs&-dV-Mi?qa4~B}JPXHP|a|H0iXw6Ce-!E$Gy#0KlB)>IH#vYl&?i#Zr!Tcx) zmF+70on2+M-0HYhbX8;NAGma=$l^q7(wv_reT#6+3~Z7;xBxXFNm>NTyu|| zs_f|MKWH#8KL1**NIe>@P-^ro3bwB$0V;@JP&A*9>xK$HF&2XnLMmPO77#Pa0cEW= z0P#e>@&%J5<=L`7+QjMFJV+UFT%-=5 z6$3ZcN^m-Sf>24*U){3T?Kcvm*s8~Z^^%vXY+g42KCr5JEFh@_Y+JYgd0JpKMu)9z zORz~o)m~K1&RWET%!Dt)Z8nZvWx3QP1n+dSebqo6$p%WdryG(B=cvrq{v|<0NR@{fn|AtipD!WTC5Md?%JeQuFJ!LY>QJV^WoYAaG9eod)gWf2Wn`dUkY(Bx81 z5Vq9BT_|x{P1G}frI!-k&&75Mk)-E=CYvfCw_4C7iNsJ~DbIX(e+HodSEK|SyE<@I zcMAZfGNILYLxGamlMw&>fPy672H;0VgpxcFTr@x%CU&KGC|cuyN&zIxq_kBerjsO@ zUP>v)FrX#Pwq#yA*uV*8xJRlO5vVsI7y6^=(wuDVcen?cEBI@ z%~T~@ZWKU*OsUOFFoHfBp&GUEm(13IR-u8+MGfZj89*aA5F-o}NIqx+@ym?F0ezc> z22kO)03jbJHf>tz*&wCjt7MDvsEzbI{PdT@+#eWz4}yHc9+gwB7;^TRyP!fK?r@a< z7Ggng(2O`Z3L;dX2+6W2HiHA13x5=dU;v4vbv_Znz>@kX&%8_4(p^;EcL{aB2Z7+d zff6^nx_7)d4Q4HYr^q^LW$pSnakq?*CV>)b*7FiPjzNg}Y`Kt~_E8@E4oFK|5xeGF z{(aZ-f%CKmU%=n37QX13N$NUdw4Qs*9dWVU_Yf)%f|m-85wjk=ok4*io*lo*lLrw> z{dxFYZJ^2az4{W?s3m;Jm%vcp)Q^b2fA^u333PIWvg z%b)Wv7&K%gsN@B_BV`DUgO?tPZ{h(5-vJIhQHziHv0loxg~l5D)eLIhMNT@7}NkHuKU2W*TrMG(vxvQgK8J`Xi&Zt zD&G<2aQK$7p@7xm-O$&hPaOZ$QTE`R%_*@eP29@N`M+e&AFov`#5w;qO#aZ^XKRcX z@BXvrH+gwaNB8Z-ihFB_ImEy;L_a<$F-|QTDje`ti%o~psvn(Jd=6FrJE-3FIH*3o z>h~v3!PK6<2@^u>{do`L!T>clJa?sy8@Yo7wx992J+-n#QUAkLBWW*4LbaJ#9kiB) z!gD;1*UV`CNafqm6p%u+a4-zCP=Q)7hiR1-3ycYL7u_}m1+vuN{sCd01t^v`sxNqn z_-aQhid}bYA%MPYQ1#nXvNdqTC&Q|09TNh_AohgAl6_}@0=;t!tnIToYg#_86!UV6 zG}YX$OE~vgIQNTcnFWf=>i3R^5Ly-D6)GB#qU48VVNhNR?nCUmesm^WYhM5Tpu@*M z7Hm5z5NjNso@$7-^e9ZV(q~0zVTyZFEr?8pr zB_~U)3qQ(9< z1%_jJ2wKt z`^mT?VAPW^B2y|oXrMff2*g&jHK#ZeYaX_iMxZr!eVntoqH>LriD!=m`$Wmm);M?s z+;VFZwSqW9g8T%gqP}%MK$e^UW>wy|+fq^K&kMHqXT2DXPQM;T1II%eJk4LZs+JSO zgz%FX1lk^_6)$#%Treqrl6`ixWx3I{0Nd+E4-x%Hc`8oAy1JZcjA!N zlO$o*5~%g;?6|hqA#$GLtg*K6g2*)nZ)%_*h5hsSLgk*}>RzHE_m-Qem(cLMQ{Icf z^LLmf!70}Ct4|!c=k)1YPQu^BJqJKdvf=(bI@l=G;{bIp>_@E-_A3{K@*Ke)7b81; z8xmD_*C@I#z&yE{dHUp;Ql>LoVMjGdZ(+1rk6TOs0JWOWSlDLPbz{34#>+lTL_7(9 z6Zh=J@#etsQR3(d{&+IpSRC(%P~Nf}b}Ee6xzzx=Av1y{GNf{No3F;2c)P+!G{D4( z#UxJru~t!rapH{Uuo$nU;ucjAB;*UECU^mVw_2lO(Uz z<8#;tz@abm@v6d4y2JJOu+}p^If3}(%4FZ&=#!61O<=emrN#mM##2ZnTW&N3 z5?bs6z-%oLJthv=k~rY?C&U5%a%h&qO;1y^x5_0L2E+IpbO+1yfiz4wkeD89ZL+Tb z2P<4)A_t5|H>x%@K;dZoH4M0D-vXq9t8sRJ$zT=fwYr(0u2{Jg6q6sY?yv4%dCV|w z3eM24<=C@<+{dvf}o(|M?jkw{ez!#$a(#p#g zKL8r4+@Cx%Frn(tyX>2V^InOf-;@b#v?EuqRaoN4t}y#WX-;uypUPGhr0Ho3Aa4Do zJL)bfaxO+89Q6c_YU7^nj8RB&BBLZ_f?czsD6((On!aNf)FYQq5+$IAhJxyHq1j*{ z`Smywv@a~9t)&UT^E_Q3sC>fTas(9Zd@&tcG5u7xAPV_lDW$o^7r;QCD^}j<6zPn2jTsfp_cUfHz~+(N^7GEW~`iXzzh!- zGxa2Z;<&1d1o8gP7ek;J{Xd{`duiAMhQKmSot!S>C?H`NShfFSPsPH7?(cXjv!icp z$b$@sSeR}!0YLlwzaVJkW8jb4xm~c9UdGa!o?Bxh1cM1lPy3796+fd>E!IJh8f-{- zA8IoZwYkte-EETOfJbS{1nG5oMSppKZ7{~W`gM%~Q!#6ehrugPBVE+sW|XIbgVu8+ z)^j(deuS1zS3IY`_bHS>*ZvC@JgsZN7yhDL&GZUzdJ{brn-aRcg@2FeijR*5s&D6C z7<;m9gbSedB>aOX@aM_0BB!>4*|~SKeG@WHLHX!`?)luC>h+t|?0nzOve)0r86)fO zpc{;=s3TZx*+U2w+Rhzqr*IKl=ZI@swpN^&ya(QT5sAQ_d+ZC&x3KhKP;8aFXB7A~ zCo4<+DviCrdeQJ6OwP}P(u@0DfRTtWD=chV+yZ!U+m=^&1ZMczL~yA&7u1aks`Ka= zf03GW+w-B4CK&;`!tAfHR^FpDhtlyp1oYzX&shT0j(|tHK4_W%w@*cnY=N`eLl`r{ zs1ONmt#7hhxdL9gwPl8#MX&#DVKMZtzar+}MY;|itCXdL1Hi>;8z)JwStSbO*N4d; zpwV|+&WKeySmn<9t-;Q~^+6&IGztgW0X$QK7iOT%O?In!tLce(Bw5%y5tQZ63nrE$ zs2n00Sn)FkIZ#OLUkT8fxzQT%WV!IO$q*&t`z(zDQjJIZ7mhyJ{>&LH_ba(G!@N6C zc{hmV^)I;*y>D0RGk@c9b61#_bD?@`!?ut@B-8CjN0EuaZOV;|<~l7!0{KmB29{Gi zk5Hvc4m)5|#fh0>uE191W^(}Mwd{d4&P(qC{F1OhY^J_$AORlOm!ag&We|$R&@k?! z01xE{g89H2PWT42q&yY#>L4tZ&J84DLQ@$2^6V(JFFix>*?s@~)ji$vnn0d5On$_c zj}fhSIZ*hAPhq%2OsN{0NlisiF@~c;TiQVv&*(zBz<0<|lOg-~Ka ztDZU;q)f*$_Dy@q{EaJP5mA%G^$@X~jDWuXTSU+0v7pHaqcJHt!z&^1)cPqa`7`kH z%S)Eh*;zjXxb{vWR-76-a7}sCi?@)zzsYW=0p^?dbg5y$K-;qVG!X2lxds!w%R$N$ zCm38nX*q_S6WMl8+Bd{1sA)&|vuZEJ(z=7lVc9L&vuh9w&8Wc)f&A_Z6S>L_q)hS^LP8%o^AiI@ zk}RO3kfm<-Q5M|3Bd%hC6|I%I>YbE<(u9oXmx1`10ce<}MFXre0YLHE(!DAih7!pC zNkF<0AuBn05|Rs$+-S$~&r>64+p@SyO8nr>G59`Ii1N@d!ZpTPG`)f{IpKQo=VTCI z@R-g5RI&`M{=)Kj;Kgkj>K!8!AtFW01k2j{7k+T!rHeBSQ+G#Y68kL|TC4ov)u)KL z_-#-pbOy#=_01$WfVB+_n+_CzyGCb;Z67ab+c@tRwdD91GMUqjKqPvCWV_Q;`_GYP z7ODNpESMyTp7Yr1?j*@kemhd8u}QABKP-W{b8_MiX;ICu?z49k|D9wKD^EmqrzB4avoOTt)mC!1|eWaD$TGkyBOEc<#%p1t_Qh` z+n(LI#*^Rqe0tgE5goN!a#2c$Kl4P)7S4Z!~yIY$N^LD^&x14JYmJICN>02_#h5_>RT&l8( z>d|lTTZaW0Q-`QPaI>`GnaZDCA*lh`rMg zihW(?0d03w^$&?dYUTx~NXpmALsZl8~`mjvO@(F#1gEXr7dMug>^4 z_GST9*KCX?eYVLTr8jDY$THv%AG6NyL{&7H=r+q*HOsM#BY~g5} zj~Q7|8+Syoh_nDm)UnDD-}40j7^_&l6tw#Pz%4eyb{A-jzl8apZh@E~&r$b~!0H@XcJ!c`qx@JNvfq!$UL1TfyLHNp z!RyW4k)Z){p%8oc92+}98d10Dippw5piD7jKn8; zKZatYeI4kZ`bzsm(sgAfukJ$Km&ExEYp&$`HhgCLrB?k${=BDy>Nj6W=l+P2jP;J( ztXR^0!XUDzOxF*2amYrTm;Z!~AF_DxCP9kp9y!e0#9}JcMWF> zS(vQoj7-2&OU(!KE?vPmM{s22{Qb9>b~{=vF+m-zC7i>!qnU|BoNg&CQm0C}mb(T` zr+l8KZrnDoSlH82Iw&UZ+Z=X(1gLUslC(S;$|{0Z(kPYn3}WjuHv$RYK1*X=I4Gq) zN^9dm`>|FLoi87NfPH%okRi@FvTNguTq=6dGN$Ld|3W(8ynGe6vOnht>E?VvN;oi1 znHA{*OJ?Tq@5727v?RvBk}PjHL56b=6pN@MH9^@YWS<|FG{_jGw8fWG(WDUyP7H|- zHUjdad~ny7_Oi9Kko^#&L5EFtxUB=g7CCd!-K{5+PNj3iOcb$45&*0iyqMO@^;#L- z-z4LTuh52W6L=5e!C%LC-_dxvS)NTMO1#$terr?9bfLYC4b`#E#fh=X!_pWlatQ`Z z&Y;rxIrrH8$kN6te7Mc8TO)(>7%R~oS)~cm!u?*7<;bq)u5+M(ec~O3+eF^ z>>OY*)47)Onvmj8^tmxUOdFMG^qd8zsIks0SSBKKM->1OW)$u!G-e)D3}woGzw_!{ zxzsDtY=Fi5K=}EbK?Dg@j8I8WO*0&#rop&Gp%9%v(U*tce$~}#N-ue!p`vKK#e4Am z8_2#Jt?%|86+~*4YrcxWGF}D^0Y+&6wO@mjwi-}Q2XMmXGelcNG4MUy%q1J4&s7p! zLYIPWa3bA?Eu`3;c8-YuS5nch&?tCUF2F4uJ9K2$k<;MsIl|$`US6Y#p}pWX94m@n zpfLmcJVC`+H)V%3rqyBpbj^IjMCzkVc!w-DOl~+#u7Vf;!l>6kJZ2KmE=$%1s8Con zL2xt#Ncz~Y(+Y?g`hJ}H3zdv7Di###KJM-qy6SN*K5c`xB|3%^vn(@B0Q;1|TA4r_ zzY>$wOLQOd8Ew@FOZ8q2q!UGI4x<65wnKzpI7ckuL(}L!S^NVXr8CQ>5ZX{jGn^Q8 zjx9hPJRGhOdsnCI3k|MeYaC{`v5JkfU*ojk`%D@)k|Xffc6vLp-i_^${b|Tk+hK80 zqiOznIx-ZeLlc|v(Wm}AKr}i$H#bYX4?^67R%F5vW2MQ3{Qa2&jjiFv%(a){)ixTK z@)p3QSJ26v?clXP8h;hV$TFBx4wy{kNm}MfiGyoO4zTj45lDQD`97mh`v8%a`@sXu z=_nboGEvm}x)T-&LL_JD&21E{UXIY{MsA|9Wke&%H3%}gOYTi4LVVVDG`451g4W7h zW;@b)1(m7^%90slFjJ0oUQdk4JR@0_LAXCS$NFm62t>U6*($!H-1mDkn&6r*RVFdb zWv4bfCSph2kV?{{jxxb8AxzMzlI1f)O&=opnKng)rDk!Tr-dSy%rQ+buvjPpgP z7XofZ(Q8NJ!UVu0^EGW{Y=tO0)7brIh}dW3n?)fK2OGMwp8B%1L?C98apkWyQP_0` z*G_KvTWU4v8D-2B`NO-@kB|^CcFTR_6m$shXCw~BZX$qUO93!t=DTf7J_YAY45zCbKc}$?%#>RBq$b%^40=bQ=wvPIg))yUy6rOkaLMYsizuJv1h$ zBn9Qc_}nSd{fiho=@zyf&;@S(r9MrT`k<3JygS(vi!4Z&36On>m^)mRoy-&uucU>% zybFw1NNaQln2zDD;EDw!dgyc;5%1c&UHx}Cvl0r8at-xLr$bD{%v~|9usKN8Ku3Go z&;C40>x#dU&UaeEJw} zc@PgxZZh9iU4Xd-wMdkae2!mtop#4c%b6hK)4X(pt zoXsz0R+h;|SQ4$A(>?QoVe_91ekUx3clbu_Nu2Q#QbI8!q%4qhBw>cr$v&ek;L*4&*3UfQ&miRP5o0eep*KIk)OR>4 z#}UkmT^a9NDZYr5cQ){v+D6Kr!69TsOd}ETKI!JYudA^-05G>SP$k5|vo)kFVryk9>cJ&WO&%e$ag(X~%s(`C>86 za3jW23MSw1tcgVMrUv4C&i=_u{cc0?aDwrSQ1Z3mMNP;VyJd+ngM?g?6h*%=VWc>f z8qsSn>wit}Ke<9~JOyunVAGbq@q?8EV@2~VL>kiaMkS>7?o|uu4Q@cA3G?HcY3TD< zXYtRnqDVpvqltEYVx|W@GKNxd%7{sh)8uZf^CX}k*9qGup2u}(j1gY~TsUTAu>tx^ zWgWRmN-u*Vj@=q+rAuKk90U7JS>hZ-vfM@UP&ZpURHm%LpS7FFAH|qqY*_IVqApm; zb0fatF`rHZW4Dyhyw;qnd_m*lj7D9nVA3yM$4MiRI-38ebR%(Dc~^`2h?^Bon?cvC zJ>VEuNO%%!lpWD(LoCbLLAk?jYQV(*F_|U8Hxt`Ykv0IhIKs{N*O(zA>N+2{=AoXc zhe0rZdWnAj0K)~HBff!RFme`Q;7LT&OO7(ID4tHpsfsqhD!L|xxM72e22KWMBk4(A zd7SnM=k&!80>h2A=%eqKT9EQe^1-046NeGtdH@2W(vJH=$`MgyJy!s{Wq`o;MLAkU5u26j3ywm*>Ejt+@D)Hb*!( zYRUWO6UY&c*QR? zML&RPCXaY#67(a;YB|8mIl}iUom5ba&o#3xK1-KK@H?pCHE>RhmOCODyC%=knP5iT z34`uXyr^%dI$KOMw*#wi8;w^xZ&Ph;=v$}G##V<}x3^vU4(P+(-a@kS*e(6`dkOQE zHGr(*>l&?=#L?6U39y?6Me)DdSM(pImya=l?!{ZGv*_9!)886cZ+Qse7N^2yQnz!& zWU}@E88a8QI!UYz=qNYV*#tN592F4!c(`fQHzL;9Z$Jp=nQEZo0ZuG&W0na}S#f)? zgv%X)uW;Kz7nAV6&%P@(gf)jCQJItGDbMy|MB-)o@S61H3pKG}EEGXXW#G>9>Rw}5 zkqqV4!4_Bso4}=Cw;CWKM8)FHAf6@=jdiXkK7LBlW*{lhYBfC*L%4g(jF+eA#uHdl z)sBuWdL|AF;(b488z}bsr936wwO9_X3!lqGp0UnNdEE%wtGI5&-g0A!Q&j(UuDU zWItt#gmhts3hA{U6nuy133-flEaMZqnfD`Q^xPE=$no5w+H0hsxz+=BbjYFVAmjWlMfd0L|iTQpr ztgxB11?0%xpurg954skge1X$2bHObeU~gtxZR+>Cf5U=gN+S}lo z8#t5)nm7E%9R2?Mf$wjiEfy6|uh$=e?U5L7;K9!R(3%>UTe2{J0krV}43X1Pi7%uR5Xw_S z<$iI2F2sT;*Nr~9a6Q8EOS?j)esKZs$AC)T-Nxk{--s}6pDKUtL1^0u=+;P8zS>3v z_ogauts{cF)l1?R6WrmGvaT1ucx?e~6@tcVY}@wnAyg-qpI-RB4Z*+tO3e8_FGBma zyR)3{cOm4fZb&)b=R)wUu1<2k--O`rTA~Wvtc1}2WfJg^=e>xK@boS_d!hU&JlrQ; zdjq`eyb11CmAjiPo$vP`_+Q@uH_Rl#L(hu$?aud`5z0efY43jj=dgdd s{oiBT%lH2{zW2WlFvz0T|K~rdq<8cCh6hc+ tuple[PydanticBaseSettingsSource]: + """Pydantic settings customisation sources. + + Defines the priority order for loading configuration settings: + 1. .env file (if exists) + 2. config.yml file (if exists) + 3. Environment variables (fallback) + + Args: + settings_cls: The settings class being configured. + init_settings: Initialization settings source. + env_settings: Environment variables settings source. + dotenv_settings: .env file settings source. + file_secret_settings: File secrets settings source. + + Returns: + Tuple containing the selected settings source. + + """ + env_path = Path(__file__).parents[2] / ".env" + yaml_path = Path(__file__).parents[2] / "config.yml" + + if env_path.exists(): + return ( + DotEnvSettingsSource( + settings_cls, + env_file=env_path, + env_ignore_empty=True, + env_file_encoding="utf-8", + ), + ) + elif yaml_path.exists(): + return ( + YamlConfigSettingsSource( + settings_cls, + yaml_file=yaml_path, + yaml_file_encoding="utf-8", + ), + ) + else: + return ( + EnvSettingsSource( + settings_cls, + env_ignore_empty=True, + ), + ) + + def to_daemon_config(self) -> Configuration: + """Convert the nested configuration to the flat format expected by BaseDaemon. + + Flattens the nested configuration structure into a dictionary format + that can be consumed by the collector daemon infrastructure. + + Returns: + Dictionary with flattened configuration keys and values suitable + for daemon initialization. + + """ + return Configuration( + config_hints={ + # OpenAEV configuration (flattened) + "openaev_url": {"data": str(self.openaev.url)}, + "openaev_token": {"data": self.openaev.token}, + # Collector configuration (flattened) + "collector_id": {"data": self.collector.id}, + "collector_name": {"data": self.collector.name}, + "collector_platform": {"data": self.collector.platform}, + "collector_log_level": {"data": self.collector.log_level}, + "collector_period": { + "data": int(self.collector.period.total_seconds()) + }, # type: ignore[union-attr] + "collector_icon_filepath": {"data": self.collector.icon_filepath}, + # SentinelOne configuration (flattened) + "sentinelone_base_url": {"data": str(self.sentinelone.base_url)}, + "sentinelone_api_key": { + "data": self.sentinelone.api_key.get_secret_value() + }, + "sentinelone_time_window": {"data": self.sentinelone.time_window}, + "sentinelone_expectation_batch_size": { + "data": self.sentinelone.expectation_batch_size + }, + "sentinelone_enable_deep_visibility_search": { + "data": self.sentinelone.enable_deep_visibility_search + }, + }, + config_base_model=self, + ) diff --git a/template/src/models/configs/sentinelone_configs.py b/template/src/models/configs/sentinelone_configs.py new file mode 100644 index 00000000..904fff71 --- /dev/null +++ b/template/src/models/configs/sentinelone_configs.py @@ -0,0 +1,39 @@ +"""Configuration for SentinelOne integration.""" + +from datetime import timedelta + +from pydantic import Field, SecretStr +from src.models.configs import ConfigBaseSettings + + +class _ConfigLoaderSentinelOne(ConfigBaseSettings): + """SentinelOne API configuration settings. + + Contains connection details, timing parameters, and retry settings + for SentinelOne API integration. + """ + + base_url: str | None = Field( + alias="SENTINELONE_BASE_URL", + default="https://api.sentinelone.com", + description="URL for the SentinelOne API.", + ) + api_key: SecretStr = Field( + alias="SENTINELONE_API_KEY", + description="API Key for the SentinelOne API.", + ) + time_window: timedelta = Field( + alias="SENTINELONE_TIME_WINDOW", + default=timedelta(hours=1), + description="Time window for SentinelOne threat searches when no date signatures are provided (ISO 8601 format).", + ) + expectation_batch_size: int = Field( + alias="SENTINELONE_EXPECTATION_BATCH_SIZE", + default=50, + description="Number of expectations to process in each batch for batch-based processing.", + ) + enable_deep_visibility_search: bool = Field( + alias="SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH", + default=False, + description="Enable deep visibility search for SentinelOne threat searches.", + ) diff --git a/template/src/py.typed b/template/src/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/template/src/services/client_api.py b/template/src/services/client_api.py new file mode 100644 index 00000000..dfc4aa7d --- /dev/null +++ b/template/src/services/client_api.py @@ -0,0 +1,71 @@ +"""SentinelOne API client for session management and core HTTP functionality.""" + +import logging +from datetime import timedelta + +import requests # type: ignore[import-untyped] + +from ..models.configs.config_loader import ConfigLoader +from .exception import SentinelOneSessionError + +LOG_PREFIX = "[SentinelOneClientAPI]" + + +class SentinelOneClientAPI: + """SentinelOne API client for managing HTTP sessions and core functionality.""" + + def __init__(self, config: ConfigLoader) -> None: + """Initialize SentinelOne API client. + + Args: + config: Configuration loader with SentinelOne settings. + + Raises: + SentinelOneValidationError: If configuration is invalid. + SentinelOneSessionError: If session setup fails. + + """ + self.logger: logging.Logger = logging.getLogger(__name__) + self.config: ConfigLoader = config + + self.base_url: str = str(config.sentinelone.base_url).rstrip("/") + self.api_key: str = config.sentinelone.api_key.get_secret_value() + + self.time_window: timedelta = config.sentinelone.time_window + + try: + self.session: requests.Session = self._create_session() + except Exception as e: + raise SentinelOneSessionError(f"Failed to create session: {e}") from e + + self.logger.debug( + f"{LOG_PREFIX} Initializing SentinelOne API client components..." + ) + + self.logger.info( + f"{LOG_PREFIX} SentinelOne API client initialized successfully" + ) + + def _create_session(self) -> requests.Session: + """Create and configure HTTP session for SentinelOne API. + + Returns: + Configured requests Session object. + + Raises: + SentinelOneSessionError: If session configuration fails. + + """ + try: + session = requests.Session() + session.headers.update( + { + "Authorization": f"ApiToken {self.api_key}", + "Content-Type": "application/json", + "Accept": "application/json", + } + ) + + return session + except Exception as e: + raise SentinelOneSessionError(f"Failed to configure session: {e}") from e diff --git a/template/src/services/converter.py b/template/src/services/converter.py new file mode 100644 index 00000000..b6c77d19 --- /dev/null +++ b/template/src/services/converter.py @@ -0,0 +1,117 @@ +"""SentinelOne Data Converter to OAEV format.""" + +import logging +from typing import Any + +from .exception import SentinelOneDataConversionError, SentinelOneValidationError +from .model_threat import SentinelOneThreat + +LOG_PREFIX = "[SentinelOneConverter]" + + +class SentinelOneConverter: + """Converter for SentinelOne threat data to OAEV format.""" + + def __init__(self) -> None: + """Initialize the SentinelOne data converter.""" + self.logger = logging.getLogger(__name__) + self.logger.debug(f"{LOG_PREFIX} SentinelOne converter initialized") + + def convert_threats_to_oaev( + self, threats: list[SentinelOneThreat] + ) -> list[dict[str, Any]]: + """Convert SentinelOne threat data to OAEV format. + + Args: + threats: List of SentinelOneThreat objects. + + Returns: + List of OAEV data dictionaries. + + Raises: + SentinelOneValidationError: If data format is invalid. + SentinelOneDataConversionError: If conversion fails. + + """ + if not threats: + self.logger.debug(f"{LOG_PREFIX} No threats to convert") + return [] + + if not isinstance(threats, list): + raise SentinelOneValidationError("threats must be a list") + + try: + self.logger.debug( + f"{LOG_PREFIX} Converting {len(threats)} threats to OAEV format" + ) + + oaev_data_list = [] + converted_count = 0 + + for i, threat in enumerate(threats, 1): + if not isinstance(threat, SentinelOneThreat): + self.logger.warning( + f"{LOG_PREFIX} Item {i} is not a SentinelOneThreat: {type(threat)}" + ) + continue + + try: + oaev_data = self._convert_threat_to_oaev(threat) + if oaev_data: + oaev_data_list.append(oaev_data) + converted_count += 1 + self.logger.debug( + f"{LOG_PREFIX} Converted threat {i}/{len(threats)}: {threat.threat_id}" + ) + except Exception as e: + self.logger.warning( + f"{LOG_PREFIX} Failed to convert threat {i}: {e}" + ) + + self.logger.info( + f"{LOG_PREFIX} Conversion completed: {converted_count} threats -> {len(oaev_data_list)} OAEV items" + ) + return oaev_data_list + + except Exception as e: + raise SentinelOneDataConversionError( + f"Failed to convert threats to OAEV format: {e}" + ) from e + + def _convert_threat_to_oaev(self, threat: SentinelOneThreat) -> dict[str, Any]: + """Convert a single threat to OAEV format. + + Args: + threat: SentinelOneThreat object to convert. + + Returns: + OAEV formatted data dictionary. + + Raises: + SentinelOneValidationError: If threat data is invalid. + + """ + if not threat.threat_id: + raise SentinelOneValidationError("Threat must have a threat_id") + + try: + oaev_data = { + "threat_id": {"type": "fuzzy", "data": [threat.threat_id], "score": 95} + } + + if threat.hostname: + oaev_data["target_hostname_address"] = { + "type": "fuzzy", + "data": [threat.hostname], + "score": 95, + } + + self.logger.debug( + f"{LOG_PREFIX} Successfully converted threat {threat.threat_id} to OAEV format" + ) + return oaev_data + + except Exception as e: + raise SentinelOneDataConversionError( + f"Error converting threat {threat.threat_id} to OAEV: {e}" + ) from e diff --git a/template/src/services/exception.py b/template/src/services/exception.py new file mode 100644 index 00000000..6593a8b6 --- /dev/null +++ b/template/src/services/exception.py @@ -0,0 +1,43 @@ +"""SentinelOne Service Exceptions.""" + + +class SentinelOneServiceError(Exception): + """Base exception for all SentinelOne service errors.""" + + pass + + +class SentinelOneExpectationError(SentinelOneServiceError): + """Raised when there's an error processing expectations.""" + + pass + + +class SentinelOneDataConversionError(SentinelOneServiceError): + """Raised when there's an error converting data.""" + + pass + + +class SentinelOneAPIError(SentinelOneServiceError): + """Raised when there's an error with SentinelOne API operations.""" + + pass + + +class SentinelOneNetworkError(SentinelOneServiceError): + """Raised when there's a network connectivity error.""" + + pass + + +class SentinelOneSessionError(SentinelOneServiceError): + """Raised when there's an error with session management.""" + + pass + + +class SentinelOneValidationError(SentinelOneServiceError): + """Raised when input validation fails.""" + + pass diff --git a/template/src/services/expectation_service.py b/template/src/services/expectation_service.py new file mode 100644 index 00000000..320e3191 --- /dev/null +++ b/template/src/services/expectation_service.py @@ -0,0 +1,703 @@ +"""SentinelOne Expectation Service with batch-based processing.""" + +import logging +from datetime import datetime, timezone +from typing import Any + +from pydantic import BaseModel, Field +from pyoaev.apis.inject_expectation.model.expectation import ( + DetectionExpectation, + PreventionExpectation, +) +from pyoaev.signatures.types import SignatureTypes + +from .client_api import SentinelOneClientAPI +from .converter import SentinelOneConverter +from .exception import SentinelOneAPIError, SentinelOneExpectationError +from .fetcher_deep_visibility import FetcherDeepVisibility +from .fetcher_threat import FetcherThreat +from .fetcher_threat_events import FetcherThreatEvents +from .model_threat import SentinelOneThreat +from .utils import SignatureExtractor, TraceBuilder + +LOG_PREFIX = "[SentinelOneExpectationService]" + + +class ExpectationResult(BaseModel): + """Model for expectation processing results.""" + + expectation_id: str = Field(..., description="ID of the processed expectation") + is_valid: bool = Field(..., description="Whether the expectation was validated") + expectation: Any | None = Field(None, description="The original expectation object") + matched_alerts: list[dict[str, Any]] | None = Field( + None, description="List of alerts that matched this expectation" + ) + error_message: str | None = Field( + None, description="Error message if processing failed" + ) + processing_time: float | None = Field( + None, description="Time taken to process this expectation in seconds" + ) + + +class SentinelOneExpectationService: + """Service for processing SentinelOne expectations in batches.""" + + def __init__( + self, + config: Any | None = None, + ) -> None: + """Initialize the SentinelOne expectation service. + + Args: + config: Configuration loader for alternative initialization. + + Raises: + SentinelOneValidationError: If required parameters are None. + + """ + self.logger: logging.Logger = logging.getLogger(__name__) + + self.client_api: SentinelOneClientAPI = SentinelOneClientAPI(config) + self.converter: SentinelOneConverter = SentinelOneConverter() + self.batch_size: int = config.sentinelone.expectation_batch_size + self.enable_deep_visibility_search = ( + config.sentinelone.enable_deep_visibility_search + ) + + self.threat_fetcher: FetcherThreat = FetcherThreat(self.client_api) + self.threat_events_fetcher: FetcherThreatEvents = FetcherThreatEvents( + self.client_api + ) + self.deep_visibility_fetcher: FetcherDeepVisibility = FetcherDeepVisibility( + self.client_api + ) + + self.logger.info( + f"{LOG_PREFIX} Service initialized with batch size: {self.batch_size}" + ) + + def get_supported_signatures(self) -> list[SignatureTypes]: + """Get list of supported signature types. + + Returns: + List of supported SignatureTypes enum values. + + """ + return [ + SignatureTypes.SIG_TYPE_PARENT_PROCESS_NAME, + SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, + SignatureTypes.SIG_TYPE_END_DATE, + ] + + def handle_batch_expectations( + self, + expectations: list[DetectionExpectation | PreventionExpectation], + detection_helper: Any, + ) -> tuple[list[ExpectationResult], int]: + """Handle a batch of expectations. + + Args: + expectations: List of expectations to process. + detection_helper: OpenAEV detection helper instance. + + Returns: + Tuple of (results, skipped_count) where: + - results: List of ExpectationResult objects for processed expectations + - skipped_count: Number of expectations skipped due to missing end_date + + Raises: + SentinelOneExpectationError: If batch processing fails. + + """ + if not expectations: + self.logger.info(f"{LOG_PREFIX} No expectations to process") + return [], 0 + + try: + self.logger.info( + f"{LOG_PREFIX} Starting new batch processing of {len(expectations)} expectations" + ) + + batches, skipped_count = self._create_expectation_batches(expectations) + self.logger.info( + f"{LOG_PREFIX} Created {len(batches)} batches of size {self.batch_size} (skipped {skipped_count} expectations without end_date)" + ) + + all_results = [] + + for batch_idx, batch in enumerate(batches, 1): + self.logger.info( + f"{LOG_PREFIX} Processing batch {batch_idx}/{len(batches)} with {len(batch)} expectations" + ) + + try: + batch_results = self._process_expectation_batch( + batch, detection_helper, batch_idx + ) + all_results.extend(batch_results) + + self.logger.info( + f"{LOG_PREFIX} Batch {batch_idx} completed: {len(batch_results)} results" + ) + except Exception as e: + self.logger.error( + f"{LOG_PREFIX} Error processing batch {batch_idx}: {e}" + ) + error_results = [ + self._create_error_result_object( + SentinelOneExpectationError(f"Batch processing error: {e}"), + expectation, + ) + for expectation in batch + ] + all_results.extend(error_results) + + valid_count = sum(1 for r in all_results if r.is_valid) + invalid_count = len(all_results) - valid_count + + self.logger.info( + f"{LOG_PREFIX} New batch processing completed: {valid_count} valid, {invalid_count} invalid, {skipped_count} skipped (no end_date)" + ) + + return all_results, skipped_count + + except Exception as e: + raise SentinelOneExpectationError( + f"Error in handle_batch_expectations: {e}" + ) from e + + def _create_expectation_batches( + self, expectations: list[DetectionExpectation | PreventionExpectation] + ) -> tuple[list[list[DetectionExpectation | PreventionExpectation]], int]: + """Group expectations into batches, filtering out those without end_date. + + Args: + expectations: List of expectations to batch. + + Returns: + Tuple of (batches, skipped_count) where: + - batches: List of expectation batches that have end_date signatures + - skipped_count: Number of expectations skipped due to missing end_date + + """ + valid_expectations = [] + skipped_count = 0 + + for expectation in expectations: + has_end_date = ( + SignatureExtractor.extract_end_date([expectation]) is not None + ) + + if has_end_date: + valid_expectations.append(expectation) + else: + skipped_count += 1 + self.logger.debug( + f"{LOG_PREFIX} Skipping expectation {expectation.inject_expectation_id} - no end_date signature found" + ) + + if skipped_count > 0: + self.logger.info( + f"{LOG_PREFIX} Filtered out {skipped_count} expectations without end_date signatures" + ) + + batches = [] + for i in range(0, len(valid_expectations), self.batch_size): + batch = valid_expectations[i : i + self.batch_size] + batches.append(batch) + + self.logger.debug( + f"{LOG_PREFIX} Created {len(batches)} batches from {len(valid_expectations)} valid expectations (skipped {skipped_count})" + ) + return batches, skipped_count + + def _process_expectation_batch( + self, + batch: list[DetectionExpectation | PreventionExpectation], + detection_helper: Any, + batch_idx: int, + ) -> list[ExpectationResult]: + """Process a single batch of expectations. + + Args: + batch: Batch of expectations to process. + detection_helper: OpenAEV detection helper. + batch_idx: Batch index for logging. + + Returns: + List of ExpectationResult objects for this batch. + + """ + try: + process_names = self._extract_process_names_from_batch(batch) + + self.logger.debug( + f"{LOG_PREFIX} Batch {batch_idx}: Found {len(process_names)} unique process names" + ) + + threats = self._fetch_threats_for_time_window(batch) + self.logger.info( + f"{LOG_PREFIX} Batch {batch_idx}: Fetched {len(threats)} threats from time window" + ) + + static_threats = [threat for threat in threats if threat.is_static] + non_static_threats = [threat for threat in threats if not threat.is_static] + + self.logger.debug( + f"{LOG_PREFIX} Batch {batch_idx}: Processing {len(static_threats)} static threats " + f"and {len(non_static_threats)} non-static threats" + ) + + threat_events = {} + for threat in non_static_threats: + try: + events = self.threat_events_fetcher.fetch_events_for_threat( + threat, process_names + ) + if events: + threat_events[threat.threat_id] = events + self.logger.debug( + f"{LOG_PREFIX} Batch {batch_idx}: Non-static threat {threat.threat_id} has {len(events)} threat events" + ) + else: + self.logger.debug( + f"{LOG_PREFIX} Batch {batch_idx}: Non-static threat {threat.threat_id} - no threat events found" + ) + except Exception as e: + self.logger.error( + f"{LOG_PREFIX} Batch {batch_idx}: Error fetching threat events for non-static threat {threat.threat_id}: {e}" + ) + + if static_threats and self.enable_deep_visibility_search: + try: + sha1_to_threat = {} + unique_sha1s = [] + + for threat in static_threats: + if threat.sha1: + if threat.sha1 not in sha1_to_threat: + sha1_to_threat[threat.sha1] = threat + unique_sha1s.append(threat.sha1) + else: + self.logger.debug( + f"{LOG_PREFIX} Batch {batch_idx}: Static threat {threat.threat_id} - no SHA1 available" + ) + + if unique_sha1s: + end_time = self._extract_end_date_from_batch(batch) + if end_time is None: + end_time = datetime.now(timezone.utc) + start_time = end_time - self.client_api.time_window + + self.logger.debug( + f"{LOG_PREFIX} Batch {batch_idx}: Fetching DV events for {len(unique_sha1s)} unique SHA1s (from {len(static_threats)} static threats) in single query for time window: {start_time} to {end_time}" + ) + + sha1_to_events = ( + self.deep_visibility_fetcher.fetch_events_for_batch_sha1( + unique_sha1s, start_time, end_time + ) + ) + + for sha1, events in sha1_to_events.items(): + if sha1 in sha1_to_threat: + threat = sha1_to_threat[sha1] + if events: + threat_events[threat.threat_id] = events + self.logger.debug( + f"{LOG_PREFIX} Batch {batch_idx}: Static threat {threat.threat_id} has {len(events)} DV events" + ) + else: + self.logger.debug( + f"{LOG_PREFIX} Batch {batch_idx}: Static threat {threat.threat_id} - no DV events found" + ) + + self.logger.info( + f"{LOG_PREFIX} Batch {batch_idx}: Processed {len(static_threats)} static threats with single DV query for {len(unique_sha1s)} unique SHA1s" + ) + else: + self.logger.debug( + f"{LOG_PREFIX} Batch {batch_idx}: No valid SHA1s found for static threats" + ) + + except Exception as e: + self.logger.error( + f"{LOG_PREFIX} Batch {batch_idx}: Error fetching DV events for static threats batch: {e}" + ) + + results = self._match_threats_to_expectations( + batch, threats, threat_events, detection_helper + ) + + return results + + except Exception as e: + raise SentinelOneExpectationError( + f"Error processing batch {batch_idx}: {e}" + ) from e + + def _extract_hostnames_from_batch( + self, batch: list[DetectionExpectation | PreventionExpectation] + ) -> list[str]: + """Extract unique hostnames from a batch of expectations. + + Args: + batch: Batch of expectations. + + Returns: + List of unique hostnames. + + """ + return SignatureExtractor.extract_hostnames(batch) + + def _extract_process_names_from_batch( + self, batch: list[DetectionExpectation | PreventionExpectation] + ) -> list[str]: + """Extract unique parent process names from a batch of expectations. + + Args: + batch: Batch of expectations. + + Returns: + List of unique parent process names. + + """ + return SignatureExtractor.extract_process_names(batch) + + def _extract_end_date_from_batch( + self, batch: list[DetectionExpectation | PreventionExpectation] | None = None + ) -> datetime | None: + """Extract end_date from batch signatures. + + Args: + batch: Batch of expectations to extract end_date from. + + Returns: + end_date as datetime or None if no end_date signature found. + + """ + end_date = SignatureExtractor.extract_end_date(batch) + if end_date: + self.logger.debug( + f"{LOG_PREFIX} Extracted end_date from signatures: {end_date}, start_date will be calculated from time_window" + ) + return end_date + + def _fetch_threats_for_time_window( + self, batch: list[DetectionExpectation | PreventionExpectation] | None = None + ) -> list[SentinelOneThreat]: + """Fetch all threats from the configured time window or date signatures. + + Args: + batch: Optional batch of expectations to extract date filters from. + + Returns: + List of SentinelOneThreat objects from the time window. + + Raises: + SentinelOneAPIError: If API call fails. + + """ + try: + end_time = self._extract_end_date_from_batch(batch) + + if end_time is None: + end_time = datetime.now(timezone.utc) + + start_time = end_time - self.client_api.time_window + + self.logger.debug( + f"{LOG_PREFIX} Delegating threat fetching to FetcherThreat for time window: {start_time} to {end_time}" + ) + + return self.threat_fetcher.fetch_threats_for_time_window( + start_time=start_time, + end_time=end_time, + limit=1000, + ) + + except Exception as e: + raise SentinelOneAPIError( + f"Error fetching threats for time window: {e}" + ) from e + + def _match_threats_to_expectations( + self, + batch: list[DetectionExpectation | PreventionExpectation], + threats: list[SentinelOneThreat], + threat_events: dict[str, list[dict[str, Any]]], + detection_helper: Any, + ) -> list[ExpectationResult]: + """Match threats and events to expectations and create results. + + Args: + batch: Batch of expectations. + threats: List of filtered threats. + threat_events: Dictionary mapping threat IDs to their events. + detection_helper: OpenAEV detection helper. + + Returns: + List of ExpectationResult objects. + + """ + results = [] + + for expectation in batch: + try: + matched = False + traces = [] + + for threat in threats: + events = threat_events.get(threat.threat_id, []) + + if self._expectation_matches_threat_data( + expectation, threat, events, detection_helper + ): + base_url = self.client_api.base_url if self.client_api else "" + trace = TraceBuilder.create_threat_trace( + threat, base_url, events + ) + traces.append(trace) + + if isinstance(expectation, PreventionExpectation): + # breakpoint() + if threat.is_mitigated: + matched = True + self.logger.debug( + f"{LOG_PREFIX} Prevention expectation {expectation.inject_expectation_id}: " + f"threat {threat.threat_id} matched signature and is mitigated -> expectation satisfied" + ) + break + self.logger.debug( + f"{LOG_PREFIX} Prevention expectation {expectation.inject_expectation_id}: " + f"threat {threat.threat_id} matched signature but not mitigated -> continuing search" + ) + else: + matched = True + self.logger.debug( + f"{LOG_PREFIX} Detection expectation {expectation.inject_expectation_id}: " + f"threat {threat.threat_id} matched signature -> expectation satisfied" + ) + break + + result_dict = { + "is_valid": matched, + "traces": traces, + "expectation_type": ( + "detection" + if isinstance(expectation, DetectionExpectation) + else "prevention" + ), + } + + result = self._convert_dict_to_result(result_dict, expectation) + results.append(result) + + self.logger.debug( + f"{LOG_PREFIX} Expectation {expectation.inject_expectation_id}: " + f"matched={matched}, traces={len(traces)}" + ) + + except Exception as e: + self.logger.error( + f"{LOG_PREFIX} Error matching expectation {expectation.inject_expectation_id}: {e}" + ) + error_result = self._create_error_result_object( + SentinelOneExpectationError(f"Matching error: {e}"), expectation + ) + results.append(error_result) + + return results + + def _expectation_matches_threat_data( + self, + expectation: DetectionExpectation | PreventionExpectation, + threat: SentinelOneThreat, + events: list[dict[str, Any]], + detection_helper: Any, + ) -> bool: + """Check if an expectation matches the given threat and events using converter and detection helper. + + Args: + expectation: The expectation to match. + threat: The threat data. + events: List of events for the threat. + detection_helper: OpenAEV detection helper for matching. + + Returns: + True if the expectation matches, False otherwise. + + """ + try: + oaev_data_list = self.converter.convert_threats_to_oaev([threat]) + + if not oaev_data_list: + self.logger.debug( + f"{LOG_PREFIX} No OAEV data generated for threat {threat.threat_id}" + ) + return False + + oaev_data = oaev_data_list[0] + + if events: + if threat.is_static: + parent_process_names = ( + SentinelOneThreat.get_parent_process_name_from_dv_events(events) + ) + else: + parent_process_names = ( + SentinelOneThreat.get_parent_process_name_from_events(events) + ) + oaev_implant_names = [ + name + for name in parent_process_names + if name.startswith("oaev-implant-") + ] + + if threat.is_static: + event_source = "DV events (parentProcessName + processName)" + else: + event_source = "threat events" + + self.logger.debug( + f"{LOG_PREFIX} Threat {threat.threat_id}: Found {len(parent_process_names)} " + f"process names from {event_source}, {len(oaev_implant_names)} with oaev-implant- prefix" + ) + + if oaev_implant_names: + # breakpoint() + oaev_data["parent_process_name"] = { + "type": "fuzzy", + "data": oaev_implant_names, + "score": 95, + } + self.logger.debug( + f"{LOG_PREFIX} Added oaev-implant parent processes to OAEV for {threat.threat_id}: {oaev_implant_names}" + ) + + supported_signatures = self.get_supported_signatures() + self.logger.debug( + f"{LOG_PREFIX} Supported signature types: {[s.value for s in supported_signatures]}" + ) + + signature_groups = SignatureExtractor.group_signatures_by_type( + expectation, supported_signatures + ) + self.logger.debug( + f"{LOG_PREFIX} Filtered signature groups: {list(signature_groups.keys())}" + ) + + supported_sig_names = { + sig_type.value if hasattr(sig_type, "value") else str(sig_type) + for sig_type in supported_signatures + } + filtered_oaev_data = { + key: value + for key, value in oaev_data.items() + if key in supported_sig_names + } + self.logger.debug( + f"{LOG_PREFIX} Available OAEV data: {list(oaev_data.keys())}" + ) + self.logger.debug( + f"{LOG_PREFIX} Filtered OAEV data: {list(filtered_oaev_data.keys())}" + ) + + for sig_type, signatures in signature_groups.items(): + filtered_data = {sig_type: filtered_oaev_data[sig_type]} + self.logger.debug( + f"{LOG_PREFIX} Detection helper input - sig_type: {sig_type}" + ) + self.logger.debug( + f"{LOG_PREFIX} Detection helper input - signatures: {signatures}" + ) + self.logger.debug( + f"{LOG_PREFIX} Detection helper input - filtered_data: {filtered_data}" + ) + + # breakpoint() + match_result = detection_helper.match_alert_elements( + signatures, filtered_data + ) + + self.logger.debug( + f"{LOG_PREFIX} Detection helper result for {sig_type}: {match_result}" + ) + + if not match_result: + self.logger.debug( + f"{LOG_PREFIX} {sig_type} signature failed for threat {threat.threat_id}" + ) + return False + + self.logger.debug( + f"{LOG_PREFIX} All signatures matched for expectation {expectation.inject_expectation_id} vs threat {threat.threat_id}" + ) + return True + + except Exception as e: + self.logger.warning(f"{LOG_PREFIX} Error in expectation matching: {e}") + return False + + def _create_error_result_object( + self, + error: Exception, + expectation: DetectionExpectation | PreventionExpectation, + ) -> ExpectationResult: + """Create an error result object. + + Args: + error: The error that occurred. + expectation: The expectation that failed. + + Returns: + ExpectationResult object representing the error. + + """ + return ExpectationResult( + expectation_id=str(expectation.inject_expectation_id), + is_valid=False, + expectation=expectation, + matched_alerts=None, + error_message=str(error), + processing_time=None, + ) + + def _convert_dict_to_result( + self, + result_dict: dict[str, Any], + expectation: DetectionExpectation | PreventionExpectation, + ) -> ExpectationResult: + """Convert result dictionary to ExpectationResult object. + + Args: + result_dict: Dictionary containing result data. + expectation: The associated expectation. + + Returns: + ExpectationResult object. + + """ + return ExpectationResult( + expectation_id=str(expectation.inject_expectation_id), + is_valid=result_dict.get("is_valid", False), + expectation=expectation, + matched_alerts=result_dict.get("traces", []), + error_message=result_dict.get("error"), + processing_time=None, + ) + + def get_service_info(self) -> dict[str, Any]: + """Get service information. + + Returns: + Dictionary containing service information. + + """ + return { + "service_name": "SentinelOneExpectationService", + "batch_size": self.batch_size, + "supported_signatures": self.get_supported_signatures(), + "flow_type": "batch_based", + } diff --git a/template/src/services/fetcher_deep_visibility.py b/template/src/services/fetcher_deep_visibility.py new file mode 100644 index 00000000..ef8a6060 --- /dev/null +++ b/template/src/services/fetcher_deep_visibility.py @@ -0,0 +1,533 @@ +"""SentinelOne Deep Visibility Fetcher for static threat analysis.""" + +import logging +import re +import time +from datetime import datetime, timedelta, timezone +from typing import Any + +from requests import ConnectionError, RequestException, Timeout + +from .client_api import SentinelOneClientAPI +from .exception import ( + SentinelOneAPIError, + SentinelOneNetworkError, + SentinelOneValidationError, +) + +LOG_PREFIX = "[FetcherDeepVisibility]" +REQUEST_TIMEOUT_SECONDS = 30 +MAX_STATUS_POLL_ATTEMPTS = 30 + + +class FetcherDeepVisibility: + """Fetcher for SentinelOne Deep Visibility data for static threats.""" + + def __init__(self, client_api: SentinelOneClientAPI): + """Initialize the Deep Visibility fetcher. + + Args: + client_api: SentinelOne API client instance. + + """ + self.client_api = client_api + self.logger = logging.getLogger(__name__) + + def fetch_events_for_sha1( + self, sha1: str, start_time: datetime = None, end_time: datetime = None + ) -> list[dict[str, Any]]: + """Fetch Deep Visibility events for a specific SHA1. + + Args: + sha1: SHA1 hash to search for. + start_time: Start time for the search (optional). + end_time: End time for the search (optional). + + Returns: + List of event dictionaries compatible with threat events. + + Raises: + SentinelOneValidationError: If SHA1 is invalid. + SentinelOneAPIError: If API call fails. + SentinelOneNetworkError: If network error occurs. + + """ + if not sha1 or not isinstance(sha1, str): + raise SentinelOneValidationError("SHA1 must be a non-empty string") + + try: + self.logger.debug(f"{LOG_PREFIX} Fetching DV events for SHA1: {sha1}") + + query_response = self._init_dv_query([sha1], start_time, end_time) + + all_events = self._execute_query(query_response) + + events = [event for event in all_events if event.get("fileSha1") == sha1] + + self.logger.info( + f"{LOG_PREFIX} Successfully fetched {len(events)} DV events for SHA1: {sha1}" + ) + return events + + except ( + SentinelOneValidationError, + SentinelOneAPIError, + SentinelOneNetworkError, + ): + raise + except Exception as e: + raise SentinelOneAPIError( + f"Unexpected error fetching DV events for SHA1 {sha1}: {e}" + ) from e + + def fetch_events_for_batch_sha1( + self, + sha1_list: list[str], + start_time: datetime = None, + end_time: datetime = None, + ) -> dict[str, list[dict[str, Any]]]: + """Fetch Deep Visibility events for multiple SHA1s in a single query. + + Args: + sha1_list: List of SHA1 hashes to search for. + start_time: Start time for the search (optional). + end_time: End time for the search (optional). + + Returns: + Dictionary mapping SHA1 to list of event dictionaries. + + Raises: + SentinelOneValidationError: If SHA1 list is invalid. + SentinelOneAPIError: If API call fails. + SentinelOneNetworkError: If network error occurs. + + """ + if not sha1_list or not isinstance(sha1_list, list): + raise SentinelOneValidationError("sha1_list must be a non-empty list") + + valid_sha1s = [sha1 for sha1 in sha1_list if sha1 and isinstance(sha1, str)] + + if not valid_sha1s: + self.logger.debug(f"{LOG_PREFIX} No valid SHA1s provided") + return {} + + try: + self.logger.debug( + f"{LOG_PREFIX} Fetching DV events for {len(valid_sha1s)} SHA1s in batch" + ) + + query_response = self._init_dv_query(valid_sha1s, start_time, end_time) + + all_events = self._execute_query(query_response) + + sha1_to_events = {} + for sha1 in valid_sha1s: + sha1_to_events[sha1] = [] + + for event in all_events: + file_sha1 = event.get("fileSha1") + if file_sha1 in sha1_to_events: + sha1_to_events[file_sha1].append(event) + + total_events = sum(len(events) for events in sha1_to_events.values()) + self.logger.info( + f"{LOG_PREFIX} Successfully fetched {total_events} total DV events for {len(valid_sha1s)} SHA1s" + ) + return sha1_to_events + + except ( + SentinelOneValidationError, + SentinelOneAPIError, + SentinelOneNetworkError, + ): + raise + except Exception as e: + raise SentinelOneAPIError( + f"Unexpected error fetching DV events for batch SHA1s: {e}" + ) from e + + def _init_dv_query( + self, + sha1_list: list[str], + start_time: datetime = None, + end_time: datetime = None, + ) -> Any: + """Initialize Deep Visibility query for SHA1s. + + Args: + sha1_list: List of SHA1 hashes to search for. + start_time: Start time for the search (optional). + end_time: End time for the search (optional). + + Returns: + Query response object. + + Raises: + SentinelOneAPIError: If API call fails. + SentinelOneNetworkError: If network error occurs. + + """ + try: + endpoint = f"{self.client_api.base_url}/web/api/v2.1/dv/init-query" + + if len(sha1_list) == 1: + query_string = f'tgtFileSha1 = "{sha1_list[0]}"' + else: + sha1_values = '","'.join(sha1_list) + query_string = f'tgtFileSha1 in ("{sha1_values}")' + + if end_time is None: + end_time = datetime.now(timezone.utc) + if start_time is None: + start_time = end_time - self.client_api.time_window + + body = { + "query": query_string, + "fromDate": self._format_timestamp_for_api(start_time), + "toDate": self._format_timestamp_for_api(end_time), + } + + self.logger.debug(f"{LOG_PREFIX} Making POST request to: {endpoint}") + self.logger.debug(f"{LOG_PREFIX} DV Query: {query_string}") + self.logger.debug(f"{LOG_PREFIX} Full body payload: {body}") + + response = self.client_api.session.post( + endpoint, json=body, timeout=REQUEST_TIMEOUT_SECONDS + ) + + if response.status_code == 200: + json_data = response.json() + + class InitQueryResponse: + def __init__(self, data: dict): + self.data = InitData(data.get("data", {})) + + class InitData: + def __init__(self, data: dict): + self.query_id = data.get("queryId") + + return InitQueryResponse(json_data) + else: + error_detail = self._parse_error_response(response) + + retention_days = self._extract_retention_days(error_detail) + if retention_days and len(sha1_list) > 0: + self.logger.warning( + f"{LOG_PREFIX} DV retention limit ({retention_days} days) exceeded. Adjusting time window and retrying..." + ) + + self.logger.info( + f"{LOG_PREFIX} Waiting 60 seconds due to DV API rate limit before retry..." + ) + time.sleep(60) + + adjusted_end_time = end_time or datetime.now(timezone.utc) + adjusted_start_time = adjusted_end_time - timedelta( + days=retention_days - 1 + ) + + self.logger.debug( + f"{LOG_PREFIX} Retrying with adjusted time window: {adjusted_start_time} to {adjusted_end_time}" + ) + + return self._init_dv_query( + sha1_list, adjusted_start_time, adjusted_end_time + ) + + raise SentinelOneAPIError( + f"DV init query failed with status {response.status_code}: {error_detail}" + ) + + except SentinelOneAPIError: + raise + except (ConnectionError, Timeout) as e: + raise SentinelOneNetworkError( + f"Network error making DV init query: {e}" + ) from e + except RequestException as e: + raise SentinelOneAPIError( + f"HTTP request failed for DV init query: {e}" + ) from e + except Exception as e: + raise SentinelOneAPIError( + f"Unexpected error making DV init query: {e}" + ) from e + + def _execute_query(self, query_response: Any) -> list[dict[str, Any]]: + """Execute the Deep Visibility query. + + Args: + query_response: Response from query initialization. + + Returns: + List of event dictionaries. + + Raises: + SentinelOneValidationError: If query response is invalid. + + """ + if not query_response or not hasattr(query_response, "data"): + raise SentinelOneValidationError("Invalid query response, cannot execute") + + query_id = query_response.data.query_id + if not query_id: + raise SentinelOneValidationError("No query ID available, cannot execute") + + try: + self.logger.debug(f"{LOG_PREFIX} Executing DV query with ID: {query_id}") + + self._wait_for_query_completion(query_id) + + return self._make_real_events_query(query_id) + + except ( + SentinelOneValidationError, + SentinelOneAPIError, + SentinelOneNetworkError, + ): + raise + except Exception as e: + raise SentinelOneAPIError(f"Error executing DV query: {e}") from e + + def _wait_for_query_completion(self, query_id: str) -> None: + """Wait for DV query to complete processing. + + Args: + query_id: Query identifier to check status for. + + Raises: + SentinelOneAPIError: If API call fails. + SentinelOneNetworkError: If network error occurs. + + """ + if not query_id: + raise SentinelOneValidationError("query_id cannot be empty") + + attempt = 0 + while attempt < MAX_STATUS_POLL_ATTEMPTS: + try: + endpoint = f"{self.client_api.base_url}/web/api/v2.1/dv/query-status" + params = {"queryId": query_id} + + self.logger.debug( + f"{LOG_PREFIX} Checking query status for ID: {query_id}" + ) + + response = self.client_api.session.get( + endpoint, params=params, timeout=REQUEST_TIMEOUT_SECONDS + ) + + if response.status_code == 200: + json_data = response.json() + data = json_data.get("data", {}) + + progress_status = data.get("progressStatus", 0) + response_state = data.get("responseState", "") + + self.logger.debug( + f"{LOG_PREFIX} Query status: {response_state}, Progress: {progress_status}%" + ) + + if response_state == "FINISHED" or progress_status >= 100: + self.logger.info( + f"{LOG_PREFIX} Query {query_id} completed (Status: {response_state}, Progress: {progress_status}%)" + ) + return + + wait_time = self._calculate_wait_time(progress_status, attempt) + + self.logger.debug( + f"{LOG_PREFIX} Query still processing (Progress: {progress_status}%), waiting {wait_time}s before next check" + ) + time.sleep(wait_time) + + attempt += 1 + else: + error_detail = self._parse_error_response(response) + raise SentinelOneAPIError( + f"Query status check failed with status {response.status_code}: {error_detail}" + ) + + except (SentinelOneValidationError, SentinelOneAPIError): + raise + except (ConnectionError, Timeout) as e: + raise SentinelOneNetworkError( + f"Network error checking query status: {e}" + ) from e + except RequestException as e: + raise SentinelOneAPIError( + f"HTTP request failed for query status: {e}" + ) from e + except Exception as e: + raise SentinelOneAPIError( + f"Unexpected error checking query status: {e}" + ) from e + + raise SentinelOneAPIError( + f"Query {query_id} did not complete within {MAX_STATUS_POLL_ATTEMPTS} attempts" + ) + + def _calculate_wait_time(self, progress_status: int, attempt: int) -> int: + """Calculate optimal wait time based on query progress and attempt number. + + Args: + progress_status: Current progress percentage (0-100). + attempt: Current attempt number. + + Returns: + Wait time in seconds. + + """ + if progress_status < 10: + base_wait = 10 + elif progress_status < 50: + base_wait = 5 + elif progress_status < 90: + base_wait = 3 + else: + base_wait = 2 + + backoff = min(attempt * 2, 10) + + return base_wait + backoff + + def _make_real_events_query(self, query_id: str) -> list[dict[str, Any]]: + """Make real API call to fetch Deep Visibility events. + + Args: + query_id: Query identifier from initialization. + + Returns: + List of event dictionaries. + + Raises: + SentinelOneValidationError: If query_id is empty. + SentinelOneAPIError: If API call fails. + SentinelOneNetworkError: If network error occurs. + + """ + if not query_id: + raise SentinelOneValidationError("query_id cannot be empty") + + try: + endpoint = f"{self.client_api.base_url}/web/api/v2.1/dv/events" + params = {"queryId": query_id} + + self.logger.debug(f"{LOG_PREFIX} Making GET request to: {endpoint}") + self.logger.debug(f"{LOG_PREFIX} Query parameters: {params}") + + response = self.client_api.session.get( + endpoint, params=params, timeout=REQUEST_TIMEOUT_SECONDS + ) + + if response.status_code == 200: + json_data = response.json() + events_data = json_data.get("data", []) + + self.logger.info( + f"{LOG_PREFIX} Retrieved {len(events_data)} Deep Visibility events from API" + ) + + self.logger.debug( + f"{LOG_PREFIX} Returning {len(events_data)} DV events as dict format" + ) + return events_data + else: + error_detail = self._parse_error_response(response) + raise SentinelOneAPIError( + f"DV events query failed with status {response.status_code}: {error_detail}" + ) + + except (SentinelOneValidationError, SentinelOneAPIError): + raise + except (ConnectionError, Timeout) as e: + raise SentinelOneNetworkError( + f"Network error making DV events query: {e}" + ) from e + except RequestException as e: + raise SentinelOneAPIError( + f"HTTP request failed for DV events query: {e}" + ) from e + except Exception as e: + raise SentinelOneAPIError( + f"Unexpected error making DV events query: {e}" + ) from e + + def _format_timestamp_for_api(self, dt: datetime) -> str: + """Format datetime object for SentinelOne API. + + SentinelOne API expects timestamps in format: 2018-02-27T04:49:26.257525Z + + Args: + dt: Datetime object to format (should be timezone-aware) + + Returns: + String formatted timestamp for SentinelOne API + + """ + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + elif dt.tzinfo != timezone.utc: + dt = dt.astimezone(timezone.utc) + + return dt.replace(tzinfo=None).isoformat() + "Z" + + def _parse_error_response(self, response: Any) -> str: + """Parse error response to extract detailed error information. + + Args: + response: HTTP response object. + + Returns: + Detailed error message string. + + """ + try: + if hasattr(response, "json"): + error_data = response.json() + errors = error_data.get("errors", []) + if errors: + error_messages = [] + for error in errors: + detail = error.get("detail", "") + title = error.get("title", "") + code = error.get("code", "") + + error_msg = f"Code {code}: {title}" + if detail: + error_msg += f" - {detail}" + error_messages.append(error_msg) + + return "; ".join(error_messages) + + return getattr(response, "text", str(response)) + + except Exception as e: + return f"Error parsing response: {e}" + + def _extract_retention_days(self, error_detail: str) -> int | None: + """Extract retention period in days from error message. + + Args: + error_detail: Error detail string from API response. + + Returns: + Number of retention days if found, None otherwise. + + """ + try: + match = re.search( + r"retains data for (\d+) days?", error_detail, re.IGNORECASE + ) + if match: + return int(match.group(1)) + + match = re.search(r"retention.*?(\d+)\s*days?", error_detail, re.IGNORECASE) + if match: + return int(match.group(1)) + + return None + + except Exception as e: + self.logger.debug(f"{LOG_PREFIX} Error extracting retention days: {e}") + return None diff --git a/template/src/services/fetcher_threat.py b/template/src/services/fetcher_threat.py new file mode 100644 index 00000000..080e5c97 --- /dev/null +++ b/template/src/services/fetcher_threat.py @@ -0,0 +1,141 @@ +"""SentinelOne Threat Fetcher.""" + +import logging +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +from requests.exceptions import ( # type: ignore[import-untyped] + ConnectionError, + RequestException, + Timeout, +) + +from .exception import ( + SentinelOneAPIError, + SentinelOneNetworkError, + SentinelOneValidationError, +) +from .model_threat import SentinelOneThreat, SentinelOneThreatsResponse + +if TYPE_CHECKING: + from .client_api import SentinelOneClientAPI + +LOG_PREFIX = "[SentinelOneThreatFetcher]" + + +class FetcherThreat: + """Fetcher for SentinelOne threat data using time-window based queries.""" + + def __init__(self, client_api: "SentinelOneClientAPI") -> None: + """Initialize the Threat fetcher. + + Args: + client_api: SentinelOne API client instance. + + Raises: + SentinelOneValidationError: If client_api is None. + + """ + if client_api is None: + raise SentinelOneValidationError("client_api cannot be None") + + self.logger = logging.getLogger(__name__) + self.client_api = client_api + self.logger.debug(f"{LOG_PREFIX} Threat fetcher initialized") + + def fetch_threats_for_time_window( + self, + start_time: datetime, + end_time: datetime, + limit: int = 1000, + ) -> list[SentinelOneThreat]: + """Fetch all threats for a given time window. + + Args: + start_time: Start time as datetime object. + end_time: End time as datetime object. + limit: Maximum number of threats to fetch. + + Returns: + List of SentinelOneThreat objects. + + Raises: + SentinelOneAPIError: If API call fails. + SentinelOneValidationError: If parameters are invalid. + + """ + if not isinstance(start_time, datetime) or not isinstance(end_time, datetime): + raise SentinelOneValidationError( + "start_time and end_time must be datetime objects" + ) + + if start_time >= end_time: + raise SentinelOneValidationError("start_time must be before end_time") + + if limit <= 0: + raise SentinelOneValidationError("limit must be positive") + + try: + start_time_str = self._format_timestamp_for_api(start_time) + end_time_str = self._format_timestamp_for_api(end_time) + + endpoint = f"{self.client_api.base_url}/web/api/v2.1/threats" + params = { + "createdAt__gte": start_time_str, + "createdAt__lt": end_time_str, + "sortOrder": "desc", + "limit": limit, + } + + self.logger.debug( + f"{LOG_PREFIX} Fetching threats for time window: {start_time_str} to {end_time_str}" + ) + + response = self.client_api.session.get(endpoint, params=params) + response.raise_for_status() + + json_data = response.json() + threats_data = json_data.get("data", []) + + response_wrapper = {"data": threats_data} + threats_response = SentinelOneThreatsResponse.from_raw_response( + response_wrapper + ) + threats = threats_response.data + + self.logger.info( + f"{LOG_PREFIX} Fetched {len(threats)} threats for time window" + ) + return threats + + except (ConnectionError, Timeout) as e: + raise SentinelOneNetworkError( + f"Network error fetching threats for time window: {e}" + ) from e + except RequestException as e: + raise SentinelOneAPIError( + f"HTTP request failed fetching threats for time window: {e}" + ) from e + except Exception as e: + raise SentinelOneAPIError( + f"Error fetching threats for time window: {e}" + ) from e + + def _format_timestamp_for_api(self, dt: datetime) -> str: + """Format datetime object for SentinelOne API. + + SentinelOne API expects timestamps in format: 2018-02-27T04:49:26.257525Z + + Args: + dt: Datetime object to format (should be timezone-aware) + + Returns: + String formatted timestamp for SentinelOne API + + """ + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + elif dt.tzinfo != timezone.utc: + dt = dt.astimezone(timezone.utc) + + return dt.replace(tzinfo=None).isoformat() + "Z" diff --git a/template/src/services/fetcher_threat_events.py b/template/src/services/fetcher_threat_events.py new file mode 100644 index 00000000..759dd863 --- /dev/null +++ b/template/src/services/fetcher_threat_events.py @@ -0,0 +1,139 @@ +"""SentinelOne Threat Events Fetcher.""" + +import logging +from typing import TYPE_CHECKING, Any + +from requests.exceptions import ( # type: ignore[import-untyped] + ConnectionError, + RequestException, + Timeout, +) + +from .exception import ( + SentinelOneAPIError, + SentinelOneNetworkError, + SentinelOneValidationError, +) +from .model_threat import SentinelOneThreat + +if TYPE_CHECKING: + from .client_api import SentinelOneClientAPI + +LOG_PREFIX = "[SentinelOneThreatEventsFetcher]" + + +class FetcherThreatEvents: + """Fetcher for SentinelOne threat events using API queries.""" + + def __init__(self, client_api: "SentinelOneClientAPI") -> None: + """Initialize the Threat Events fetcher. + + Args: + client_api: SentinelOne API client instance. + + Raises: + SentinelOneValidationError: If client_api is None. + + """ + if client_api is None: + raise SentinelOneValidationError("client_api cannot be None") + + self.logger = logging.getLogger(__name__) + self.client_api = client_api + self.logger.debug(f"{LOG_PREFIX} Threat events fetcher initialized") + + def fetch_events_for_threat( + self, + threat: SentinelOneThreat, + process_names: list[str] | None = None, + limit: int = 100, + ) -> list[dict[str, Any]]: + """Fetch events for a specific threat filtered by process names. + + Args: + threat: The threat to fetch events for. + process_names: Optional list of process names to filter by. + limit: Maximum number of events to fetch per process. + + Returns: + List of event dictionaries. + + Raises: + SentinelOneValidationError: If parameters are invalid. + SentinelOneAPIError: If API call fails. + + """ + if not isinstance(threat, SentinelOneThreat): + raise SentinelOneValidationError( + "threat must be a SentinelOneThreat instance" + ) + + if not threat.threat_id: + raise SentinelOneValidationError("threat must have a threat_id") + + if limit <= 0: + raise SentinelOneValidationError("limit must be positive") + + try: + self.logger.debug( + f"{LOG_PREFIX} Fetching events for threat {threat.threat_id}, {threat}" + ) + + all_events = self._fetch_all_events_for_threat(threat, limit) + + self.logger.info( + f"{LOG_PREFIX} Fetched {len(all_events)} total events for threat {threat.threat_id}" + ) + return all_events + + except (SentinelOneValidationError, SentinelOneAPIError): + raise + except Exception as e: + raise SentinelOneAPIError( + f"Unexpected error fetching events for threat {threat.threat_id}: {e}" + ) from e + + def _fetch_all_events_for_threat( + self, threat: SentinelOneThreat, limit: int + ) -> list[dict[str, Any]]: + """Fetch all events for a threat without process name filtering. + + Args: + threat: The threat to fetch events for. + limit: Maximum number of events to fetch. + + Returns: + List of event dictionaries. + + """ + try: + endpoint = f"{self.client_api.base_url}/web/api/v2.1/threats/{threat.threat_id}/explore/events" + params = {"limit": limit} + + self.logger.debug( + f"{LOG_PREFIX} Making API call to fetch events for threat {threat.threat_id}" + ) + + response = self.client_api.session.get(endpoint, params=params) + response.raise_for_status() + + json_data = response.json() + events = json_data.get("data", []) + + self.logger.debug( + f"{LOG_PREFIX} Retrieved {len(events)} events for threat {threat.threat_id}" + ) + return events + + except (ConnectionError, Timeout) as e: + raise SentinelOneNetworkError( + f"Network error fetching events for threat {threat.threat_id}: {e}" + ) from e + except RequestException as e: + raise SentinelOneAPIError( + f"HTTP request failed for threat {threat.threat_id}: {e}" + ) from e + except Exception as e: + raise SentinelOneAPIError( + f"Error fetching events for threat {threat.threat_id}: {e}" + ) from e diff --git a/template/src/services/model_threat.py b/template/src/services/model_threat.py new file mode 100644 index 00000000..e451873d --- /dev/null +++ b/template/src/services/model_threat.py @@ -0,0 +1,132 @@ +"""SentinelOne Threat Models.""" + +from typing import Any, Optional + +from pydantic import BaseModel, Field, PrivateAttr + + +class SentinelOneThreat(BaseModel): + """SentinelOne threat model.""" + + threat_id: str = Field(..., description="Unique identifier for the threat") + hostname: Optional[str] = Field(None, description="Agent computer name") + is_mitigated: bool = Field(False, description="Whether threat has been mitigated") + is_static: bool = Field(False, description="Whether threat is static") + sha1: Optional[str] = Field(None, description="SHA1 hash of the threat file") + _raw: Optional[dict[str, Any]] = PrivateAttr(default=None) + + def __str__(self) -> str: + """Detaield representation with key debugging information.""" + return ( + f"SentinelOneThreat(threat_id='{self.threat_id}', " + f"hostname='{self.hostname}', is_mitigated={self.is_mitigated}, is_static={self.is_static}, sha1='{self.sha1}')" + ) + + @staticmethod + def get_parent_process_name_from_events(events: list[dict]) -> list[str]: + """Extract parent process names from threat events data. + + Args: + events: List of event dictionaries from threat_events endpoint. + + Returns: + List of unique parent process names found in events. + + """ + if not events: + return [] + + parent_process_names = set() + for event in events: + parent_process_name = event.get("parentProcessName") + if parent_process_name: + parent_process_names.add(parent_process_name) + + return list(parent_process_names) + + @staticmethod + def get_parent_process_name_from_dv_events(events: list[dict]) -> list[str]: + """Extract parent process names from Deep Visibility events data. + + Checks both parentProcessName and processName fields since OAEV implants + can appear in either field. + + Args: + events: List of Deep Visibility event dictionaries. + + Returns: + List of unique process names found in events (both parent and process names). + + """ + if not events: + return [] + + process_names = set() + for event in events: + parent_process_name = event.get("parentProcessName") + if parent_process_name: + process_names.add(parent_process_name) + + process_name = event.get("processName") + if process_name: + process_names.add(process_name) + + return list(process_names) + + +class SentinelOneThreatsResponse(BaseModel): + """Response from threats endpoint.""" + + data: list[SentinelOneThreat] = Field( + default_factory=list, description="List of SentinelOne threats" + ) + + @classmethod + def from_raw_response( + cls, response_data: dict[str, Any] + ) -> "SentinelOneThreatsResponse": + """Create from raw API response. + + Args: + response_data: Raw response data from the threats API. + + Returns: + SentinelOneThreatsResponse instance with parsed threats. + + """ + threats = [] + raw_threats = response_data.get("data", []) + + for raw_threat in raw_threats: + threat_info = raw_threat.get("threatInfo", {}) + threat_id = threat_info.get("threatId") + + if threat_id: + agent_realtime_info = raw_threat.get("agentRealtimeInfo", {}) + hostname = agent_realtime_info.get("agentComputerName") + + mitigation_status = raw_threat.get("mitigationStatus", []) + is_mitigated = False + if isinstance(mitigation_status, list): + is_mitigated = any( + status.get("status") == "success" + for status in mitigation_status + if isinstance(status, dict) + ) + + detection_type = threat_info.get("detectionType", "").lower() + is_static = detection_type == "static" + + sha1 = threat_info.get("sha1") + + threat = SentinelOneThreat( + threat_id=threat_id, + hostname=hostname, + is_mitigated=is_mitigated, + is_static=is_static, + sha1=sha1, + ) + threat._raw = raw_threat + threats.append(threat) + + return cls(data=threats) diff --git a/template/src/services/trace_service.py b/template/src/services/trace_service.py new file mode 100644 index 00000000..99af6a72 --- /dev/null +++ b/template/src/services/trace_service.py @@ -0,0 +1,198 @@ +"""SentinelOne Trace Service Provider.""" + +import logging +from datetime import UTC, datetime +from typing import Any + +from ..collector.models import ExpectationResult, ExpectationTrace +from ..models.configs.config_loader import ConfigLoader +from .exception import SentinelOneDataConversionError, SentinelOneValidationError + +LOG_PREFIX = "[SentinelOneTraceService]" + + +class SentinelOneTraceService: + """SentinelOne-specific trace service provider. + + This service extracts trace information from expectation processing results + and converts them into OpenAEV expectation traces using proper Pydantic models. + """ + + def __init__(self, config: ConfigLoader | None = None) -> None: + """Initialize the SentinelOne trace service. + + Args: + config: Configuration loader instance for trace service settings. + + Raises: + SentinelOneValidationError: If config is None. + + """ + if config is None: + raise SentinelOneValidationError("Config is required for trace service") + + self.logger = logging.getLogger(__name__) + self.config = config + self.logger.debug(f"{LOG_PREFIX} SentinelOne trace service initialized") + + def create_traces_from_results( + self, results: list[ExpectationResult], collector_id: str + ) -> list[ExpectationTrace]: + """Create trace data from processing results. + + Args: + results: List of expectation processing results. + collector_id: ID of the collector. + + Returns: + List of ExpectationTrace models for OpenAEV. + + Raises: + SentinelOneValidationError: If inputs are invalid. + SentinelOneDataConversionError: If trace creation fails. + + """ + if not collector_id: + raise SentinelOneValidationError("collector_id cannot be empty") + + if not isinstance(results, list): + raise SentinelOneValidationError("results must be a list") + + try: + valid_results = [r for r in results if r.is_valid and r.matched_alerts] + + if not valid_results: + self.logger.info( + f"{LOG_PREFIX} No valid results with matching data for traces out of {len(results)} results" + ) + return [] + + self.logger.info( + f"{LOG_PREFIX} Creating traces for {len(valid_results)} valid results out of {len(results)} total" + ) + + traces = [] + + for i, result in enumerate(valid_results, 1): + expectation_id = result.expectation_id + if not expectation_id: + self.logger.warning( + f"{LOG_PREFIX} Skipping result {i} - missing expectation_id" + ) + continue + + self.logger.debug( + f"{LOG_PREFIX} Creating trace {i}/{len(valid_results)} for expectation {expectation_id}" + ) + + try: + trace = self._create_expectation_trace( + result, expectation_id, collector_id + ) + + if trace: + traces.append(trace) + self.logger.debug( + f"{LOG_PREFIX} Created trace for expectation {expectation_id}: {trace.inject_expectation_trace_alert_name}" + ) + else: + self.logger.warning( + f"{LOG_PREFIX} Trace creation returned None for expectation {expectation_id}" + ) + except Exception as e: + raise SentinelOneDataConversionError( + f"Error creating trace for expectation {expectation_id}: {e}" + ) from e + + self.logger.info( + f"{LOG_PREFIX} Successfully created {len(traces)} traces from {len(valid_results)} valid results" + ) + return traces + + except SentinelOneDataConversionError: + raise + except Exception as e: + raise SentinelOneDataConversionError( + f"Unexpected error creating traces from results: {e}" + ) from e + + def _create_expectation_trace( + self, result: ExpectationResult, expectation_id: str, collector_id: str + ) -> ExpectationTrace: + """Create ExpectationTrace model from a single result. + + Args: + result: Processing result dictionary. + expectation_id: ID of the expectation. + collector_id: ID of the collector. + + Returns: + ExpectationTrace model for OpenAEV. + + Raises: + SentinelOneValidationError: If inputs are invalid. + SentinelOneDataConversionError: If trace creation fails. + + """ + if not expectation_id: + raise SentinelOneValidationError("expectation_id cannot be empty") + + if not collector_id: + raise SentinelOneValidationError("collector_id cannot be empty") + + if not result.matched_alerts: + raise SentinelOneValidationError( + "result must have matched_alerts for trace creation" + ) + + try: + matching_data = result.matched_alerts[0] or {} + self.logger.debug( + f"{LOG_PREFIX} Processing matching data with {len(matching_data)} fields" + ) + + alert_name = matching_data.get("alert_name", "SentinelOne Alert") + + trace_link = matching_data.get("alert_link", "") + self.logger.debug(f"{LOG_PREFIX} Using trace builder URL: {trace_link}") + + trace_date = datetime.now(UTC).replace(microsecond=0) + date_str = trace_date.isoformat().replace("+00:00", "Z") + self.logger.debug(f"{LOG_PREFIX} Generated trace date: {date_str}") + + trace = ExpectationTrace( + inject_expectation_trace_expectation=str(expectation_id), + inject_expectation_trace_source_id=str(collector_id), + inject_expectation_trace_alert_name=alert_name, + inject_expectation_trace_alert_link=trace_link, + inject_expectation_trace_date=date_str, + ) + + self.logger.debug( + f"{LOG_PREFIX} Created ExpectationTrace with alert name: {alert_name}" + ) + return trace + + except SentinelOneValidationError: + raise + except Exception as e: + raise SentinelOneDataConversionError( + f"Error creating expectation trace: {e}" + ) from e + + def get_service_info(self) -> dict[str, Any]: + """Get information about this trace service. + + Returns: + Dictionary containing service metadata and capabilities. + + """ + info = { + "service_type": "sentinelone_trace", + "supported_result_types": ["SentinelOne processing results"], + "creates_detection_traces": True, + "creates_prevention_traces": True, + "description": "Creates traces from SentinelOne expectation processing results using trace builder URLs", + } + self.logger.debug(f"{LOG_PREFIX} Trace service info: {info}") + return info diff --git a/template/src/services/utils/__init__.py b/template/src/services/utils/__init__.py new file mode 100644 index 00000000..2aa07028 --- /dev/null +++ b/template/src/services/utils/__init__.py @@ -0,0 +1,9 @@ +from src.services.utils.config_loader import SentinelOneConfig +from src.services.utils.signature_extractor import SignatureExtractor +from src.services.utils.trace_builder import TraceBuilder + +__all__ = [ + "SentinelOneConfig", + "SignatureExtractor", + "TraceBuilder", +] diff --git a/template/src/services/utils/config_loader.py b/template/src/services/utils/config_loader.py new file mode 100644 index 00000000..c29ac902 --- /dev/null +++ b/template/src/services/utils/config_loader.py @@ -0,0 +1,72 @@ +"""Configuration loader.""" + +import logging + +from pydantic import ValidationError +from src.models import ConfigLoader + +LOG_PREFIX = "[CollectorConfig]" + + +class SentinelOneConfig: + """Class for loading SentinelOne configuration.""" + + def __init__(self) -> None: + """Initialize SentinelOne configuration loader. + + Loads configuration from YAML files, environment variables, and defaults. + Sets up logging and validates the configuration structure. + + Raises: + ValueError: If configuration loading or validation fails. + + """ + self.logger = logging.getLogger(__name__) + self.logger.debug(f"{LOG_PREFIX} Initializing SentinelOne configuration loader") + self.load = self._load_config() + self.logger.info(f"{LOG_PREFIX} SentinelOne configuration loaded successfully") + + def _load_config(self) -> ConfigLoader: + """Load configuration with proper error handling and logging. + + Loads configuration from multiple sources and validates the structure. + Logs configuration details for debugging purposes. + + Returns: + ConfigLoader instance with validated configuration. + + Raises: + ValueError: If configuration validation or loading fails. + + """ + try: + self.logger.debug( + f"{LOG_PREFIX} Loading configuration from sources (YAML/ENV/defaults)" + ) + load_settings = ConfigLoader() + + self.logger.debug( + f"{LOG_PREFIX} Collector ID: {load_settings.collector.id}" + ) + self.logger.debug( + f"{LOG_PREFIX} Collector name: {load_settings.collector.name}" + ) + self.logger.debug( + f"{LOG_PREFIX} Log level: {load_settings.collector.log_level}" + ) + self.logger.debug(f"{LOG_PREFIX} OpenAEV URL: {load_settings.openaev.url}") + self.logger.debug( + f"{LOG_PREFIX} SentinelOne base URL: {load_settings.sentinelone.base_url}" + ) + + return load_settings + except ValidationError as err: + self.logger.error( + f"{LOG_PREFIX} Error in configuration validation: {err} (Context: error_type=ValidationError)" + ) + raise ValueError(f"Configuration validation failed: {err}") from err + except Exception as err: + self.logger.error( + f"{LOG_PREFIX} Error in configuration loading: {err} (Context: error_type={type(err).__name__})" + ) + raise ValueError(f"Configuration loading failed: {err}") from err diff --git a/template/src/services/utils/signature_extractor.py b/template/src/services/utils/signature_extractor.py new file mode 100644 index 00000000..be61cb4c --- /dev/null +++ b/template/src/services/utils/signature_extractor.py @@ -0,0 +1,127 @@ +"""Signature extraction utilities for SentinelOne expectation processing.""" + +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +from pyoaev.signatures.types import SignatureTypes + +if TYPE_CHECKING: + from pyoaev.apis.inject_expectation.model.expectation import ( + DetectionExpectation, + PreventionExpectation, + ) + + +class SignatureExtractor: + """Utility class for extracting signatures from expectations.""" + + @staticmethod + def extract_hostnames( + batch: list["DetectionExpectation | PreventionExpectation"], + ) -> list[str]: + """Extract unique hostnames from a batch of expectations. + + Args: + batch: List of expectations to extract hostnames from. + + Returns: + List of unique hostname values. + + """ + hostnames = set() + for expectation in batch: + for signature in expectation.inject_expectation_signatures: + if signature.type == SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS: + hostnames.add(signature.value) + return list(hostnames) + + @staticmethod + def extract_process_names( + batch: list["DetectionExpectation | PreventionExpectation"], + ) -> list[str]: + """Extract unique parent process names from a batch of expectations. + + Args: + batch: List of expectations to extract process names from. + + Returns: + List of unique parent process name values. + + """ + process_names = set() + for expectation in batch: + for signature in expectation.inject_expectation_signatures: + if signature.type.value == "parent_process_name": + process_names.add(signature.value) + return list(process_names) + + @staticmethod + def extract_end_date( + batch: list["DetectionExpectation | PreventionExpectation"] | None = None, + ) -> datetime | None: + """Extract end_date from batch signatures. + + Args: + batch: List of expectations to extract end_date from. If None, returns None. + + Returns: + Parsed end_date as datetime or None if no valid end_date signature found. + + """ + if not batch: + return None + + for expectation in batch: + for signature in expectation.inject_expectation_signatures: + if signature.type.value == "end_date": + try: + end_date = datetime.fromisoformat( + signature.value.replace("Z", "+00:00") + ) + if end_date.tzinfo is None: + end_date = end_date.replace(tzinfo=timezone.utc) + return end_date + except (ValueError, AttributeError): + continue + return None + + @staticmethod + def group_signatures_by_type( + expectation: "DetectionExpectation | PreventionExpectation", + supported_signatures: list[SignatureTypes] | None = None, + ) -> dict[str, list[dict[str, str]]]: + """Group signatures by type for detection helper matching. + + Args: + expectation: Single expectation to group signatures from. + supported_signatures: List of supported signature types to filter by. + If None, all signature types are included. + + Returns: + Dictionary mapping signature types to lists of signature dictionaries + in the format expected by detection helper (with 'value' and 'type' keys). + Only includes signature types that are in the supported list. + Excludes end_date as it's only used for query criteria, not matching. + + """ + supported_types = None + if supported_signatures: + supported_types = { + sig_type.value if hasattr(sig_type, "value") else str(sig_type) + for sig_type in supported_signatures + } + + signature_groups = {} + for sig in expectation.inject_expectation_signatures: + sig_type = sig.type.value if hasattr(sig.type, "value") else str(sig.type) + + if supported_types and sig_type not in supported_types: + continue + + if sig_type == "end_date": + continue + + if sig_type not in signature_groups: + signature_groups[sig_type] = [] + signature_groups[sig_type].append({"type": sig_type, "value": sig.value}) + return signature_groups diff --git a/template/src/services/utils/trace_builder.py b/template/src/services/utils/trace_builder.py new file mode 100644 index 00000000..d44b6529 --- /dev/null +++ b/template/src/services/utils/trace_builder.py @@ -0,0 +1,74 @@ +"""Trace building utilities for SentinelOne expectation processing.""" + +import logging +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any +from urllib.parse import quote + +if TYPE_CHECKING: + from ..model_threat import SentinelOneThreat + +LOG_PREFIX = "[SentinelOneTraceBuilder]" + + +class TraceBuilder: + """Utility class for building trace information.""" + + @staticmethod + def create_threat_trace( + threat: "SentinelOneThreat", + base_url: str, + events: list[dict[str, Any]], + ) -> dict[str, Any]: + """Create trace information for a threat. + + Args: + threat: SentinelOne threat object. + base_url: Base URL for SentinelOne web interface. + events: List of events associated with the threat. + + Returns: + Dictionary containing trace information with alert name, link, date, + and additional metadata. + + """ + logger = logging.getLogger(__name__) + alert_link = "" + if base_url and threat.threat_id: + try: + web_base = base_url.rstrip("/") + encoded_threat_id = quote(threat.threat_id) + alert_link = ( + f"{web_base}/incidents/threats/{encoded_threat_id}/overview" + ) + logger.debug(f"{LOG_PREFIX} Generated threat URL: {alert_link}") + except Exception as e: + logger.error(f"{LOG_PREFIX} Error generating URL: {e}") + alert_link = "" + else: + logger.warning( + f"{LOG_PREFIX} Cannot generate URL - base_url='{base_url}', threat_id='{threat.threat_id}'" + ) + + alert_name = "SentinelOne Alert" + if threat.hostname: + alert_name = f"{alert_name} - {threat.hostname}" + elif threat.threat_id: + alert_name = f"{alert_name} {threat.threat_id}" + + trace_data = { + "alert_name": alert_name, + "alert_link": alert_link, + "alert_date": datetime.now(timezone.utc).isoformat(), + "additional_data": { + "threat_id": threat.threat_id, + "hostname": threat.hostname, + "is_mitigated": threat.is_mitigated, + "is_static": threat.is_static, + "events_count": len(events), + "data_source": "sentinelone", + }, + } + + logger.debug(f"{LOG_PREFIX} Created trace data: {trace_data}") + return trace_data diff --git a/template/tests/__init__.py b/template/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/tests/conftest.py b/template/tests/conftest.py new file mode 100644 index 00000000..4b52f820 --- /dev/null +++ b/template/tests/conftest.py @@ -0,0 +1,93 @@ +"""Conftest file for Pytest fixtures.""" + +from typing import TYPE_CHECKING, Any +from unittest.mock import patch + +from pytest import fixture + +if TYPE_CHECKING: + from os import _Environ + + +def mock_env_vars(os_environ: "_Environ[str]", wanted_env: dict[str, str]) -> Any: + """Fixture to mock environment variables dynamically and clean up after. + + Args: + os_environ: The os.environ object to patch. + wanted_env: Dictionary of environment variables to mock. + + Returns: + Mock object for environment variable patching. + + """ + mock_env = patch.dict(os_environ, wanted_env) + mock_env.start() + + return mock_env + + +@fixture(autouse=True) +def mock_openaev_client() -> Any: + """Fixture to mock OpenAEV calls and clean up after. + + Auto-applies to all tests to prevent actual OpenAEV API calls. + Mocks urllib3, pyoaev client, and collector daemon setup. + + Yields: + Tuple of mock objects (urllib, pyoaev, daemon_setup). + + """ + mock_urllib = patch("urllib3.connectionpool.HTTPConnectionPool.urlopen") + mock_pyoaev = patch("pyoaev.client.OpenAEV.http_request") + mock_daemon_setup = patch("pyoaev.daemons.collector_daemon.CollectorDaemon._setup") + + mock_urllib.start() + mock_pyoaev.start() + mock_daemon_setup.start() + + yield mock_urllib, mock_pyoaev, mock_daemon_setup + + mock_urllib.stop() + mock_pyoaev.stop() + mock_daemon_setup.stop() + + +@fixture(autouse=True) +def disable_config_yml() -> Any: + """Fixture to disable config.yml and .env files for tests, forcing environment variable usage only. + + Auto-applies to all tests to ensure consistent configuration loading + from environment variables instead of config files. + + Yields: + Patcher object for the settings customization. + + """ + + def fake_settings_customise_sources( + cls, + settings_cls, + init_settings, + env_settings, + dotenv_settings, + file_secret_settings, + ): + from pydantic_settings import EnvSettingsSource + + # Return only environment settings source, ignoring files + return ( + EnvSettingsSource( + settings_cls, + env_ignore_empty=True, + ), + ) + + patcher = patch( + "src.models.configs.config_loader.ConfigLoader.settings_customise_sources", + new=classmethod(fake_settings_customise_sources), + ) + patcher.start() + + yield patcher + + patcher.stop() diff --git a/template/tests/gwt_shared.py b/template/tests/gwt_shared.py new file mode 100644 index 00000000..11513e11 --- /dev/null +++ b/template/tests/gwt_shared.py @@ -0,0 +1,757 @@ +"""Shared Given-When-Then methods for Gherkin-style tests. + +This module provides reusable Given, When, and Then methods that can be used +across multiple test files to maximize code reusability and maintain consistency +in test structure. +""" + +from os import environ as os_environ +from typing import Any, Dict, List +from unittest.mock import Mock, patch + +import pytest +from src.collector import Collector +from src.services.client_api import SentinelOneClientAPI +from src.services.converter import SentinelOneConverter +from src.services.exception import SentinelOneValidationError +from src.services.expectation_service import SentinelOneExpectationService +from src.services.model_threat import SentinelOneThreat +from tests.conftest import mock_env_vars +from tests.services.fixtures.factories import create_test_config + +# ======================================================================== +# SHARED GIVEN METHODS - Set up test preconditions +# ======================================================================== + + +# Configuration Setup Given Methods +# --------------------------------- + + +def given_valid_collector_config(config_data: Dict[str, str]) -> Any: + """Set up valid collector configuration environment. + + Args: + config_data: Dictionary of environment variables to mock. + + Returns: + Mock environment variable patcher object. + + """ + return mock_env_vars(os_environ, config_data) + + +def given_test_config(): + """Create a standard test configuration. + + Returns: + Test configuration object. + + """ + return create_test_config() + + +def given_config_missing_field( + base_config: Dict[str, str], field_name: str +) -> Dict[str, str]: + """Create configuration with a missing required field. + + Args: + base_config: Base configuration dictionary. + field_name: Name of the field to remove. + + Returns: + Configuration dictionary without the specified field. + + """ + config = base_config.copy() + config.pop(field_name, None) + + if field_name in os_environ: + del os_environ[field_name] + + return config + + +def given_config_with_invalid_value( + base_config: Dict[str, str], field_name: str, invalid_value: str +) -> Dict[str, str]: + """Create configuration with an invalid field value. + + Args: + base_config: Base configuration dictionary. + field_name: Name of the field to modify. + invalid_value: Invalid value to set. + + Returns: + Configuration dictionary with invalid value. + + """ + config = base_config.copy() + config[field_name] = invalid_value + return config + + +# Object Creation Given Methods +# ----------------------------- + + +def given_initialized_converter() -> SentinelOneConverter: + """Create an initialized converter. + + Returns: + Initialized SentinelOneConverter instance. + + """ + return SentinelOneConverter() + + +def given_initialized_client_api(): + """Create an initialized client API. + + Returns: + Initialized SentinelOneClientAPI instance. + + """ + config = given_test_config() + return SentinelOneClientAPI(config=config) + + +def given_initialized_expectation_service(): + """Create an initialized expectation service. + + Returns: + Initialized SentinelOneExpectationService instance. + + """ + config = given_test_config() + return SentinelOneExpectationService(config=config) + + +# Data Creation Given Methods +# --------------------------- + + +def given_threat_with_complete_data( + threat_id: str = "test_threat_123", hostname: str = "test-host.example.com" +) -> SentinelOneThreat: + """Create a threat with complete data. + + Args: + threat_id: ID for the threat. + hostname: Hostname for the threat. + + Returns: + SentinelOneThreat with complete data. + + """ + return SentinelOneThreat(threat_id=threat_id, hostname=hostname) + + +def given_threat_without_hostname( + threat_id: str = "no_hostname_threat", +) -> SentinelOneThreat: + """Create a threat without hostname. + + Args: + threat_id: ID for the threat. + + Returns: + SentinelOneThreat without hostname. + + """ + return SentinelOneThreat(threat_id=threat_id, hostname=None) + + +def given_threat_with_empty_id( + hostname: str = "empty-id-host.example.com", +) -> SentinelOneThreat: + """Create a threat with empty threat ID. + + Args: + hostname: Hostname for the threat. + + Returns: + SentinelOneThreat with empty threat_id. + + """ + return SentinelOneThreat(threat_id="", hostname=hostname) + + +def given_multiple_threats(count: int = 3) -> List[SentinelOneThreat]: + """Create multiple threats with different data combinations. + + Args: + count: Number of threats to create. + + Returns: + List of SentinelOneThreat objects. + + """ + threats = [] + for i in range(count): + threat_id = f"multi_threat_{i + 1}" + hostname = f"host{i + 1}.example.com" if i % 2 == 0 else None + threats.append(SentinelOneThreat(threat_id=threat_id, hostname=hostname)) + return threats + + +def given_large_batch_of_threats(count: int = 100) -> List[SentinelOneThreat]: + """Create a large batch of threats for performance testing. + + Args: + count: Number of threats to create. + + Returns: + List of SentinelOneThreat objects. + + """ + return [ + SentinelOneThreat( + threat_id=f"bulk_threat_{i}", + hostname=f"host{i}.example.com" if i % 2 == 0 else None, + ) + for i in range(count) + ] + + +def given_mixed_valid_invalid_objects(valid_threat_id: str = "valid_mixed_123") -> List: + """Create a list with mixed valid and invalid objects. + + Args: + valid_threat_id: ID for the valid threat in the list. + + Returns: + List containing valid threats and invalid objects. + + """ + valid_threat = SentinelOneThreat( + threat_id=valid_threat_id, hostname="valid-mixed.example.com" + ) + return [ + valid_threat, + {"threat_id": "dict_threat"}, + "string_threat", + 42, + ] + + +def given_invalid_input_data() -> str: + """Create invalid input data for testing. + + Returns: + Invalid data (string instead of list). + + """ + return "invalid_string_data" + + +# Mock Setup Given Methods +# ------------------------ + + +def given_mock_session_that_fails(): + """Create a mock session that fails during creation. + + Returns: + Context manager for mocking session failure. + + """ + return patch("requests.Session", side_effect=Exception("Session creation failed")) + + +def given_mock_session_with_header_failure(): + """Create a mock session that fails during header setup. + + Returns: + Context manager for mocking header setup failure. + + """ + + def create_failing_session(): + mock_session = Mock() + mock_session.headers.update.side_effect = Exception("Header setup failed") + return mock_session + + return patch("requests.Session", side_effect=create_failing_session) + + +def given_conversion_error_setup(converter: SentinelOneConverter) -> List: + """Set up threats that will cause conversion errors. + + Args: + converter: The converter instance to mock. + + Returns: + List with valid threats and error-causing mock threats. + + """ + error_threat = Mock(spec=SentinelOneThreat) + error_threat.threat_id = "error_threat" + error_threat.hostname = None + + valid_threat = SentinelOneThreat( + threat_id="valid_error_test_123", hostname="valid-error-test.example.com" + ) + + original_convert = converter._convert_threat_to_oaev + + def mock_convert(threat): + if hasattr(threat, "threat_id") and threat.threat_id == "error_threat": + raise Exception("Conversion error") + return original_convert(threat) + + converter._convert_threat_to_oaev = mock_convert + + return [error_threat, valid_threat] + + +# ======================================================================== +# SHARED WHEN METHODS - Execute actions being tested +# ======================================================================== + + +# Object Creation When Methods +# ---------------------------- + + +def when_create_collector() -> Collector: + """Create a collector instance. + + Returns: + Collector instance. + + """ + return Collector() + + +def when_initialize_converter() -> SentinelOneConverter: + """Initialize a converter. + + Returns: + Initialized SentinelOneConverter instance. + + """ + return SentinelOneConverter() + + +def when_initialize_client_api(): + """Initialize a client API. + + Returns: + Initialized SentinelOneClientAPI instance. + + """ + config = given_test_config() + return SentinelOneClientAPI(config=config) + + +def when_initialize_expectation_service(): + """Initialize an expectation service. + + Returns: + Initialized SentinelOneExpectationService instance. + + """ + config = given_test_config() + return SentinelOneExpectationService(config=config) + + +# Data Processing When Methods +# ---------------------------- + + +def when_convert_threats_to_oaev( + converter: SentinelOneConverter, threats: List +) -> List: + """Convert threats to OAEV format. + + Args: + converter: The converter instance. + threats: List of threats to convert. + + Returns: + List of converted OAEV format data. + + """ + return converter.convert_threats_to_oaev(threats) + + +def when_call_private_conversion_method( + converter: SentinelOneConverter, threat: SentinelOneThreat +) -> Dict: + """Call the private conversion method directly. + + Args: + converter: The converter instance. + threat: The threat to convert. + + Returns: + Converted OAEV format dictionary. + + """ + return converter._convert_threat_to_oaev(threat) + + +# Error Handling When Methods +# --------------------------- + + +def when_create_collector_expecting_error(mock_env: Any) -> None: + """Attempt to create collector and expect configuration error. + + Args: + mock_env: Mock environment variable patcher to clean up. + + """ + try: + with pytest.raises((Exception, ValueError)): + when_create_collector() + finally: + mock_env.stop() + + +def when_convert_invalid_data_expecting_validation_error( + converter: SentinelOneConverter, invalid_data: Any +) -> None: + """Attempt to convert invalid data and expect validation error. + + Args: + converter: The converter instance. + invalid_data: Invalid input data. + + """ + with pytest.raises(SentinelOneValidationError) as exc_info: + converter.convert_threats_to_oaev(invalid_data) + + assert "threats must be a list" in str(exc_info.value) # noqa: S101 + + +def when_call_private_method_expecting_validation_error( + converter: SentinelOneConverter, threat: SentinelOneThreat +) -> None: + """Call private method and expect validation error. + + Args: + converter: The converter instance. + threat: The threat with empty threat_id. + + """ + with pytest.raises(SentinelOneValidationError) as exc_info: + converter._convert_threat_to_oaev(threat) + + assert "Threat must have a threat_id" in str(exc_info.value) # noqa: S101 + + +# ======================================================================== +# SHARED THEN METHODS - Validate results and assert expectations +# ======================================================================== + + +# Object Validation Then Methods +# ------------------------------ + + +def then_collector_created_successfully(collector: Collector) -> None: + """Verify collector was created successfully. + + Args: + collector: The collector instance to verify. + + """ + assert collector is not None # noqa: S101 + assert hasattr(collector, "config_instance") # noqa: S101 + + +def then_collector_has_valid_configuration( + collector: Collector, expected_config: Dict[str, str] +) -> None: + """Verify collector has valid configuration. + + Args: + collector: The collector instance to verify. + expected_config: Expected configuration values. + + """ + daemon_config = collector.config_instance.to_daemon_config() + + assert daemon_config.get("openaev_url") == expected_config.get( # noqa: S101 + "OPENAEV_URL" + ) + assert daemon_config.get("openaev_token") == expected_config.get( # noqa: S101 + "OPENAEV_TOKEN" + ) + assert daemon_config.get("collector_id") == expected_config.get( # noqa: S101 + "COLLECTOR_ID" + ) + assert daemon_config.get("collector_name") == expected_config.get( # noqa: S101 + "COLLECTOR_NAME" + ) + assert daemon_config.get( # noqa: S101 + "sentinelone_base_url" + ) == expected_config.get("SENTINELONE_BASE_URL") + assert daemon_config.get( # noqa: S101 + "sentinelone_api_key" + ) == expected_config.get("SENTINELONE_API_KEY") + + +def then_converter_initialized_successfully(converter: SentinelOneConverter) -> None: + """Verify converter was initialized successfully. + + Args: + converter: The converter instance to verify. + + """ + assert converter is not None # noqa: S101 + assert converter.logger is not None # noqa: S101 + + +def then_client_api_initialized_successfully(client: SentinelOneClientAPI) -> None: + """Verify client API was initialized successfully. + + Args: + client: The client API instance to verify. + + """ + assert client is not None # noqa: S101 + assert hasattr(client, "config") # noqa: S101 + assert hasattr(client, "session") # noqa: S101 + assert hasattr(client, "base_url") # noqa: S101 + assert hasattr(client, "api_key") # noqa: S101 + + +def then_expectation_service_initialized_successfully( + service: SentinelOneExpectationService, +) -> None: + """Verify expectation service was initialized successfully. + + Args: + service: The expectation service instance to verify. + + """ + assert service is not None # noqa: S101 + assert service.client_api is not None # noqa: S101 + assert service.converter is not None # noqa: S101 + assert service.threat_fetcher is not None # noqa: S101 + + +# Data Validation Then Methods +# ---------------------------- + + +def then_empty_list_returned(result: List) -> None: + """Verify an empty list was returned. + + Args: + result: The result to verify. + + """ + assert result == [] # noqa: S101 + + +def then_single_threat_converted_completely( + result: List, threat: SentinelOneThreat +) -> None: + """Verify single threat was converted with all fields. + + Args: + result: The conversion result to verify. + threat: The original threat object. + + """ + assert len(result) == 1 # noqa: S101 + + converted = result[0] + + assert "threat_id" in converted # noqa: S101 + assert converted["threat_id"]["type"] == "fuzzy" # noqa: S101 + assert converted["threat_id"]["data"] == [threat.threat_id] # noqa: S101 + assert converted["threat_id"]["score"] == 95 # noqa: S101 + + if threat.hostname: + assert "target_hostname_address" in converted # noqa: S101 + assert converted["target_hostname_address"]["type"] == "fuzzy" # noqa: S101 + assert converted["target_hostname_address"]["data"] == [ # noqa: S101 + threat.hostname + ] + assert converted["target_hostname_address"]["score"] == 95 # noqa: S101 + + +def then_single_threat_converted_without_hostname( + result: List, threat: SentinelOneThreat +) -> None: + """Verify single threat was converted without hostname field. + + Args: + result: The conversion result to verify. + threat: The original threat object. + + """ + assert len(result) == 1 # noqa: S101 + + converted = result[0] + + assert "threat_id" in converted # noqa: S101 + assert converted["threat_id"]["data"] == [threat.threat_id] # noqa: S101 + + assert "target_hostname_address" not in converted # noqa: S101 + + +def then_multiple_threats_converted( + result: List, threats: List[SentinelOneThreat] +) -> None: + """Verify multiple threats were converted correctly. + + Args: + result: The conversion result to verify. + threats: The original threats list. + + """ + valid_threats = [t for t in threats if t.threat_id and t.threat_id.strip()] + assert len(result) == len(valid_threats) # noqa: S101 + + threat_ids = [item["threat_id"]["data"][0] for item in result] + for threat in valid_threats: + assert threat.threat_id in threat_ids # noqa: S101 + + items_with_hostname = [item for item in result if "target_hostname_address" in item] + expected_hostname_count = sum(1 for threat in valid_threats if threat.hostname) + assert len(items_with_hostname) == expected_hostname_count # noqa: S101 + + +def then_only_valid_threats_converted( + result: List, expected_valid_count: int = 1 +) -> None: + """Verify only valid threats were converted from mixed data. + + Args: + result: The conversion result to verify. + expected_valid_count: Expected number of valid conversions. + + """ + assert len(result) == expected_valid_count # noqa: S101 + + +def then_large_batch_converted_efficiently(result: List, expected_count: int) -> None: + """Verify large batch was converted efficiently. + + Args: + result: The conversion result to verify. + expected_count: Expected number of converted items. + + """ + assert len(result) == expected_count # noqa: S101 + + converted_ids = {item["threat_id"]["data"][0] for item in result} + assert len(converted_ids) == expected_count # noqa: S101 + + +def then_private_method_converts_properly( + result: Dict, threat: SentinelOneThreat +) -> None: + """Verify private method converts threat properly. + + Args: + result: The conversion result to verify. + threat: The original threat object. + + """ + assert isinstance(result, dict) # noqa: S101 + assert "threat_id" in result # noqa: S101 + assert result["threat_id"]["type"] == "fuzzy" # noqa: S101 + assert result["threat_id"]["data"] == [threat.threat_id] # noqa: S101 + assert result["threat_id"]["score"] == 95 # noqa: S101 + + if threat.hostname: + assert "target_hostname_address" in result # noqa: S101 + assert result["target_hostname_address"]["data"] == [ # noqa: S101 + threat.hostname + ] + + +# Session and Configuration Validation Then Methods +# ------------------------------------------------- + + +def then_session_configured_properly( + client: SentinelOneClientAPI, expected_api_key: str +) -> None: + """Verify session is configured properly. + + Args: + client: The client API instance to verify. + expected_api_key: Expected API key value. + + """ + expected_auth = f"ApiToken {expected_api_key}" + assert client.session.headers["Authorization"] == expected_auth # noqa: S101 + assert client.session.headers["Content-Type"] == "application/json" # noqa: S101 + assert client.session.headers["Accept"] == "application/json" # noqa: S101 + + +def then_base_url_normalized(client: SentinelOneClientAPI, expected_base: str) -> None: + """Verify base URL is properly normalized. + + Args: + client: The client API instance to verify. + expected_base: Expected base URL without trailing slash. + + """ + assert not client.base_url.endswith("/") # noqa: S101 + assert client.base_url == expected_base # noqa: S101 + + +def then_collector_logged_initialization_success( + capfd: Any, daemon_config: Dict[str, str] +) -> None: + """Verify collector initialization was logged appropriately. + + Args: + capfd: Pytest fixture for capturing stdout and stderr output. + daemon_config: Daemon configuration to check log level. + + """ + log_records = capfd.readouterr() + if daemon_config.get("collector_log_level") in ["info", "debug"]: + registered_message = "SentinelOne Collector initialized successfully" + assert registered_message in log_records.err # noqa: S101 + + +# Error Validation Then Methods +# ----------------------------- + + +def then_validation_error_raised_with_message(error_message: str) -> None: + """Verify validation error was raised with specific message. + + Args: + error_message: Expected error message substring. + + """ + pass + + +def then_session_error_raised_with_message(error_message: str) -> None: + """Verify session error was raised with specific message. + + Args: + error_message: Expected error message substring. + + """ + pass + + +# Cleanup Then Methods +# ------------------- + + +def then_cleanup_environment_mocks(*mock_envs: Any) -> None: + """Clean up environment variable mocks. + + Args: + mock_envs: Variable number of mock environment objects to stop. + + """ + for mock_env in mock_envs: + if mock_env and hasattr(mock_env, "stop"): + mock_env.stop() diff --git a/template/tests/services/__init__.py b/template/tests/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/tests/services/conftest.py b/template/tests/services/conftest.py new file mode 100644 index 00000000..9b09251c --- /dev/null +++ b/template/tests/services/conftest.py @@ -0,0 +1,396 @@ +"""Conftest for services tests with polyfactory fixtures.""" + +from unittest.mock import Mock, patch + +import pytest +from tests.services.fixtures.factories import ( + ConfigLoaderFactory, + ExpectationResultFactory, + ExpectationTraceFactory, + MockObjectsFactory, + SentinelOneThreatFactory, + TestDataFactory, + create_test_config, +) + + +@pytest.fixture +def mock_config(): + """Provide a mock configuration for tests. + + Returns: + ConfigLoader instance with test configuration values. + + """ + return create_test_config() + + +@pytest.fixture +def mock_client_api(): + """Provide a mock SentinelOne client API. + + Returns: + Mock SentinelOne client API instance for testing. + + """ + return MockObjectsFactory.create_mock_client_api() + + +@pytest.fixture +def mock_detection_helper(): + """Provide a mock detection helper that matches by default. + + Returns: + Mock OpenAEV detection helper that returns True for matches. + + """ + return MockObjectsFactory.create_mock_detection_helper(match_result=True) + + +@pytest.fixture +def mock_detection_helper_no_match(): + """Provide a mock detection helper that doesn't match. + + Returns: + Mock OpenAEV detection helper that returns False for matches. + + """ + return MockObjectsFactory.create_mock_detection_helper(match_result=False) + + +@pytest.fixture +def sample_threat(): + """Provide a sample SentinelOne threat. + + Returns: + SentinelOneThreat instance for testing. + + """ + return SentinelOneThreatFactory.build() + + +@pytest.fixture +def sample_threats(): + """Provide a list of sample SentinelOne threats. + + Returns: + List of 2 SentinelOneThreat instances for testing. + + """ + return [SentinelOneThreatFactory.build() for _ in range(2)] + + +@pytest.fixture +def sample_expectation_result(): + """Provide a sample expectation result. + + Returns: + ExpectationResult instance for testing. + + """ + return ExpectationResultFactory.build() + + +@pytest.fixture +def sample_expectation_trace(): + """Provide a sample expectation trace. + + Returns: + ExpectationTrace instance for testing. + + """ + return ExpectationTraceFactory.build() + + +@pytest.fixture +def detection_signatures(): + """Provide sample detection expectation signatures. + + Returns: + List of signature dictionaries for detection expectations. + + """ + return TestDataFactory.create_expectation_signatures( + signature_type="parent_process_name" + ) + + +@pytest.fixture +def prevention_signatures(): + """Provide sample prevention expectation signatures. + + Returns: + List of signature dictionaries for prevention expectations. + + """ + return TestDataFactory.create_expectation_signatures( + signature_type="parent_process_name" + ) + + +@pytest.fixture +def oaev_detection_data(): + """Provide sample OAEV detection data. + + Returns: + List of OAEV-formatted detection data dictionaries. + + """ + return TestDataFactory.create_oaev_detection_data() + + +@pytest.fixture +def oaev_prevention_data(): + """Provide sample OAEV prevention data. + + Returns: + List of OAEV-formatted prevention data dictionaries. + + """ + return TestDataFactory.create_oaev_prevention_data() + + +@pytest.fixture +def mixed_sentinelone_data(): + """Provide mixed SentinelOne data (DV events + threats). + + Returns: + List containing both DeepVisibilityEvent and SentinelOneThreat instances. + + """ + return TestDataFactory.create_mixed_sentinelone_data() + + +@pytest.fixture +def mock_expectation_detection(): + """Provide a mock detection expectation. + + Returns: + Mock DetectionExpectation instance for testing. + + """ + return MockObjectsFactory.create_mock_expectation(expectation_type="detection") + + +@pytest.fixture +def mock_expectation_prevention(): + """Provide a mock prevention expectation. + + Returns: + Mock PreventionExpectation instance for testing. + + """ + return MockObjectsFactory.create_mock_expectation(expectation_type="prevention") + + +@pytest.fixture +def mock_requests_session(): + """Provide a mock requests session. + + Returns: + Mock requests.Session instance for HTTP testing. + + """ + return MockObjectsFactory.create_mock_session() + + +@pytest.fixture(autouse=True) +def mock_logging(): + """Auto-mock logging to reduce noise in tests. + + Auto-applies to all tests to prevent logging output during test execution. + + Yields: + Mock logger instance. + + """ + with patch("logging.getLogger") as mock_logger: + mock_logger.return_value = Mock() + yield mock_logger + + +@pytest.fixture +def disable_sleep(): + """Disable time.sleep in tests for faster execution. + + Patches time.sleep to prevent actual delays during testing. + + Yields: + None (context manager for sleep patching). + + """ + with patch("time.sleep"): + yield + + +@pytest.fixture(params=[1, 3, 5]) +def various_counts(request): + """Provide various counts for testing different data sizes. + + Args: + request: Pytest request object containing parameter values. + + Returns: + Integer count (1, 3, or 5) for parameterized testing. + + """ + return request.param + + +@pytest.fixture(params=[True, False]) +def match_scenarios(request): + """Provide both matching and non-matching scenarios. + + Args: + request: Pytest request object containing parameter values. + + Returns: + Boolean value (True or False) for match testing scenarios. + + """ + return request.param + + +@pytest.fixture(params=["detection", "prevention"]) +def expectation_types(request): + """Provide different expectation types. + + Args: + request: Pytest request object containing parameter values. + + Returns: + String expectation type ("detection" or "prevention"). + + """ + return request.param + + +@pytest.fixture +def config_factory(): + """Provide the ConfigLoaderFactory for creating configs in tests. + + Returns: + ConfigLoaderFactory class for generating test configurations. + + """ + return ConfigLoaderFactory + + +@pytest.fixture +def threat_factory(): + """Provide the SentinelOneThreatFactory for creating threats. + + Returns: + SentinelOneThreatFactory class for generating test threats. + + """ + return SentinelOneThreatFactory + + +@pytest.fixture +def expectation_result_factory(): + """Provide the ExpectationResultFactory for creating results. + + Returns: + ExpectationResultFactory class for generating test results. + + """ + return ExpectationResultFactory + + +@pytest.fixture +def expectation_trace_factory(): + """Provide the ExpectationTraceFactory for creating traces. + + Returns: + ExpectationTraceFactory class for generating test traces. + + """ + return ExpectationTraceFactory + + +@pytest.fixture +def test_data_factory(): + """Provide the TestDataFactory for creating test data combinations. + + Returns: + TestDataFactory class for generating complex test data scenarios. + + """ + return TestDataFactory + + +@pytest.fixture +def mock_objects_factory(): + """Provide the MockObjectsFactory for creating mock objects. + + Returns: + MockObjectsFactory class for generating mock instances. + + """ + return MockObjectsFactory + + +@pytest.fixture(autouse=True) +def cleanup_mocks(): + """Auto-cleanup mocks after each test. + + Auto-applies to all tests to ensure proper mock cleanup. + + Yields: + None (context manager for cleanup operations). + + """ + yield + + +@pytest.fixture +def api_error_responses(): + """Provide various API error responses for testing error handling. + + Returns: + Dictionary mapping HTTP status codes to error response data. + + """ + return { + "400": { + "status_code": 400, + "text": "Bad Request", + "json": {"errors": ["Bad request"]}, + }, + "401": { + "status_code": 401, + "text": "Unauthorized", + "json": {"errors": ["Unauthorized"]}, + }, + "403": { + "status_code": 403, + "text": "Forbidden", + "json": {"errors": ["Forbidden"]}, + }, + "404": { + "status_code": 404, + "text": "Not Found", + "json": {"errors": ["Not found"]}, + }, + "500": { + "status_code": 500, + "text": "Internal Server Error", + "json": {"errors": ["Server error"]}, + }, + } + + +@pytest.fixture +def network_errors(): + """Provide various network errors for testing error handling. + + Returns: + List of different exception types for network error testing. + + """ + return [ + ConnectionError("Connection failed"), + TimeoutError("Request timeout"), + Exception("Generic network error"), + ] diff --git a/template/tests/services/fixtures/__init__.py b/template/tests/services/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/tests/services/fixtures/factories.py b/template/tests/services/fixtures/factories.py new file mode 100644 index 00000000..6062d882 --- /dev/null +++ b/template/tests/services/fixtures/factories.py @@ -0,0 +1,324 @@ +"""Essential polyfactory factories for SentinelOne models and test fixtures.""" + +import os +import uuid +from typing import Any +from unittest.mock import Mock + +from polyfactory import Use +from polyfactory.factories.pydantic_factory import ModelFactory +from src.collector.models import ExpectationResult, ExpectationTrace +from src.models.configs.collector_configs import _ConfigLoaderOAEV +from src.models.configs.config_loader import ConfigLoader, ConfigLoaderCollector +from src.models.configs.sentinelone_configs import _ConfigLoaderSentinelOne +from src.services.model_threat import SentinelOneThreat + + +class ConfigLoaderOAEVFactory(ModelFactory[_ConfigLoaderOAEV]): + """Factory for OpenAEV configuration. + + Creates test instances of OpenAEV configuration with required + environment variables automatically set. + """ + + __check_model__ = False + + @classmethod + def build(cls, **kwargs): + """Build the model with required environment variables set. + + Args: + **kwargs: Additional keyword arguments for model creation. + + Returns: + _ConfigLoaderOAEV instance with test configuration. + + """ + os.environ["OPENAEV_URL"] = "https://test-openaev.example.com" + os.environ["OPENAEV_TOKEN"] = "test-openaev-token-12345" # noqa: S105 + return super().build(**kwargs) + + +class ConfigLoaderSentinelOneFactory(ModelFactory[_ConfigLoaderSentinelOne]): + """Factory for SentinelOne configuration. + + Creates test instances of SentinelOne configuration with required + environment variables automatically set. + """ + + __check_model__ = False + + @classmethod + def build(cls, **kwargs): + """Build the model with required environment variables set. + + Args: + **kwargs: Additional keyword arguments for model creation. + + Returns: + _ConfigLoaderSentinelOne instance with test configuration. + + """ + os.environ["SENTINELONE_API_KEY"] = "test-sentinelone-api-key" + os.environ["SENTINELONE_BASE_URL"] = "https://test-sentinelone.example.com" + return super().build(**kwargs) + + +class ConfigLoaderCollectorFactory(ModelFactory[ConfigLoaderCollector]): + """Factory for Collector configuration. + + Creates test instances of collector configuration with auto-generated + UUIDs and sensible defaults. + """ + + __check_model__ = False + + id = Use(lambda: f"sentinelone--{uuid.uuid4()}") + name = "SentinelOne" + + +class ConfigLoaderFactory(ModelFactory[ConfigLoader]): + """Factory for main configuration. + + Creates complete test configuration instances combining OpenAEV, + collector, and SentinelOne settings using subfactories. + """ + + __check_model__ = False + + openaev = Use(ConfigLoaderOAEVFactory.build) + collector = Use(ConfigLoaderCollectorFactory.build) + sentinelone = Use(ConfigLoaderSentinelOneFactory.build) + + +class SentinelOneThreatFactory(ModelFactory[SentinelOneThreat]): + """Factory for SentinelOne threats. + + Creates test instances of SentinelOne threat objects with + auto-generated threat IDs. + """ + + __check_model__ = False + + +class ExpectationResultFactory(ModelFactory[ExpectationResult]): + """Factory for ExpectationResult. + + Creates test instances of expectation processing results with + valid expectation IDs and configurable validation status. + """ + + __check_model__ = False + + expectation_id = Use(lambda: str(uuid.uuid4())) + is_valid = True + error_message = None + matched_alerts = Use(lambda: []) + + +class ExpectationTraceFactory(ModelFactory[ExpectationTrace]): + """Factory for ExpectationTrace. + + Creates test instances of expectation traces for OpenAEV + with properly formatted trace data. + """ + + __check_model__ = False + + +class MockObjectsFactory: + """Factory for creating mock objects. + + Provides static methods for creating various mock objects + used throughout the test suite. + """ + + @staticmethod + def create_mock_client_api(): + """Create mock SentinelOne client API. + + Returns: + Mock SentinelOneClientAPI instance with basic attributes set. + + """ + mock_client = Mock() + mock_client.base_url = "https://test-api.example.com" + mock_client.session = Mock() + mock_client.session.headers = {} + return mock_client + + @staticmethod + def create_mock_detection_helper(match_result: bool = True): + """Create mock detection helper. + + Args: + match_result: Whether the helper should return matches (default True). + + Returns: + Mock OpenAEVDetectionHelper instance. + + """ + mock_helper = Mock() + mock_helper.match_alert_elements.return_value = match_result + return mock_helper + + @staticmethod + def create_mock_expectation( + expectation_type: str = "detection", expectation_id: str = None + ): + """Create mock expectation for testing. + + Args: + expectation_type: Type of expectation ("detection" or "prevention"). + expectation_id: Optional custom expectation ID. + + Returns: + Mock expectation object with required attributes. + + """ + mock_expectation = Mock() + mock_expectation.inject_expectation_id = expectation_id or str(uuid.uuid4()) + mock_expectation.inject_expectation_signatures = [] + mock_expectation.expectation_type = expectation_type + return mock_expectation + + @staticmethod + def create_mock_session(): + """Create mock requests session. + + Returns: + Mock requests.Session instance with headers attribute. + + """ + mock_session = Mock() + mock_session.headers = {} + return mock_session + + +class TestDataFactory: + """Factory for creating essential test data. + + Provides static methods for creating complex test data structures + that simulate real-world scenarios. + """ + + @staticmethod + def create_expectation_signatures( + signature_type: str = "parent_process_name", signature_value: str = None + ) -> list[dict[str, Any]]: + """Create expectation signatures. + + Args: + signature_type: Type of signature to create. + signature_value: Optional custom signature value. + + Returns: + List of signature dictionaries for testing. + + """ + if signature_value is None: + signature_value = f"test-{signature_type}-{uuid.uuid4().hex[:8]}" + + return [{"type": signature_type, "value": signature_value}] + + @staticmethod + def create_oaev_detection_data() -> list[dict[str, Any]]: + """Create OAEV detection data. + + Returns: + List of OAEV-formatted detection data dictionaries. + + """ + return [ + { + "parent_process_name": { + "type": "simple", + "data": [f"oaev-implant-test-{uuid.uuid4().hex[:8]}"], + }, + "threat_id": { + "type": "simple", + "data": [f"threat-{uuid.uuid4().hex[:8]}"], + }, + } + ] + + @staticmethod + def create_oaev_prevention_data() -> list[dict[str, Any]]: + """Create OAEV prevention data. + + Returns: + List of OAEV-formatted prevention data dictionaries. + + """ + return [ + { + "parent_process_name": { + "type": "simple", + "data": [f"oaev-implant-test-{uuid.uuid4().hex[:8]}"], + }, + "threat_id": { + "type": "simple", + "data": [f"threat-{uuid.uuid4().hex[:8]}"], + }, + } + ] + + @staticmethod + def create_mixed_sentinelone_data() -> list[Any]: + """Create mixed SentinelOne data (DV events + threats). + + Returns: + List containing both DV event dicts and SentinelOneThreat instances. + + """ + return [] + + +# Helper functions +def create_test_config(**overrides) -> ConfigLoader: + """Create test configuration. + + Args: + **overrides: Configuration values to override defaults. + + Returns: + ConfigLoader instance with test configuration. + + """ + return ConfigLoaderFactory.build(**overrides) + + +def create_test_dv_events(count: int = 1) -> list[dict]: + """Create test Deep Visibility events. + + Args: + count: Number of DV events to create (default 1). + + Returns: + List of dictionary representations of DV events for testing. + + """ + events = [] + for i in range(count): + events.append( + { + "src_proc_parent_name": f"oaev-implant-test-{uuid.uuid4().hex[:8]}", + "src_proc_name": f"process-{i}.exe", + "event_type": "process_creation", + "timestamp": "2024-01-01T10:00:00Z", + } + ) + return events + + +def create_test_threats(count: int = 1) -> list[SentinelOneThreat]: + """Create test SentinelOne threats. + + Args: + count: Number of threats to create (default 1). + + Returns: + List of SentinelOneThreat instances for testing. + + """ + return [SentinelOneThreatFactory.build() for _ in range(count)] diff --git a/template/tests/services/test_client_api.py b/template/tests/services/test_client_api.py new file mode 100644 index 00000000..9c2fcac0 --- /dev/null +++ b/template/tests/services/test_client_api.py @@ -0,0 +1,117 @@ +"""Essential tests for SentinelOne Client API service - Gherkin GWT Format.""" + +from requests import Session +from src.services.client_api import SentinelOneClientAPI +from tests.gwt_shared import given_test_config # Given methods +from tests.gwt_shared import then_client_api_initialized_successfully + +# -------- +# Scenarios +# -------- + + +# Scenario: Initialize client API with valid configuration +def test_initialize_client_api_with_valid_config(): + """Scenario: Initialize client API with valid configuration.""" + # Given: A valid configuration is available + config = _given_valid_config_for_client_api() + + # When: I initialize the client API + client = _when_initialize_client_api_with_config(config) + + # Then: The client API should be initialized successfully + _then_client_api_initialized_with_valid_config(client, config) + + +# Scenario: Initialize with invalid configuration raises error +def test_initialize_with_invalid_config(): + """Scenario: Initialize with invalid configuration raises error.""" + # Given: An invalid configuration (None) + invalid_config = _given_invalid_config() + + # When: I attempt to initialize the client API + # Then: An AttributeError should be raised + _when_initialize_client_api_then_attribute_error_raised(invalid_config) + + +# -------- +# Given Methods +# -------- + + +# Given: A valid configuration is available +def _given_valid_config_for_client_api(): + """Create a valid configuration for client API testing. + + Returns: + Test configuration object. + + """ + return given_test_config() + + +# Given: An invalid configuration (None) +def _given_invalid_config(): + """Create an invalid configuration. + + Returns: + None (invalid configuration). + + """ + return None + + +# -------- +# When Methods +# -------- + + +# When: I initialize the client API with configuration +def _when_initialize_client_api_with_config(config): + """Initialize client API with given configuration. + + Args: + config: Configuration object to use. + + Returns: + Initialized SentinelOneClientAPI instance. + + """ + return SentinelOneClientAPI(config=config) + + +# When: I attempt to initialize with invalid config and expect AttributeError +def _when_initialize_client_api_then_attribute_error_raised(invalid_config): + """Attempt to initialize with invalid config and expect AttributeError. + + Args: + invalid_config: Invalid configuration to test. + + """ + import pytest + + with pytest.raises(AttributeError): + SentinelOneClientAPI(config=invalid_config) + + +# -------- +# Then Methods +# -------- + + +# Then: The client API should be initialized successfully with valid config +def _then_client_api_initialized_with_valid_config(client, config): + """Verify client API was initialized successfully with valid configuration. + + Args: + client: The client API instance to verify. + config: The configuration used for initialization. + + """ + then_client_api_initialized_successfully(client) + + assert client.config == config # noqa: S101 + assert client.base_url == str(config.sentinelone.base_url).rstrip("/") # noqa: S101 + assert client.api_key == config.sentinelone.api_key.get_secret_value() # noqa: S101 + assert isinstance(client.session, Session) # noqa: S101 + assert client.time_window == config.sentinelone.time_window # noqa: S101 diff --git a/template/tests/services/test_converter.py b/template/tests/services/test_converter.py new file mode 100644 index 00000000..98d2a35b --- /dev/null +++ b/template/tests/services/test_converter.py @@ -0,0 +1,216 @@ +"""Essential tests for SentinelOne Converter services - Gherkin GWT Format.""" + +import pytest +from src.services.converter import SentinelOneConverter +from src.services.exception import SentinelOneValidationError +from src.services.model_threat import SentinelOneThreat + +# -------- +# Scenarios +# -------- + + +# Scenario: Initialize converter successfully +def test_initialize_converter(): + """Scenario: Initialize converter successfully.""" + # Given: All dependencies are available + _given_converter_dependencies_available() + + # When: I initialize the converter + converter = _when_initialize_converter() + + # Then: The converter should be initialized successfully + _then_converter_initialized_successfully(converter) + + +# Scenario: Convert empty threats list +def test_convert_empty_threats(): + """Scenario: Convert empty threats list.""" + # Given: A converter is available + converter = _given_initialized_converter() + + # When: I convert an empty threats list + result = _when_convert_threats_to_oaev(converter, []) + + # Then: An empty list should be returned + _then_empty_list_returned(result) + + +# Scenario: Convert single threat with complete data +def test_convert_single_threat_complete_data(): + """Scenario: Convert single threat with complete data.""" + # Given: A converter is available + converter = _given_initialized_converter() + # Given: A threat with complete data + threat = _given_threat_with_complete_data() + + # When: I convert the threat to OAEV format + result = _when_convert_threats_to_oaev(converter, [threat]) + + # Then: The threat should be converted with all fields + _then_single_threat_converted_completely(result, threat) + + +# Scenario: Convert invalid data type +def test_convert_invalid_data_type(): + """Scenario: Convert invalid data type.""" + # Given: A converter is available + converter = _given_initialized_converter() + # Given: Invalid input data (not a list) + invalid_data = _given_invalid_input_data() + + # When: I attempt to convert invalid data + # Then: A validation error should be raised + _when_convert_invalid_data_then_validation_error_raised(converter, invalid_data) + + +# -------- +# Given Methods +# -------- + + +# Given: All dependencies are available +def _given_converter_dependencies_available(): + """Ensure all converter dependencies are available.""" + pass + + +# Given: A converter is available +def _given_initialized_converter() -> SentinelOneConverter: + """Create and return an initialized converter. + + Returns: + Initialized SentinelOneConverter instance. + + """ + return SentinelOneConverter() + + +# Given: A threat with complete data +def _given_threat_with_complete_data() -> SentinelOneThreat: + """Create a threat with complete data. + + Returns: + SentinelOneThreat with threat_id and hostname. + + """ + return SentinelOneThreat( + threat_id="complete_threat_123", hostname="complete-host.example.com" + ) + + +# Given: Invalid input data (not a list) +def _given_invalid_input_data() -> str: + """Create invalid input data for testing. + + Returns: + Invalid data (string instead of list). + + """ + return "invalid_string_data" + + +# -------- +# When Methods +# -------- + + +# When: I initialize the converter +def _when_initialize_converter() -> SentinelOneConverter: + """Initialize the converter. + + Returns: + Initialized SentinelOneConverter instance. + + """ + return SentinelOneConverter() + + +# When: I convert threats to OAEV format +def _when_convert_threats_to_oaev( + converter: SentinelOneConverter, threats: list +) -> list: + """Convert threats to OAEV format. + + Args: + converter: The converter instance. + threats: List of threats to convert. + + Returns: + List of converted OAEV format data. + + """ + return converter.convert_threats_to_oaev(threats) + + +# When: I attempt to convert invalid data and expect validation error +def _when_convert_invalid_data_then_validation_error_raised( + converter: SentinelOneConverter, invalid_data: str +) -> None: + """Attempt to convert invalid data and expect validation error. + + Args: + converter: The converter instance. + invalid_data: Invalid input data. + + """ + with pytest.raises(SentinelOneValidationError) as exc_info: + converter.convert_threats_to_oaev(invalid_data) + + assert "threats must be a list" in str(exc_info.value) # noqa: S101 + + +# -------- +# Then Methods +# -------- + + +# Then: The converter should be initialized successfully +def _then_converter_initialized_successfully(converter: SentinelOneConverter) -> None: + """Verify the converter was initialized successfully. + + Args: + converter: The converter instance to verify. + + """ + assert converter is not None # noqa: S101 + assert converter.logger is not None # noqa: S101 + + +# Then: An empty list should be returned +def _then_empty_list_returned(result: list) -> None: + """Verify an empty list was returned. + + Args: + result: The conversion result to verify. + + """ + assert result == [] # noqa: S101 + + +# Then: The threat should be converted with all fields +def _then_single_threat_converted_completely( + result: list, threat: SentinelOneThreat +) -> None: + """Verify single threat was converted with all fields. + + Args: + result: The conversion result to verify. + threat: The original threat object. + + """ + assert len(result) == 1 # noqa: S101 + + converted = result[0] + + assert "threat_id" in converted # noqa: S101 + assert converted["threat_id"]["type"] == "fuzzy" # noqa: S101 + assert converted["threat_id"]["data"] == [threat.threat_id] # noqa: S101 + assert converted["threat_id"]["score"] == 95 # noqa: S101 + + assert "target_hostname_address" in converted # noqa: S101 + assert converted["target_hostname_address"]["type"] == "fuzzy" # noqa: S101 + assert converted["target_hostname_address"]["data"] == [ # noqa: S101 + threat.hostname + ] + assert converted["target_hostname_address"]["score"] == 95 # noqa: S101 diff --git a/template/tests/services/test_expectation_service.py b/template/tests/services/test_expectation_service.py new file mode 100644 index 00000000..b1b305be --- /dev/null +++ b/template/tests/services/test_expectation_service.py @@ -0,0 +1,797 @@ +"""Essential tests for SentinelOne Expectation Service - Gherkin GWT Format.""" + +from unittest.mock import Mock +from uuid import uuid4 + +import pytest +from pyoaev.signatures.types import SignatureTypes +from src.services.expectation_service import ( + ExpectationResult, + SentinelOneExpectationService, +) +from src.services.model_threat import SentinelOneThreat +from tests.gwt_shared import ( + given_initialized_expectation_service, + given_test_config, + then_expectation_service_initialized_successfully, +) + +# -------- +# Scenarios +# -------- + + +# Scenario: Initialize expectation service with valid configuration +def test_initialize_expectation_service_with_valid_config(): + """Scenario: Initialize expectation service with valid configuration.""" + # Given: A valid configuration is available + config = _given_valid_config_for_expectation_service() + + # When: I initialize the expectation service + service = _when_initialize_expectation_service(config) + + # Then: The expectation service should be initialized successfully + _then_expectation_service_initialized_with_valid_config(service, config) + + +# Scenario: Initialize with invalid configuration raises error +def test_initialize_with_invalid_config(): + """Scenario: Initialize with invalid configuration raises error.""" + # Given: An invalid configuration (None) + invalid_config = _given_invalid_config() + + # When: I attempt to initialize the expectation service + # Then: An AttributeError should be raised + _when_initialize_expectation_service_then_attribute_error_raised(invalid_config) + + +# Scenario: Handle single detection expectation +def test_handle_single_detection_expectation(): + """Scenario: Handle single detection expectation.""" + # Given: An initialized expectation service + service = _given_initialized_expectation_service() + # Given: A detection helper + detection_helper = _given_mock_detection_helper() + # Given: Mock threats are available + _given_mock_threats_for_service(service) + # Given: A detection expectation + expectation = _given_detection_expectation() + + # When: I handle the detection expectation + result = _when_handle_batch_expectations(service, [expectation], detection_helper) + + # Then: A detection result should be returned + _then_detection_result_returned(result, expectation) + + +# Scenario: Handle prevention expectation +def test_handle_prevention_expectation(): + """Scenario: Handle prevention expectation.""" + # Given: An initialized expectation service + service = _given_initialized_expectation_service() + # Given: A detection helper + detection_helper = _given_mock_detection_helper() + # Given: Mock mitigated threats are available + _given_mock_mitigated_threats_for_service(service) + # Given: A prevention expectation + expectation = _given_prevention_expectation() + + # When: I handle the prevention expectation + result = _when_handle_batch_expectations(service, [expectation], detection_helper) + + # Then: A prevention result should be returned + _then_prevention_result_returned(result, expectation) + + +# Scenario: Handle static threats with Deep Visibility enabled +def test_handle_static_threats_with_deep_visibility_enabled(): + """Scenario: Handle static threats with Deep Visibility enabled.""" + # Given: A detection helper + detection_helper = _given_mock_detection_helper() + # Given: A static expectation + expectation = _given_static_expectation() + + # When: I handle the static expectation with Deep Visibility enabled + with _given_expectation_service_with_deep_visibility_enabled() as service: + mock_static_threats = _given_mock_static_threats_for_service(service) + mock_dv_events = _given_mock_deep_visibility_events_for_service(service) + + with mock_static_threats, mock_dv_events: + result = _when_handle_batch_expectations( + service, [expectation], detection_helper + ) + + # Then: A static result with Deep Visibility events should be returned + _then_static_result_with_deep_visibility_returned(result, expectation) + + +# Scenario: Handle static threats with Deep Visibility disabled +def test_handle_static_threats_with_deep_visibility_disabled(): + """Scenario: Handle static threats with Deep Visibility disabled.""" + # Given: A detection helper + detection_helper = _given_mock_detection_helper() + # Given: A static expectation + expectation = _given_static_expectation() + + # When: I handle the static expectation with Deep Visibility disabled + with _given_expectation_service_with_deep_visibility_disabled() as service: + mock_static_threats = _given_mock_static_threats_for_service(service) + + with mock_static_threats: + result = _when_handle_batch_expectations( + service, [expectation], detection_helper + ) + + # Then: A static result without Deep Visibility events should be returned + _then_static_result_without_deep_visibility_returned(result, expectation) + + +# Scenario: Verify Deep Visibility fetcher is called when enabled +def test_deep_visibility_fetcher_called_when_enabled(): + """Scenario: Verify Deep Visibility fetcher is called when enabled.""" + # Given: A detection helper + detection_helper = _given_mock_detection_helper() + # Given: A static expectation + static_expectation = _given_static_expectation() + + # When: I handle the static expectation with Deep Visibility enabled + with _given_expectation_service_with_deep_visibility_enabled() as service: + mock_static_threats = _given_mock_static_threats_for_service(service) + mock_dv_events = _given_mock_deep_visibility_events_for_service(service) + + with mock_static_threats, mock_dv_events as dv_mock: + _when_handle_batch_expectations( + service, [static_expectation], detection_helper + ) + + # Then: Deep Visibility fetcher should have been called + dv_mock.assert_called_once() + + +# Scenario: Verify Deep Visibility fetcher is not called when disabled +def test_deep_visibility_fetcher_not_called_when_disabled(): + """Scenario: Verify Deep Visibility fetcher is not called when disabled.""" + # Given: A detection helper + detection_helper = _given_mock_detection_helper() + # Given: A static expectation + static_expectation = _given_static_expectation() + + # When: I handle the static expectation with Deep Visibility disabled + with _given_expectation_service_with_deep_visibility_disabled() as service: + mock_static_threats = _given_mock_static_threats_for_service(service) + mock_dv_events = _given_mock_deep_visibility_events_for_service(service) + + with mock_static_threats, mock_dv_events as dv_mock: + _when_handle_batch_expectations( + service, [static_expectation], detection_helper + ) + + # Then: Deep Visibility fetcher should not have been called + dv_mock.assert_not_called() + + +# Scenario: Match threats to expectations +def test_match_threats_to_expectations(): + """Scenario: Match threats to expectations.""" + # Given: An initialized expectation service + service = _given_initialized_expectation_service() + # Given: Threats and expectations + threats, expectations = _given_threats_and_expectations() + + # When: I match threats to expectations + matches = _when_match_threats_to_expectations(service, threats, expectations) + + # Then: Proper matches should be found + _then_proper_matches_found(matches, threats, expectations) + # Then: The match should succeed without requiring mitigation + _then_match_succeeds_without_mitigation_requirement(matches) + + +# -------- +# Given Methods +# -------- + + +# Given: A valid configuration is available +def _given_valid_config_for_expectation_service(): + """Create a valid configuration for expectation service testing. + + Returns: + Test configuration object. + + """ + return given_test_config() + + +# Given: An invalid configuration (None) +def _given_invalid_config(): + """Create an invalid configuration. + + Returns: + None (invalid configuration). + + """ + return None + + +# Given: An initialized expectation service +def _given_initialized_expectation_service(): + """Create an initialized expectation service. + + Returns: + Initialized SentinelOneExpectationService instance. + + """ + return given_initialized_expectation_service() + + +# Given: A detection helper +def _given_mock_detection_helper(): + """Create a mock detection helper. + + Returns: + Mock detection helper instance. + + """ + return Mock() + + +# Given: Mock threats are available +def _given_mock_threats_for_service(service): + """Set up mock threats for the service. + + Args: + service: The expectation service instance. + + """ + mock_threats = [ + SentinelOneThreat( + threat_id="test_threat_1", + hostname="target-host.example.com", + is_mitigated=False, + ) + ] + service.threat_fetcher.fetch_threats_for_time_window = Mock( + return_value=mock_threats + ) + + +# Given: Mock mitigated threats are available +def _given_mock_mitigated_threats_for_service(service): + """Set up mock mitigated threats for the service. + + Args: + service: The expectation service instance. + + """ + mock_threats = [ + SentinelOneThreat( + threat_id="test_threat_1", + hostname="target-host.example.com", + is_mitigated=True, + ) + ] + service.threat_fetcher.fetch_threats_for_time_window = Mock( + return_value=mock_threats + ) + + +# Given: A detection expectation +def _given_detection_expectation(): + """Create a detection expectation. + + Returns: + Mock detection expectation. + + """ + hostname_sig = _create_mock_signature( + SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, "target-host.example.com" + ) + end_date_sig = _create_mock_signature( + Mock(value="end_date"), "2024-01-01T12:00:00Z" + ) + + expectation = _create_mock_expectation( + expectation_id="detection_test_1", signatures=[hostname_sig, end_date_sig] + ) + return expectation + + +# Given: A prevention expectation +def _given_prevention_expectation(): + """Create a prevention expectation. + + Returns: + Mock prevention expectation. + + """ + hostname_sig = _create_mock_signature( + SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, "target-host.example.com" + ) + end_date_sig = _create_mock_signature( + Mock(value="end_date"), "2024-01-01T12:00:00Z" + ) + + expectation = _create_mock_expectation( + expectation_id="prevention_test_1", signatures=[hostname_sig, end_date_sig] + ) + expectation.is_prevention = True + return expectation + + +# Given: Threats and expectations +def _given_threats_and_expectations(): + """Create threats and expectations for matching tests. + + Returns: + Tuple of (threats, expectations). + + """ + threats = [ + SentinelOneThreat( + threat_id="match_threat_1", + hostname="match-host.example.com", + is_mitigated=False, + ) + ] + + hostname_sig = _create_mock_signature( + SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, "match-host.example.com" + ) + expectation = _create_mock_expectation(signatures=[hostname_sig]) + expectations = [expectation] + + return threats, expectations + + +# Given: An unmitigated threat +def _given_unmitigated_threat(): + """Create an unmitigated threat. + + Returns: + SentinelOneThreat instance that is not mitigated. + + """ + return SentinelOneThreat( + threat_id="unmitigated_threat", + hostname="unmitigated-host.example.com", + is_mitigated=False, + ) + + +# Given: An expectation service with Deep Visibility enabled +def _given_expectation_service_with_deep_visibility_enabled(): + """Create an expectation service with Deep Visibility enabled. + + Returns: + Context manager that yields SentinelOneExpectationService with Deep Visibility enabled. + + """ + import os + from contextlib import contextmanager + from unittest.mock import patch + + @contextmanager + def _service_context(): + with patch.dict( + os.environ, {"SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH": "true"} + ): + config = given_test_config() + yield SentinelOneExpectationService(config) + + return _service_context() + + +# Given: An expectation service with Deep Visibility disabled +def _given_expectation_service_with_deep_visibility_disabled(): + """Create an expectation service with Deep Visibility disabled. + + Returns: + Context manager that yields SentinelOneExpectationService with Deep Visibility disabled. + + """ + import os + from contextlib import contextmanager + from unittest.mock import patch + + @contextmanager + def _service_context(): + with patch.dict( + os.environ, {"SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH": "false"} + ): + config = given_test_config() + yield SentinelOneExpectationService(config) + + return _service_context() + + +# Given: A static expectation +def _given_static_expectation(): + """Create a static expectation. + + Returns: + Mock static expectation. + + """ + hostname_sig = _create_mock_signature( + SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, "static-host.example.com" + ) + end_date_sig = _create_mock_signature( + Mock(value="end_date"), "2024-01-01T12:00:00Z" + ) + + expectation = _create_mock_expectation( + expectation_id="static_test_1", signatures=[hostname_sig, end_date_sig] + ) + return expectation + + +# Given: Mock static threats for service +def _given_mock_static_threats_for_service(service): + """Set up mock static threats for the service. + + Args: + service: The expectation service to mock. + + """ + from unittest.mock import patch + + static_threats = [ + SentinelOneThreat( + threat_id="static_threat_1", + hostname="static-host.example.com", + is_mitigated=False, + is_static=True, + sha1="a1b2c3d4e5f6789012345678901234567890abcd", + ), + SentinelOneThreat( + threat_id="static_threat_2", + hostname="static-host.example.com", + is_mitigated=False, + is_static=True, + sha1="b2c3d4e5f6789012345678901234567890abcdef", + ), + ] + + return patch.object( + service.threat_fetcher, + "fetch_threats_for_time_window", + return_value=static_threats, + ) + + +# Given: Mock mixed threats for service +def _given_mock_mixed_threats_for_service(service): + """Set up mock mixed threats (static and non-static) for the service. + + Args: + service: The expectation service to mock. + + """ + from unittest.mock import patch + + mixed_threats = [ + SentinelOneThreat( + threat_id="static_threat_1", + hostname="mixed-host.example.com", + is_mitigated=False, + is_static=True, + sha1="a1b2c3d4e5f6789012345678901234567890abcd", + ), + SentinelOneThreat( + threat_id="behavior_threat_1", + hostname="mixed-host.example.com", + is_mitigated=False, + is_static=False, + sha1=None, + ), + ] + + return patch.object( + service.threat_fetcher, + "fetch_threats_for_time_window", + return_value=mixed_threats, + ) + + +# Given: Mock Deep Visibility events for service +def _given_mock_deep_visibility_events_for_service(service): + """Set up mock Deep Visibility events for the service. + + Args: + service: The expectation service to mock. + + """ + from unittest.mock import patch + + mock_dv_events = { + "a1b2c3d4e5f6789012345678901234567890abcd": [ + { + "fileSha1": "a1b2c3d4e5f6789012345678901234567890abcd", + "processName": "oaev-implant-test.exe", + "timestamp": "2024-01-01T12:00:00Z", + "eventType": "Process Creation", + "parentProcessName": "cmd.exe", + } + ], + "b2c3d4e5f6789012345678901234567890abcdef": [ + { + "fileSha1": "b2c3d4e5f6789012345678901234567890abcdef", + "processName": "oaev-implant-test2.exe", + "timestamp": "2024-01-01T12:01:00Z", + "eventType": "Process Creation", + "parentProcessName": "powershell.exe", + } + ], + } + + return patch.object( + service.deep_visibility_fetcher, + "fetch_events_for_batch_sha1", + return_value=mock_dv_events, + ) + + +# Given: Mock threat events for service +def _given_mock_threat_events_for_service(service): + """Set up mock threat events for the service. + + Args: + service: The expectation service to mock. + + """ + from unittest.mock import patch + + mock_threat_events = { + "behavior_threat_1": [ + { + "processName": "oaev-implant-behavior.exe", + "parentProcessName": "cmd.exe", + "timestamp": "2024-01-01T12:00:00Z", + "eventType": "Process Creation", + } + ] + } + + return patch.object( + service.threat_events_fetcher, + "fetch_events_for_threat", + return_value=mock_threat_events, + ) + + +# -------- +# When Methods +# -------- + + +# When: I initialize the expectation service +def _when_initialize_expectation_service(config): + """Initialize expectation service with given configuration. + + Args: + config: Configuration object to use. + + Returns: + Initialized SentinelOneExpectationService instance. + + """ + return SentinelOneExpectationService(config=config) + + +# When: I attempt to initialize with invalid config and expect AttributeError +def _when_initialize_expectation_service_then_attribute_error_raised(invalid_config): + """Attempt to initialize with invalid config and expect AttributeError. + + Args: + invalid_config: Invalid configuration to test. + + """ + with pytest.raises(AttributeError): + SentinelOneExpectationService(config=invalid_config) + + +# When: I handle batch expectations +def _when_handle_batch_expectations(service, expectations, detection_helper): + """Handle batch expectations using the service. + + Args: + service: The expectation service instance. + expectations: List of expectations to handle. + detection_helper: The detection helper to use. + + Returns: + List of expectation results. + + """ + results, _ = service.handle_batch_expectations(expectations, detection_helper) + return results + + +# When: I match threats to expectations +def _when_match_threats_to_expectations(service, threats, expectations): + """Match threats to expectations. + + Args: + service: The expectation service instance. + threats: List of threats. + expectations: List of expectations. + + Returns: + List of matches. + + """ + threat_events = {threat.threat_id: [] for threat in threats} + return service._match_threats_to_expectations( + expectations, threats, threat_events, "detection" + ) + + +# When: I check if expectation matches threat data +def _when_check_expectation_matches_threat(service, expectation, threat): + """Check if expectation matches threat data. + + Args: + service: The expectation service instance. + expectation: The expectation to check. + threat: The threat to match against. + + Returns: + Boolean indicating if there's a match. + + """ + events = [] + expectation_type = ( + "prevention" + if hasattr(expectation, "is_prevention") and expectation.is_prevention + else "detection" + ) + return service._expectation_matches_threat_data( + expectation, threat, events, expectation_type + ) + + +# -------- +# Then Methods +# -------- + + +# Then: The expectation service should be initialized successfully with valid config +def _then_expectation_service_initialized_with_valid_config(service, config): + """Verify expectation service was initialized successfully. + + Args: + service: The service instance to verify. + config: The configuration used for initialization. + + """ + then_expectation_service_initialized_successfully(service) + assert service.batch_size == config.sentinelone.expectation_batch_size # noqa: S101 + + +# Then: A detection result should be returned +def _then_detection_result_returned(result, expectation): + """Verify a detection result was returned. + + Args: + result: The result to verify. + expectation: The original expectation. + + """ + assert len(result) == 1 # noqa: S101 + assert isinstance(result[0], ExpectationResult) # noqa: S101 + assert result[0].expectation_id == expectation.inject_expectation_id # noqa: S101 + + +# Then: A prevention result should be returned +def _then_prevention_result_returned(result, expectation): + """Verify a prevention result was returned. + + Args: + result: The result to verify. + expectation: The original expectation. + + """ + assert len(result) == 1 # noqa: S101 + assert isinstance(result[0], ExpectationResult) # noqa: S101 + assert result[0].expectation_id == expectation.inject_expectation_id # noqa: S101 + + +# Then: Proper matches should be found +def _then_proper_matches_found(matches, threats, expectations): + """Verify proper matches were found. + + Args: + matches: The found matches. + threats: The original threats. + expectations: The original expectations. + + """ + assert len(matches) > 0 # noqa: S101 + + +# Then: The match should succeed without requiring mitigation +def _then_match_succeeds_without_mitigation_requirement(matches): + """Verify match succeeds without requiring mitigation. + + Args: + matches: The match results. + + """ + assert matches is not None # noqa: S101 + + +# Then: A static result with Deep Visibility events should be returned +def _then_static_result_with_deep_visibility_returned(result, expectation): + """Verify a static result with Deep Visibility events was returned. + + Args: + result: The result to verify. + expectation: The original expectation. + + """ + assert len(result) == 1 # noqa: S101 + assert isinstance(result[0], ExpectationResult) # noqa: S101 + assert result[0].expectation_id == expectation.inject_expectation_id # noqa: S101 + assert result[0].is_valid # noqa: S101 + assert len(result[0].matched_alerts) > 0 # noqa: S101 + + +# Then: A static result without Deep Visibility events should be returned +def _then_static_result_without_deep_visibility_returned(result, expectation): + """Verify a static result without Deep Visibility events was returned. + + Args: + result: The result to verify. + expectation: The original expectation. + + """ + assert len(result) == 1 # noqa: S101 + assert isinstance(result[0], ExpectationResult) # noqa: S101 + assert result[0].expectation_id == expectation.inject_expectation_id # noqa: S101 + assert result[0].is_valid # noqa: S101 + + +# -------- +# Helper Methods +# -------- + + +def _create_mock_signature(sig_type, value): + """Create a mock signature with proper attributes. + + Args: + sig_type: The signature type. + value: The signature value. + + Returns: + Mock signature object. + + """ + sig = Mock() + sig.type = sig_type + sig.value = value + return sig + + +def _create_mock_expectation(expectation_id=None, signatures=None): + """Create a mock expectation with proper attributes. + + Args: + expectation_id: The expectation ID. + signatures: List of signatures. + + Returns: + Mock expectation object. + + """ + if expectation_id is None: + expectation_id = str(uuid4()) + if signatures is None: + signatures = [] + + expectation = Mock() + expectation.inject_expectation_id = expectation_id + expectation.inject_expectation_signatures = signatures + expectation.id = expectation_id + return expectation diff --git a/template/tests/services/test_fetcher_deep_visibility.py b/template/tests/services/test_fetcher_deep_visibility.py new file mode 100644 index 00000000..0af9fd45 --- /dev/null +++ b/template/tests/services/test_fetcher_deep_visibility.py @@ -0,0 +1,431 @@ +"""Essential tests for SentinelOne Deep Visibility Fetcher service - Gherkin GWT Format.""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import Mock, patch + +import pytest +from src.services.exception import SentinelOneValidationError +from src.services.fetcher_deep_visibility import FetcherDeepVisibility +from tests.gwt_shared import given_initialized_client_api + +# -------- +# Scenarios +# -------- + + +# Scenario: Initialize deep visibility fetcher with valid client API +def test_initialize_deep_visibility_fetcher_with_valid_client_api(): + """Scenario: Initialize deep visibility fetcher with valid client API.""" + # Given: A valid client API is available + client_api = _given_valid_client_api() + + # When: I initialize the deep visibility fetcher + fetcher = _when_initialize_deep_visibility_fetcher(client_api) + + # Then: The deep visibility fetcher should be initialized successfully + _then_deep_visibility_fetcher_initialized_successfully(fetcher, client_api) + + +# Scenario: Fetch events for single SHA1 successfully +def test_fetch_events_for_single_sha1_successfully(): + """Scenario: Fetch events for single SHA1 successfully.""" + # Given: A valid deep visibility fetcher + fetcher = _given_valid_deep_visibility_fetcher() + # Given: A valid SHA1 hash + sha1 = _given_valid_sha1() + # Given: A valid time range + start_time, end_time = _given_valid_time_range() + + # When: I fetch events for the SHA1 (with mocked API) + with _mock_deep_visibility_success_response(fetcher, sha1): + events = _when_fetch_events_for_sha1(fetcher, sha1, start_time, end_time) + + # Then: Events should be returned successfully + _then_events_returned_successfully_for_single_sha1(events, sha1) + + +# Scenario: Fetch events for batch SHA1s successfully +def test_fetch_events_for_batch_sha1s_successfully(): + """Scenario: Fetch events for batch SHA1s successfully.""" + # Given: A valid deep visibility fetcher + fetcher = _given_valid_deep_visibility_fetcher() + # Given: A list of valid SHA1 hashes + sha1_list = _given_valid_sha1_list() + # Given: A valid time range + start_time, end_time = _given_valid_time_range() + + # When: I fetch events for the SHA1 batch (with mocked API) + with _mock_deep_visibility_batch_success_response(fetcher, sha1_list): + events_dict = _when_fetch_events_for_batch_sha1( + fetcher, sha1_list, start_time, end_time + ) + + # Then: Events should be returned successfully for all SHA1s + _then_events_returned_successfully_for_batch_sha1s(events_dict, sha1_list) + + +# Scenario: Handle invalid SHA1 input +def test_handle_invalid_sha1_input(): + """Scenario: Handle invalid SHA1 input.""" + # Given: A valid deep visibility fetcher + fetcher = _given_valid_deep_visibility_fetcher() + # Given: Invalid SHA1 inputs + invalid_sha1_cases = [None, "", 123, [], {}] + + for invalid_sha1 in invalid_sha1_cases: + # When: I attempt to fetch events with invalid SHA1 + # Then: A validation error should be raised + _when_fetch_events_for_sha1_then_validation_error_raised(fetcher, invalid_sha1) + + +# Scenario: Handle invalid SHA1 list input +def test_handle_invalid_sha1_list_input(): + """Scenario: Handle invalid SHA1 list input.""" + # Given: A valid deep visibility fetcher + fetcher = _given_valid_deep_visibility_fetcher() + + # When: I attempt to fetch events with None SHA1 list + # Then: A validation error should be raised + with pytest.raises(SentinelOneValidationError): + fetcher.fetch_events_for_batch_sha1(None) + + # When: I attempt to fetch events with empty SHA1 list + # Then: A validation error should be raised + with pytest.raises(SentinelOneValidationError): + fetcher.fetch_events_for_batch_sha1([]) + + +# Scenario: Handle API connection error during single SHA1 fetch +def test_handle_api_connection_error_single_sha1(): + """Scenario: Handle API connection error during single SHA1 fetch.""" + # Given: A valid deep visibility fetcher + fetcher = _given_valid_deep_visibility_fetcher() + # Given: A valid SHA1 hash + sha1 = _given_valid_sha1() + + # When: I attempt to fetch events with connection error + with _mock_connection_error(fetcher): + _when_fetch_events_for_sha1_then_api_error_raised(fetcher, sha1) + + +# Scenario: Filter events correctly by SHA1 in single fetch +def test_filter_events_correctly_by_sha1_single_fetch(): + """Scenario: Filter events correctly by SHA1 in single fetch.""" + # Given: A valid deep visibility fetcher + fetcher = _given_valid_deep_visibility_fetcher() + # Given: A target SHA1 hash + target_sha1 = _given_valid_sha1() + + # When: I fetch events and API returns mixed SHA1 events + with _mock_mixed_sha1_events_response(fetcher, target_sha1): + events = _when_fetch_events_for_sha1(fetcher, target_sha1) + + # Then: Only events matching the target SHA1 should be returned + _then_only_target_sha1_events_returned(events, target_sha1) + + +# -------- +# Given Methods +# -------- + + +# Given: A valid client API is available +def _given_valid_client_api(): + """Create a valid client API for testing. + + Returns: + Valid client API instance. + + """ + return given_initialized_client_api() + + +# Given: A valid deep visibility fetcher +def _given_valid_deep_visibility_fetcher(): + """Create a valid deep visibility fetcher for testing. + + Returns: + Initialized FetcherDeepVisibility instance. + + """ + client_api = given_initialized_client_api() + return FetcherDeepVisibility(client_api) + + +# Given: A valid SHA1 hash +def _given_valid_sha1(): + """Create a valid SHA1 hash for testing. + + Returns: + Valid SHA1 string. + + """ + return "a1b2c3d4e5f6789012345678901234567890abcd" + + +# Given: A list of valid SHA1 hashes +def _given_valid_sha1_list(): + """Create a list of valid SHA1 hashes for testing. + + Returns: + List of valid SHA1 strings. + + """ + return [ + "a1b2c3d4e5f6789012345678901234567890abcd", + "b2c3d4e5f6789012345678901234567890abcdef", + "c3d4e5f6789012345678901234567890abcdef01", + ] + + +# Given: A valid time range +def _given_valid_time_range(): + """Create a valid time range for testing. + + Returns: + Tuple of (start_time, end_time) datetime objects. + + """ + end_time = datetime.now(timezone.utc) + start_time = end_time - timedelta(hours=1) + return start_time, end_time + + +# -------- +# Mock Context Managers +# -------- + + +def _mock_deep_visibility_success_response(fetcher, sha1): + """Mock Deep Visibility API to return successful response for single SHA1.""" + mock_query_response = Mock() + mock_query_response.data = Mock() + mock_query_response.data.query_id = "test_query_id" + + mock_events = [ + { + "fileSha1": sha1, + "processName": "test_process.exe", + "timestamp": "2023-01-01T12:00:00Z", + "eventType": "Process Creation", + }, + { + "fileSha1": "different_sha1", + "processName": "other_process.exe", + "timestamp": "2023-01-01T12:01:00Z", + "eventType": "Process Creation", + }, + ] + + return patch.multiple( + fetcher, + _init_dv_query=Mock(return_value=mock_query_response), + _execute_query=Mock(return_value=mock_events), + ) + + +def _mock_deep_visibility_batch_success_response(fetcher, sha1_list): + """Mock Deep Visibility API to return successful response for batch SHA1s.""" + mock_query_response = Mock() + mock_query_response.data = Mock() + mock_query_response.data.query_id = "test_batch_query_id" + + mock_events = [] + for i, sha1 in enumerate(sha1_list): + mock_events.append( + { + "fileSha1": sha1, + "processName": f"test_process_{i}.exe", + "timestamp": f"2023-01-01T12:0{i}:00Z", + "eventType": "Process Creation", + } + ) + + return patch.multiple( + fetcher, + _init_dv_query=Mock(return_value=mock_query_response), + _execute_query=Mock(return_value=mock_events), + ) + + +def _mock_connection_error(fetcher): + """Mock connection error during API call.""" + from requests.exceptions import ConnectionError + + return patch.object( + fetcher, + "_init_dv_query", + side_effect=ConnectionError("Connection failed"), + ) + + +def _mock_mixed_sha1_events_response(fetcher, target_sha1): + """Mock API to return events with mixed SHA1s.""" + mock_query_response = Mock() + mock_query_response.data = Mock() + mock_query_response.data.query_id = "mixed_query_id" + + mock_events = [ + {"fileSha1": target_sha1, "processName": "target_process.exe"}, + {"fileSha1": "other_sha1_1", "processName": "other_process1.exe"}, + {"fileSha1": target_sha1, "processName": "target_process2.exe"}, + {"fileSha1": "other_sha1_2", "processName": "other_process2.exe"}, + ] + + return patch.multiple( + fetcher, + _init_dv_query=Mock(return_value=mock_query_response), + _execute_query=Mock(return_value=mock_events), + ) + + +# -------- +# When Methods +# -------- + + +# When: I initialize the deep visibility fetcher +def _when_initialize_deep_visibility_fetcher(client_api): + """Initialize deep visibility fetcher with given client API. + + Args: + client_api: Client API instance to use. + + Returns: + Initialized FetcherDeepVisibility instance. + + """ + return FetcherDeepVisibility(client_api) + + +# When: I fetch events for SHA1 +def _when_fetch_events_for_sha1(fetcher, sha1, start_time=None, end_time=None): + """Fetch events for given SHA1. + + Args: + fetcher: The deep visibility fetcher instance. + sha1: SHA1 hash to fetch events for. + start_time: Start time for search (optional). + end_time: End time for search (optional). + + Returns: + List of fetched events. + + """ + return fetcher.fetch_events_for_sha1(sha1, start_time, end_time) + + +# When: I fetch events for batch SHA1s +def _when_fetch_events_for_batch_sha1( + fetcher, sha1_list, start_time=None, end_time=None +): + """Fetch events for given SHA1 list. + + Args: + fetcher: The deep visibility fetcher instance. + sha1_list: List of SHA1 hashes to fetch events for. + start_time: Start time for search (optional). + end_time: End time for search (optional). + + Returns: + Dictionary mapping SHA1 to events. + + """ + return fetcher.fetch_events_for_batch_sha1(sha1_list, start_time, end_time) + + +# When: I attempt to fetch events for SHA1 and expect validation error +def _when_fetch_events_for_sha1_then_validation_error_raised(fetcher, invalid_sha1): + """Attempt to fetch events for invalid SHA1 and expect validation error. + + Args: + fetcher: The deep visibility fetcher instance. + invalid_sha1: Invalid SHA1 to test. + + """ + with pytest.raises(SentinelOneValidationError): + fetcher.fetch_events_for_sha1(invalid_sha1) + + +# When: I attempt to fetch events for SHA1 and expect API error +def _when_fetch_events_for_sha1_then_api_error_raised(fetcher, sha1): + """Attempt to fetch events and expect API error. + + Args: + fetcher: The deep visibility fetcher instance. + sha1: SHA1 hash to fetch events for. + + """ + from src.services.exception import SentinelOneAPIError + + with pytest.raises(SentinelOneAPIError): + fetcher.fetch_events_for_sha1(sha1) + + +# -------- +# Then Methods +# -------- + + +# Then: The deep visibility fetcher should be initialized successfully +def _then_deep_visibility_fetcher_initialized_successfully(fetcher, client_api): + """Verify deep visibility fetcher was initialized successfully. + + Args: + fetcher: The fetcher instance to verify. + client_api: The client API used for initialization. + + """ + assert fetcher is not None # noqa: S101 + assert fetcher.client_api == client_api # noqa: S101 + assert fetcher.logger is not None # noqa: S101 + + +# Then: Events should be returned successfully for single SHA1 +def _then_events_returned_successfully_for_single_sha1(events, sha1): + """Verify events were returned successfully for single SHA1. + + Args: + events: The fetched events to verify. + sha1: The SHA1 that was searched for. + + """ + assert isinstance(events, list) # noqa: S101 + assert len(events) > 0 # noqa: S101 + + for event in events: + assert event.get("fileSha1") == sha1 # noqa: S101 + assert isinstance(event, dict) # noqa: S101 + + +# Then: Events should be returned successfully for batch SHA1s +def _then_events_returned_successfully_for_batch_sha1s(events_dict, sha1_list): + """Verify events were returned successfully for batch SHA1s. + + Args: + events_dict: Dictionary of SHA1 to events. + sha1_list: The SHA1 list that was searched for. + + """ + assert isinstance(events_dict, dict) # noqa: S101 + assert len(events_dict) == len(sha1_list) # noqa: S101 + + for sha1 in sha1_list: + assert sha1 in events_dict # noqa: S101 + assert isinstance(events_dict[sha1], list) # noqa: S101 + + +# Then: Only events matching the target SHA1 should be returned +def _then_only_target_sha1_events_returned(events, target_sha1): + """Verify only events matching target SHA1 are returned. + + Args: + events: The fetched events to verify. + target_sha1: The target SHA1 that should match. + + """ + assert isinstance(events, list) # noqa: S101 + assert len(events) > 0 # noqa: S101 + + for event in events: + assert event.get("fileSha1") == target_sha1 # noqa: S101 diff --git a/template/tests/services/test_fetcher_threat.py b/template/tests/services/test_fetcher_threat.py new file mode 100644 index 00000000..63512f4f --- /dev/null +++ b/template/tests/services/test_fetcher_threat.py @@ -0,0 +1,336 @@ +"""Essential tests for SentinelOne Threat Fetcher service - Gherkin GWT Format.""" + +import pytest +from src.services.exception import SentinelOneNetworkError, SentinelOneValidationError +from src.services.fetcher_threat import FetcherThreat +from tests.gwt_shared import given_initialized_client_api + +# -------- +# Scenarios +# -------- + + +# Scenario: Initialize threat fetcher with valid client API +def test_initialize_threat_fetcher_with_valid_client_api(): + """Scenario: Initialize threat fetcher with valid client API.""" + # Given: A valid client API is available + client_api = _given_valid_client_api() + + # When: I initialize the threat fetcher + fetcher = _when_initialize_threat_fetcher(client_api) + + # Then: The threat fetcher should be initialized successfully + _then_threat_fetcher_initialized_successfully(fetcher, client_api) + + +# Scenario: Initialize with invalid client API raises error +def test_initialize_with_invalid_client_api(): + """Scenario: Initialize with invalid client API raises error.""" + # Given: An invalid client API (None) + invalid_client_api = _given_invalid_client_api() + + # When: I attempt to initialize the threat fetcher + # Then: A validation error should be raised + _when_initialize_threat_fetcher_then_validation_error_raised(invalid_client_api) + + +# Scenario: Fetch threats for time window successfully +def test_fetch_threats_for_time_window_successfully(): + """Scenario: Fetch threats for time window successfully.""" + # Given: A valid threat fetcher + fetcher = _given_valid_threat_fetcher() + # Given: A valid time window + time_window = _given_valid_time_window() + # When: I fetch threats for the time window (with mocked API) + from unittest.mock import Mock, patch + + mock_response = Mock() + mock_response.json.return_value = { + "data": [ + { + "threatInfo": { + "threatId": "test_threat_1", + "computerName": "test-host.example.com", + "mitigationStatus": "not_mitigated", + } + } + ] + } + mock_response.raise_for_status.return_value = None + + with patch.object(fetcher.client_api.session, "get", return_value=mock_response): + threats = _when_fetch_threats_for_time_window(fetcher, time_window) + + # Then: Threats should be returned successfully + _then_threats_returned_successfully(threats) + + +# Scenario: Handle API connection error +def test_handle_api_connection_error(): + """Scenario: Handle API connection error.""" + # Given: A valid threat fetcher + fetcher = _given_valid_threat_fetcher() + # Given: A valid time window + time_window = _given_valid_time_window() + + # When: I attempt to fetch threats with connection error + from unittest.mock import patch + + from requests.exceptions import ConnectionError + + with patch.object( + fetcher.client_api.session, + "get", + side_effect=ConnectionError("Connection failed"), + ): + _when_fetch_threats_then_network_error_raised(fetcher, time_window) + + +# Scenario: Handle invalid time window +def test_handle_invalid_time_window(): + """Scenario: Handle invalid time window.""" + # Given: A valid threat fetcher + fetcher = _given_valid_threat_fetcher() + # Given: An invalid time window + invalid_time_window = _given_invalid_time_window() + + # When: I attempt to fetch threats with invalid time window + # Then: A validation error should be raised + _when_fetch_threats_then_validation_error_raised(fetcher, invalid_time_window) + + +# -------- +# Given Methods +# -------- + + +# Given: A valid client API is available +def _given_valid_client_api(): + """Create a valid client API for testing. + + Returns: + Valid client API instance. + + """ + return given_initialized_client_api() + + +# Given: An invalid client API (None) +def _given_invalid_client_api(): + """Create an invalid client API. + + Returns: + None (invalid client API). + + """ + return None + + +# Given: A valid threat fetcher +def _given_valid_threat_fetcher(): + """Create a valid threat fetcher for testing. + + Returns: + Initialized SentinelOneThreatFetcher instance. + + """ + client_api = given_initialized_client_api() + return FetcherThreat(client_api) + + +# Given: A valid time window +def _given_valid_time_window(): + """Create a valid time window for testing. + + Returns: + Valid timedelta object. + + """ + from datetime import timedelta + + return timedelta(hours=24) + + +# Given: An invalid time window +def _given_invalid_time_window(): + """Create an invalid time window. + + Returns: + Invalid time window (None). + + """ + return None + + +# Given: Mock API returns threat data +def _given_mock_api_returns_threat_data(fetcher): + """Set up mock API to return threat data. + + Args: + fetcher: The threat fetcher instance to mock. + + """ + from unittest.mock import Mock, patch + + mock_response = Mock() + mock_response.json.return_value = { + "data": [ + { + "threatInfo": { + "threatId": "test_threat_1", + "computerName": "test-host.example.com", + "mitigationStatus": "not_mitigated", + } + } + ] + } + mock_response.raise_for_status.return_value = None + + with patch.object(fetcher.client_api.session, "get", return_value=mock_response): + pass + + +# Given: API connection will fail +def _given_api_connection_will_fail(fetcher): + """Set up API connection to fail. + + Args: + fetcher: The threat fetcher instance to mock. + + """ + from unittest.mock import patch + + from requests.exceptions import ConnectionError + + with patch.object( + fetcher.client_api.session, + "get", + side_effect=ConnectionError("Connection failed"), + ): + pass + + +# -------- +# When Methods +# -------- + + +# When: I initialize the threat fetcher +def _when_initialize_threat_fetcher(client_api): + """Initialize threat fetcher with given client API. + + Args: + client_api: Client API instance to use. + + Returns: + Initialized SentinelOneThreatFetcher instance. + + """ + return FetcherThreat(client_api) + + +# When: I attempt to initialize with invalid client API and expect validation error +def _when_initialize_threat_fetcher_then_validation_error_raised(invalid_client_api): + """Attempt to initialize with invalid client API and expect validation error. + + Args: + invalid_client_api: Invalid client API to test. + + """ + with pytest.raises(SentinelOneValidationError): + FetcherThreat(invalid_client_api) + + +# When: I fetch threats for the time window +def _when_fetch_threats_for_time_window(fetcher, time_window): + """Fetch threats for given time window. + + Args: + fetcher: The threat fetcher instance. + time_window: Time window to fetch threats for. + + Returns: + List of fetched threats. + + """ + from datetime import datetime, timezone + + end_time = datetime.now(timezone.utc) + start_time = end_time - time_window + return fetcher.fetch_threats_for_time_window(start_time, end_time) + + +# When: I attempt to fetch threats and expect network error +def _when_fetch_threats_then_network_error_raised(fetcher, time_window): + """Attempt to fetch threats and expect network error. + + Args: + fetcher: The threat fetcher instance. + time_window: Time window to fetch threats for. + + """ + from datetime import datetime, timezone + + end_time = datetime.now(timezone.utc) + start_time = end_time - time_window + with pytest.raises(SentinelOneNetworkError): + fetcher.fetch_threats_for_time_window(start_time, end_time) + + +# When: I attempt to fetch threats and expect validation error +def _when_fetch_threats_then_validation_error_raised(fetcher, invalid_time_window): + """Attempt to fetch threats and expect validation error. + + Args: + fetcher: The threat fetcher instance. + invalid_time_window: Invalid time window to test. + + """ + from datetime import datetime, timezone + + if invalid_time_window is not None: + end_time = datetime.now(timezone.utc) + start_time = end_time - invalid_time_window + with pytest.raises(SentinelOneValidationError): + fetcher.fetch_threats_for_time_window(start_time, end_time) + else: + with pytest.raises(SentinelOneValidationError): + fetcher.fetch_threats_for_time_window(None, None) + + +# -------- +# Then Methods +# -------- + + +# Then: The threat fetcher should be initialized successfully +def _then_threat_fetcher_initialized_successfully(fetcher, client_api): + """Verify threat fetcher was initialized successfully. + + Args: + fetcher: The threat fetcher instance to verify. + client_api: The client API used for initialization. + + """ + assert fetcher is not None # noqa: S101 + assert fetcher.client_api == client_api # noqa: S101 + assert fetcher.logger is not None # noqa: S101 + + +# Then: Threats should be returned successfully +def _then_threats_returned_successfully(threats): + """Verify threats were returned successfully. + + Args: + threats: The fetched threats to verify. + + """ + assert isinstance(threats, list) # noqa: S101 + assert len(threats) > 0 # noqa: S101 + + # Basic verification that we got threats back + from src.services.model_threat import SentinelOneThreat + + assert all( # noqa: S101 + isinstance(threat, SentinelOneThreat) for threat in threats + ) diff --git a/template/tests/services/test_fetcher_threat_events.py b/template/tests/services/test_fetcher_threat_events.py new file mode 100644 index 00000000..37f8ea4a --- /dev/null +++ b/template/tests/services/test_fetcher_threat_events.py @@ -0,0 +1,212 @@ +"""Essential tests for SentinelOne Threat Events Fetcher service - Gherkin GWT Format.""" + +import pytest +from src.services.exception import SentinelOneValidationError +from src.services.fetcher_threat_events import FetcherThreatEvents +from tests.gwt_shared import ( + given_initialized_client_api, + given_threat_with_complete_data, +) + +# -------- +# Scenarios +# -------- + + +# Scenario: Initialize threat events fetcher with valid client API +def test_initialize_threat_events_fetcher_with_valid_client_api(): + """Scenario: Initialize threat events fetcher with valid client API.""" + # Given: A valid client API is available + client_api = _given_valid_client_api() + + # When: I initialize the threat events fetcher + fetcher = _when_initialize_threat_events_fetcher(client_api) + + # Then: The threat events fetcher should be initialized successfully + _then_threat_events_fetcher_initialized_successfully(fetcher, client_api) + + +# Scenario: Initialize with invalid client API raises error +def test_initialize_with_invalid_client_api(): + """Scenario: Initialize with invalid client API raises error.""" + # Given: An invalid client API (None) + invalid_client_api = _given_invalid_client_api() + + # When: I attempt to initialize the threat events fetcher + # Then: A validation error should be raised + _when_initialize_threat_events_fetcher_then_validation_error_raised( + invalid_client_api + ) + + +# Scenario: Fetch events for threat successfully +def test_fetch_events_for_threat_successfully(): + """Scenario: Fetch events for threat successfully.""" + # Given: A valid threat events fetcher + fetcher = _given_valid_threat_events_fetcher() + # Given: A threat to fetch events for + threat = _given_threat_for_event_fetching() + + # When: I fetch events for the threat (with mocked API) + from unittest.mock import Mock, patch + + mock_response = Mock() + mock_response.json.return_value = { + "data": [ + { + "id": "event_1", + "processName": "test.exe", + "createdAt": "2024-01-01T12:00:00Z", + } + ] + } + mock_response.raise_for_status.return_value = None + + with patch.object(fetcher.client_api.session, "get", return_value=mock_response): + events = _when_fetch_events_for_threat(fetcher, threat) + + # Then: Events should be returned successfully + _then_events_returned_successfully(events) + + +# -------- +# Given Methods +# -------- + + +# Given: A valid client API is available +def _given_valid_client_api(): + """Create a valid client API for testing. + + Returns: + Valid client API instance. + + """ + return given_initialized_client_api() + + +# Given: An invalid client API (None) +def _given_invalid_client_api(): + """Create an invalid client API. + + Returns: + None (invalid client API). + + """ + return None + + +# Given: A valid threat events fetcher +def _given_valid_threat_events_fetcher(): + """Create a valid threat events fetcher for testing. + + Returns: + Initialized FetcherThreatEvents instance. + + """ + client_api = given_initialized_client_api() + return FetcherThreatEvents(client_api) + + +# Given: A threat to fetch events for +def _given_threat_for_event_fetching(): + """Create a threat for event fetching testing. + + Returns: + Threat object for testing. + + """ + return given_threat_with_complete_data() + + +# Given: Mock API returns event data +def _given_mock_api_returns_event_data(fetcher): + """Set up mock API to return event data. + + Args: + fetcher: The threat events fetcher instance to mock. + + """ + pass + + +# -------- +# When Methods +# -------- + + +# When: I initialize the threat events fetcher +def _when_initialize_threat_events_fetcher(client_api): + """Initialize threat events fetcher with given client API. + + Args: + client_api: Client API instance to use. + + Returns: + Initialized FetcherThreatEvents instance. + + """ + return FetcherThreatEvents(client_api) + + +# When: I attempt to initialize with invalid client API and expect validation error +def _when_initialize_threat_events_fetcher_then_validation_error_raised( + invalid_client_api, +): + """Attempt to initialize with invalid client API and expect validation error. + + Args: + invalid_client_api: Invalid client API to test. + + """ + with pytest.raises(SentinelOneValidationError): + FetcherThreatEvents(invalid_client_api) + + +# When: I fetch events for the threat +def _when_fetch_events_for_threat(fetcher, threat): + """Fetch events for given threat. + + Args: + fetcher: The threat events fetcher instance. + threat: Threat to fetch events for. + + Returns: + List of fetched events. + + """ + return fetcher.fetch_events_for_threat(threat) + + +# -------- +# Then Methods +# -------- + + +# Then: The threat events fetcher should be initialized successfully +def _then_threat_events_fetcher_initialized_successfully(fetcher, client_api): + """Verify threat events fetcher was initialized successfully. + + Args: + fetcher: The fetcher instance to verify. + client_api: The client API used for initialization. + + """ + assert fetcher is not None # noqa: S101 + assert fetcher.client_api == client_api # noqa: S101 + assert fetcher.logger is not None # noqa: S101 + + +# Then: Events should be returned successfully +def _then_events_returned_successfully(events): + """Verify events were returned successfully. + + Args: + events: The fetched events to verify. + + """ + assert isinstance(events, list) # noqa: S101 + assert len(events) > 0 # noqa: S101 + + # Basic verification that we got events back + assert all(isinstance(event, dict) for event in events) # noqa: S101 diff --git a/template/tests/services/test_trace_service.py b/template/tests/services/test_trace_service.py new file mode 100644 index 00000000..c5296918 --- /dev/null +++ b/template/tests/services/test_trace_service.py @@ -0,0 +1,235 @@ +"""Essential tests for SentinelOne Trace Service - Gherkin GWT Format.""" + +import pytest +from src.services.exception import SentinelOneValidationError +from src.services.trace_service import SentinelOneTraceService +from tests.gwt_shared import given_test_config + +# -------- +# Scenarios +# -------- + + +# Scenario: Initialize trace service with valid configuration +def test_initialize_trace_service_with_valid_config(): + """Scenario: Initialize trace service with valid configuration.""" + # Given: A valid configuration is available + config = _given_valid_config_for_trace_service() + + # When: I initialize the trace service + service = _when_initialize_trace_service(config) + + # Then: The trace service should be initialized successfully + _then_trace_service_initialized_successfully(service, config) + + +# Scenario: Initialize with invalid configuration raises error +def test_initialize_with_invalid_config(): + """Scenario: Initialize with invalid configuration raises error.""" + # Given: An invalid configuration (None) + invalid_config = _given_invalid_config() + + # When: I attempt to initialize the trace service + # Then: A validation error should be raised + _when_initialize_trace_service_then_validation_error_raised(invalid_config) + + +# Scenario: Create traces from valid expectation results +def test_create_traces_from_valid_expectation_results(): + """Scenario: Create traces from valid expectation results.""" + # Given: A valid trace service + service = _given_valid_trace_service() + # Given: Valid expectation results with matching alerts + results = _given_valid_expectation_results_with_alerts() + + # When: I create traces from the results + traces = _when_create_traces_from_results(service, results) + + # Then: Traces should be created successfully + _then_traces_created_successfully(traces, results) + + +# Scenario: Trace timestamp format is correct for Java backend +def test_trace_timestamp_format_is_correct(): + """Scenario: Trace timestamp format is correct for Java backend.""" + # Given: A valid trace service + service = _given_valid_trace_service() + # Given: Valid expectation results with matching alerts + results = _given_valid_expectation_results_with_alerts() + + # When: I create traces from the results + traces = _when_create_traces_from_results(service, results) + + # Then: Timestamp format should be valid for Java backend + _then_timestamp_format_is_valid(traces) + + +# -------- +# Given Methods +# -------- + + +# Given: A valid configuration is available +def _given_valid_config_for_trace_service(): + """Create a valid configuration for trace service testing. + + Returns: + Test configuration object. + + """ + return given_test_config() + + +# Given: An invalid configuration (None) +def _given_invalid_config(): + """Create an invalid configuration. + + Returns: + None (invalid configuration). + + """ + return None + + +# Given: A valid trace service +def _given_valid_trace_service(): + """Create a valid trace service for testing. + + Returns: + Initialized SentinelOneTraceService instance. + + """ + config = given_test_config() + return SentinelOneTraceService(config=config) + + +# Given: Valid expectation results with matching alerts +def _given_valid_expectation_results_with_alerts(): + """Create valid expectation results with alerts for testing. + + Returns: + List of expectation results with alerts. + + """ + from unittest.mock import Mock + + result1 = Mock() + result1.expectation_id = "test_expectation_1" + result1.is_valid = True + result1.matched_alerts = [ + { + "alert_id": "alert_1", + "severity": "high", + "message": "Test alert 1", + "alert_name": "SentinelOne Test Alert 1", + "alert_link": "https://console.sentinelone.com/alerts/alert_1", + } + ] + result1.expectation = Mock() + result1.expectation.inject_expectation_id = "test_expectation_1" + + return [result1] + + +# -------- +# When Methods +# -------- + + +# When: I initialize the trace service +def _when_initialize_trace_service(config): + """Initialize trace service with given configuration. + + Args: + config: Configuration object to use. + + Returns: + Initialized SentinelOneTraceService instance. + + """ + return SentinelOneTraceService(config=config) + + +# When: I attempt to initialize with invalid config and expect validation error +def _when_initialize_trace_service_then_validation_error_raised(invalid_config): + """Attempt to initialize with invalid config and expect validation error. + + Args: + invalid_config: Invalid configuration to test. + + """ + with pytest.raises(SentinelOneValidationError): + SentinelOneTraceService(config=invalid_config) + + +# When: I create traces from the results +def _when_create_traces_from_results(service, results): + """Create traces from expectation results. + + Args: + service: The trace service instance. + results: List of expectation results. + + Returns: + List of created traces. + + """ + return service.create_traces_from_results(results, "test_collector_id") + + +# -------- +# Then Methods +# -------- + + +# Then: The trace service should be initialized successfully +def _then_trace_service_initialized_successfully(service, config): + """Verify trace service was initialized successfully. + + Args: + service: The service instance to verify. + config: The configuration used for initialization. + + """ + assert service is not None # noqa: S101 + assert service.config == config # noqa: S101 + assert service.logger is not None # noqa: S101 + + +# Then: Traces should be created successfully +def _then_traces_created_successfully(traces, results): + """Verify traces were created successfully from results. + + Args: + traces: The created traces to verify. + results: The original expectation results. + + """ + assert isinstance(traces, list) # noqa: S101 + assert len(traces) > 0 # noqa: S101 + + from src.collector.models import ExpectationTrace + + for trace in traces: + assert isinstance(trace, ExpectationTrace) # noqa: S101 + assert hasattr(trace, "inject_expectation_trace_expectation") # noqa: S101 + assert hasattr(trace, "inject_expectation_trace_alert_name") # noqa: S101 + assert hasattr(trace, "inject_expectation_trace_alert_link") # noqa: S101 + + +# Then: Timestamp format should be valid for Java backend +def _then_timestamp_format_is_valid(traces): + """Verify timestamp format is valid for Java backend. + + Args: + traces: The created traces to verify. + + """ + import re + + valid_timestamp_pattern = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$" + + for trace in traces: + timestamp = trace.inject_expectation_trace_date + assert re.match(valid_timestamp_pattern, timestamp) # noqa: S101 + assert "+00:00Z" not in timestamp # noqa: S101 diff --git a/template/tests/test_create_collector.py b/template/tests/test_create_collector.py new file mode 100644 index 00000000..e40b9bfa --- /dev/null +++ b/template/tests/test_create_collector.py @@ -0,0 +1,233 @@ +"""Test module for the SentinelOne Collector initialization - Gherkin GWT Format.""" + +from os import environ as os_environ +from typing import Any + +import pytest +from src.collector import Collector +from src.collector.exception import CollectorConfigError +from tests.conftest import mock_env_vars + +# -------- +# Fixtures +# -------- + + +@pytest.fixture() +def collector_config() -> dict[str, str]: # type: ignore + """Fixture for minimum required configuration. + + Returns: + Dictionary containing all required environment variables + for collector initialization with test values. + + """ + return { + "OPENAEV_URL": "http://fake-url/", + "OPENAEV_TOKEN": "fake-oaev-token", + "COLLECTOR_ID": "fake-collector-id", + "COLLECTOR_NAME": "SentinelOne", + "SENTINELONE_BASE_URL": "https://fake-sentinelone.net/", + "SENTINELONE_API_KEY": "fake-api-key", + "COLLECTOR_ICON_FILEPATH": "src/img/sentinelone-logo.png", + "COLLECTOR_LOG_LEVEL": "debug", + } + + +# -------- +# Scenarios +# -------- + + +# Scenario: Create a collector with success +def test_create_collector_with_valid_config(capfd, collector_config): # type: ignore + """Scenario: Create a collector with success. + + Args: + capfd: Pytest fixture for capturing stdout and stderr output. + collector_config: Fixture providing valid collector configuration. + + """ + # Given: A valid configuration is available + mock_env = _given_valid_collector_config(collector_config) + + # When: I create the collector + collector = _when_create_collector() + + # Then: The collector should be created successfully + _then_collector_created_successfully(capfd, mock_env, collector, collector_config) + + +# Scenario: Create a collector with missing required config +def test_create_collector_with_missing_api_key(collector_config) -> None: + """Scenario: Create a collector with missing required config. + + Args: + collector_config: Fixture providing base collector configuration. + + """ + # Given: Configuration with missing required SentinelOne API key + incomplete_config = _given_config_missing_api_key(collector_config) + mock_env = _given_valid_collector_config(incomplete_config) + + # When: I attempt to create the collector + # Then: The collector creation should fail with configuration error + _when_create_collector_then_raises_config_error(mock_env) + + +# -------- +# Given Methods +# -------- + + +# Given: A valid configuration is available +def _given_valid_collector_config(config_data: dict[str, str]) -> Any: # type: ignore + """Set up valid collector configuration environment. + + Args: + config_data: Dictionary of environment variables to mock. + + Returns: + Mock environment variable patcher object. + + """ + mock_env = mock_env_vars(os_environ, config_data) + return mock_env + + +# Given: Configuration with missing required SentinelOne API key +def _given_config_missing_api_key(base_config: dict[str, str]) -> dict[str, str]: + """Create configuration with missing SentinelOne API key. + + Args: + base_config: Base configuration dictionary. + + Returns: + Configuration dictionary without SentinelOne API key. + + """ + config = base_config.copy() + config.pop("SENTINELONE_API_KEY", None) + + if "SENTINELONE_API_KEY" in os_environ: + del os_environ["SENTINELONE_API_KEY"] + + return config + + +# -------- +# When Methods +# -------- + + +# When: I create the collector +def _when_create_collector() -> Collector: # type: ignore + """Create the collector instance. + + Returns: + Collector instance for testing. + + """ + collector = Collector() + return collector + + +# When: I attempt to create the collector and expect configuration error +def _when_create_collector_then_config_error_raised(mock_env: Any) -> None: # type: ignore + """Attempt to create collector and expect configuration error. + + Args: + mock_env: Mock environment variable patcher to clean up. + + """ + try: + with pytest.raises((CollectorConfigError, ValueError)): + _when_create_collector() + finally: + mock_env.stop() + + +# When: I attempt to create the collector and expect configuration error (alias) +def _when_create_collector_then_raises_config_error(mock_env: Any) -> None: # type: ignore + """Attempt to create collector and expect configuration error. + + Args: + mock_env: Mock environment variable patcher to clean up. + + """ + _when_create_collector_then_config_error_raised(mock_env) + + +# -------- +# Then Methods +# -------- + + +# Then: The collector should be created successfully +def _then_collector_created_successfully( + capfd: Any, # type: ignore + mock_env: Any, # type: ignore + collector: Collector, # type: ignore + expected_config: dict[str, str], +) -> None: + """Verify the collector was created successfully with correct configuration. + + Args: + capfd: Pytest fixture for capturing stdout and stderr output. + mock_env: Mock environment variable patcher to clean up. + collector: The created collector instance to verify. + expected_config: Expected configuration data to validate against. + + """ + assert collector is not None # noqa: S101 + + daemon_config = collector.config_instance.to_daemon_config() + + assert daemon_config.get("openaev_url") == expected_config.get( + "OPENAEV_URL" + ) # noqa: S101 + assert daemon_config.get("openaev_token") == expected_config.get( + "OPENAEV_TOKEN" + ) # noqa: S101 + assert daemon_config.get("collector_id") == expected_config.get( + "COLLECTOR_ID" + ) # noqa: S101 + assert daemon_config.get("collector_name") == expected_config.get( + "COLLECTOR_NAME" + ) # noqa: S101 + assert daemon_config.get( + "sentinelone_base_url" + ) == expected_config.get( # noqa: S101 + "SENTINELONE_BASE_URL" + ) + assert daemon_config.get( + "sentinelone_api_key" + ) == expected_config.get( # noqa: S101 + "SENTINELONE_API_KEY" + ) + assert daemon_config.get( + "collector_log_level" + ) == expected_config.get( # noqa: S101 + "COLLECTOR_LOG_LEVEL" + ) + + _then_collector_logged_initialization_success(capfd, daemon_config) + mock_env.stop() + + +# Then: The collector initialization should be logged +def _then_collector_logged_initialization_success( + capfd: Any, # type: ignore + daemon_config: dict[str, str], +) -> None: + """Verify that collector initialization was logged appropriately. + + Args: + capfd: Pytest fixture for capturing stdout and stderr output. + daemon_config: Daemon configuration to check log level. + + """ + log_records = capfd.readouterr() + if daemon_config.get("collector_log_level") in ["info", "debug"]: + registered_message = "SentinelOne Collector initialized successfully" + assert registered_message in log_records.err # noqa: S101 From 2460166a9e5f0d580f5dfbdddbf5174c6708df06 Mon Sep 17 00:00:00 2001 From: guzmud Date: Wed, 8 Apr 2026 16:15:41 +0200 Subject: [PATCH 02/25] [template] feat(collector): withdrawing SentinelOne specific code and features --- template/CONTRIBUTING.md | 53 +- template/README.md | 193 ++----- template/docker-compose.yml | 7 +- template/manifest-metadata.json | 14 +- template/pyproject.toml | 8 +- template/src/__main__.py | 2 +- template/src/collector/collector.py | 30 +- template/src/config.yml.sample | 5 +- template/src/img/sentinelone-logo.png | Bin 63856 -> 0 bytes template/src/models/configs/__init__.py | 4 +- .../src/models/configs/collector_configs.py | 2 +- template/src/models/configs/config_loader.py | 46 +- .../src/models/configs/sentinelone_configs.py | 39 -- .../src/models/configs/template_configs.py | 30 + template/src/services/client_api.py | 71 --- template/src/services/converter.py | 88 ++- template/src/services/exception.py | 28 +- template/src/services/expectation_service.py | 289 ++-------- template/src/services/fetcher_data.py | 69 +++ .../src/services/fetcher_deep_visibility.py | 533 ------------------ template/src/services/fetcher_threat.py | 141 ----- .../src/services/fetcher_threat_events.py | 139 ----- template/src/services/model_data.py | 15 + template/src/services/model_threat.py | 132 ----- template/src/services/trace_service.py | 54 +- template/src/services/utils/__init__.py | 4 +- template/src/services/utils/config_loader.py | 12 +- .../src/services/utils/signature_extractor.py | 2 +- template/src/services/utils/trace_builder.py | 50 +- template/tests/gwt_shared.py | 385 ++++--------- template/tests/services/conftest.py | 45 +- template/tests/services/fixtures/factories.py | 111 +--- template/tests/services/test_client_api.py | 117 ---- template/tests/services/test_converter.py | 106 ++-- .../services/test_expectation_service.py | 406 ++----------- template/tests/services/test_fetcher_data.py | 168 ++++++ .../services/test_fetcher_deep_visibility.py | 431 -------------- .../tests/services/test_fetcher_threat.py | 336 ----------- .../services/test_fetcher_threat_events.py | 212 ------- template/tests/services/test_trace_service.py | 22 +- template/tests/test_create_collector.py | 59 +- 41 files changed, 793 insertions(+), 3665 deletions(-) delete mode 100644 template/src/img/sentinelone-logo.png delete mode 100644 template/src/models/configs/sentinelone_configs.py create mode 100644 template/src/models/configs/template_configs.py delete mode 100644 template/src/services/client_api.py create mode 100644 template/src/services/fetcher_data.py delete mode 100644 template/src/services/fetcher_deep_visibility.py delete mode 100644 template/src/services/fetcher_threat.py delete mode 100644 template/src/services/fetcher_threat_events.py create mode 100644 template/src/services/model_data.py delete mode 100644 template/src/services/model_threat.py delete mode 100644 template/tests/services/test_client_api.py create mode 100644 template/tests/services/test_fetcher_data.py delete mode 100644 template/tests/services/test_fetcher_deep_visibility.py delete mode 100644 template/tests/services/test_fetcher_threat.py delete mode 100644 template/tests/services/test_fetcher_threat_events.py diff --git a/template/CONTRIBUTING.md b/template/CONTRIBUTING.md index 7900626b..1c924a3f 100644 --- a/template/CONTRIBUTING.md +++ b/template/CONTRIBUTING.md @@ -1,32 +1,28 @@ -# Contributing to SentinelOne Collector +# Contributing to Template Collector -This document provides guidance for contributing to the SentinelOne collector for OpenAEV. This collector is now feature-complete with SentinelOne-specific implementation. +This document provides guidance for contributing to the Template collector for OpenAEV. This collector is now feature-complete with Template-specific implementation. ## Current Implementation Status -**COMPLETED**: The SentinelOne collector is fully implemented with the following components: +**COMPLETED**: The Template collector is fully implemented with the following components: ### Core Components -- **Collector Core** ([`src/collector/collector.py`](src/collector/collector.py)) - Main daemon with SentinelOne service integration +- **Collector Core** ([`src/collector/collector.py`](src/collector/collector.py)) - Main daemon with Template service integration - **Expectation Handler** ([`src/collector/expectation_handler.py`](src/collector/expectation_handler.py)) - Generic handler using service provider pattern - **Expectation Manager** ([`src/collector/expectation_manager.py`](src/collector/expectation_manager.py)) - Batch processing and API interactions -- **Configuration System** ([`src/models/configs/`](src/models/configs/)) - Hierarchical configuration with SentinelOne settings -- **Service Providers** - Complete SentinelOne-specific implementation +- **Configuration System** ([`src/models/configs/`](src/models/configs/)) - Hierarchical configuration with Template settings +- **Service Providers** - Complete Template-specific implementation -### SentinelOne Implementation -- **SentinelOne API Client** ([`src/services/client_api.py`](src/services/client_api.py)) - Full API integration -- **Deep Visibility Fetcher** ([`src/services/fetcher_deep_visibility.py`](src/services/fetcher_deep_visibility.py)) - Process event queries -- **Threat Fetcher** ([`src/services/fetcher_threat.py`](src/services/fetcher_threat.py)) - Prevention data correlation +### Template Implementation +- **Data Fetcher** ([`src/services/fetcher_data.py`](src/services/fetcher_data.py)) - Prevention data correlation - **Expectation Service** ([`src/services/expectation_service.py`](src/services/expectation_service.py)) - Business logic implementation -- **Trace Service** ([`src/services/trace_service.py`](src/services/trace_service.py)) - Trace creation with SentinelOne links -- **Data Converter** ([`src/services/converter.py`](src/services/converter.py)) - SentinelOne to OAEV format conversion +- **Trace Service** ([`src/services/trace_service.py`](src/services/trace_service.py)) - Trace creation +- **Data Converter** ([`src/services/converter.py`](src/services/converter.py)) - Template to OAEV format conversion ### Supported Features -- **Signature Support**: `parent_process_name`, `start_date`, `end_date` -- **Detection Expectations**: Deep Visibility event validation -- **Prevention Expectations**: Combined event + threat validation +- **Signature Support**: `start_date`, `end_date` - **Retry Mechanism**: Configurable retries with ingestion delay handling -- **Trace Generation**: Links back to SentinelOne console +- **Trace Generation**: Links back to external tool available - **Error Handling**: Comprehensive exception handling and logging - **Configuration Management**: YAML, environment variables, defaults @@ -60,7 +56,7 @@ poetry install -E local --with dev,test ```bash # Direct execution -SentinelOneCollector +TemplateCollector # Using Python module execution python -m src @@ -76,7 +72,7 @@ poetry run python -m src 1. **Clone and Install**: ```bash git clone - cd sentinelone + cd template poetry install -E current --with dev,test ``` @@ -85,7 +81,7 @@ poetry run python -m src # Copy sample config cp src/config.yml.sample src/config.yml - # Edit with your SentinelOne details + # Edit with your Template details vim src/config.yml ``` @@ -116,12 +112,11 @@ src/ │ ├── expectation_manager.py │ ├── trace_manager.py │ └── models.py # Pydantic data models -├── services/ # SentinelOne-specific implementation -│ ├── client_api.py # API client +├── services/ # Template-specific implementation │ ├── expectation_service.py # Business logic │ ├── trace_service.py # Trace creation │ ├── converter.py # Data conversion -│ ├── fetcher_*.py # API-specific fetchers +│ ├── fetcher_*.py # Data fetchers │ └── model_*.py # Data models └── models/ # Configuration management └── configs/ # Hierarchical config system @@ -145,7 +140,7 @@ poetry run pytest -v ### Test Categories - **Unit Tests**: Test individual components in isolation -- **Integration Tests**: Test SentinelOne API interactions +- **Integration Tests**: Test external tool interactions - **Configuration Tests**: Validate config loading and validation - **Service Provider Tests**: Test expectation handling logic @@ -175,8 +170,8 @@ from src.collector.exception import CollectorProcessingError try: result = process_expectation(expectation) -except SentinelOneServiceError as e: - logger.error(f"SentinelOne API error: {e}") +except TemplateServiceError as e: + logger.error(f"Template error: {e}") raise CollectorProcessingError(f"Processing failed: {e}") from e ``` @@ -234,7 +229,7 @@ logger.error( #### Adding New Signature Types 1. Update `SUPPORTED_SIGNATURES` in [`src/services/expectation_service.py`](src/services/expectation_service.py) -2. Modify query building in [`src/services/client_api.py`](src/services/client_api.py) +2. Update fetching processes in [`src/services/fetcher_data.py`](src/services/fetcher_data.py) 3. Update data conversion logic in [`src/services/converter.py`](src/services/converter.py) 4. Add corresponding tests @@ -256,7 +251,7 @@ logger.error( This collector is built on a reusable foundation that can be adapted for other security platforms. If you want to create a similar collector for another platform (e.g., CrowdStrike, Microsoft Defender): -### SentinelOne-Specific References to Change +### Template-Specific References to Change #### Configuration Files - [ ] `pyproject.toml` - Update project name and script names @@ -294,7 +289,7 @@ The following components are platform-agnostic and can be reused: #### API Integration Testing - Use mock objects for unit tests -- Set up test SentinelOne environment for integration tests +- Set up test Template environment for integration tests - Handle rate limits in test environments ### Production Issues @@ -321,4 +316,4 @@ The following components are platform-agnostic and can be reused: - Provide example configurations for different scenarios - Include troubleshooting guides for common issues -This collector provides a production-ready SentinelOne integration for OpenAEV with comprehensive error handling, configurable retry logic, and detailed trace generation. +This collector provides a production-ready Template integration for OpenAEV with comprehensive error handling, configurable retry logic, and detailed trace generation. diff --git a/template/README.md b/template/README.md index d1742ebc..6ce84d06 100644 --- a/template/README.md +++ b/template/README.md @@ -1,31 +1,27 @@ -# OpenAEV SentinelOne Collector +# OpenAEV Template Collector -A SentinelOne EDR integration for OpenAEV that validates security expectations by querying SentinelOne's Deep Visibility and Threats APIs. +A template for OpenAEV collector built from late-2025/early-2026 collectors (e.g. SentinelOne). Provides a modular approach to collector development based on a service-provider architecture with `Protocol` based interfaces for advanced customisation. -**Note**: Requires access to a SentinelOne Management Console with appropriate API permissions. +## Overview -**⚠️ Deep Visibility License Warning**: All static engine alerts rely on Deep Visibility, which is only compatible with Complete licenses. However, behavioral detection will be properly handled even with Core or Control licenses. The `enable_deep_visibility_search` configuration option (defaulted to False) allows you to enable this feature when you have the appropriate license. +This collector is not meant to be used directly in OpenAEV but as a first support for collector development. Please update this README.md with the relevant elements to describe your collector. -## Overview +The codebase to adapt to your specific needs can be found under `src/services/`, by replacing reference to the abstract TemplateData with your specific objects, updating the DataFetcher and the various services according to this custom object and your specific needs (keywords such as data and template can be used to help parsing the generic code that should be customized). -This collector validates OpenAEV expectations by querying your SentinelOne environment for threat data via the SentinelOne API. When OpenAEV runs security exercises, this collector automatically checks if the expected security threats were detected in your EDR by matching threat information and associated events, providing visibility into your detection capabilities. +Once `src/services/` updated, the imports must be updated in `src/collector/collector.py`. Finally, new configuration parameters for your collector should be integrated under the `src/models/configs/` folder, replacing the `template_configs.py` file with yours. -The collector uses SentinelOne's Threats API to fetch threat data and correlates it with threat events to validate expectations. +Do not hesitate to check the `CONTRIBUTING.md` for more details regarding the collector design and help regarding development setup. ## Features -- **Threat-Based Validation**: Queries SentinelOne Threats API to validate security expectations against detected threats - **Batch Processing**: Processes expectations in configurable batches for improved performance -- **Event Correlation**: Correlates threat data with threat events to extract process execution details -- **Trace Generation**: Creates detailed traces with links back to SentinelOne console +- **Trace Generation**: Creates detailed traces with links back if available - **Flexible Configuration**: Support for YAML, environment variables, and multiple deployment scenarios ## Requirements - OpenAEV Platform -- SentinelOne Management Console with API access - Python 3.12+ (for manual deployment) -- SentinelOne API token with Threats and Threat Events permissions ## Configuration @@ -54,12 +50,12 @@ Below are the parameters you'll need to set for running the collector properly: | Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | |------------------|---------------------|-----------------------------|-------------------------|-----------|-----------------------------------------------------------------------------------------------| -| Collector ID | collector.id | `COLLECTOR_ID` | sentinelone--0b13e3f7-5c9e-46f5-acc4-33032e9b4921 | Yes | A unique `UUIDv4` identifier for this collector instance. | -| Collector Name | collector.name | `COLLECTOR_NAME` | SentinelOne | No | Name of the collector. | +| Collector ID | collector.id | `COLLECTOR_ID` | template--0b13e3f7-5c9e-46f5-acc4-33032e9b4921 | Yes | A unique `UUIDv4` identifier for this collector instance. | +| Collector Name | collector.name | `COLLECTOR_NAME` | Template | No | Name of the collector. | | Collector Period | collector.period | `COLLECTOR_PERIOD` | PT2M | No | Collection interval (ISO 8601 format). | | Log Level | collector.log_level | `COLLECTOR_LOG_LEVEL` | error | No | Determines the verbosity of the logs. Options are `debug`, `info`, `warn`, or `error`. | | Platform | collector.platform | `COLLECTOR_PLATFORM` | EDR | No | Type of security platform this collector works for. One of: `EDR, XDR, SIEM, SOAR, NDR, ISPM` | -| Icon Filepath | collector.icon_filepath | `COLLECTOR_ICON_FILEPATH` | src/img/sentinelone-logo.png | No | Path to the icon file of the collector. | +| Icon Filepath | collector.icon_filepath | `COLLECTOR_ICON_FILEPATH` | src/img/template-logo.png | No | Path to the icon file of the collector. | ### Collector extra parameters environment variables @@ -67,11 +63,9 @@ Below are the parameters you'll need to set for the collector: | Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | |--------------------------|--------------------------------------|----------------------------------------|-----------------------------|-----------|----------------------------------------------------------------------------------------------------| -| Base URL | sentinelone.base_url | `SENTINELONE_BASE_URL` | https://api.sentinelone.com | No | SentinelOne Management Console URL | -| API Key | sentinelone.api_key | `SENTINELONE_API_KEY` | | Yes | SentinelOne API token with Threats and Threat Events permissions | -| Time Window | sentinelone.time_window | `SENTINELONE_TIME_WINDOW` | PT1H | No | Default search time window when no date signatures are provided (ISO 8601 format) | -| Expectation Batch Size | sentinelone.expectation_batch_size | `SENTINELONE_EXPECTATION_BATCH_SIZE` | 50 | No | Number of expectations to process in each batch for batch-based processing | -| Enable Deep Visibility | sentinelone.enable_deep_visibility_search | `SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH` | false | No | Enable Deep Visibility search for advanced threat detection (requires Complete license) | +| Base URL | template.key | `TEMPLATE_KEY` |value| No | Template example key value | +| Time Window | template.time_window | `TEMPLATE_TIME_WINDOW` | PT1H | No | Default search time window when no date signatures are provided (ISO 8601 format) | +| Expectation Batch Size | template.expectation_batch_size | `TEMPLATE_EXPECTATION_BATCH_SIZE` | 50 | No | Number of expectations to process in each batch for batch-based processing | ### Example Configuration Files @@ -82,27 +76,23 @@ openaev: token: "your-openaev-token" collector: - id: "sentinelone--your-unique-uuid" - name: "SentinelOne Production" + id: "template--your-unique-uuid" + name: "Template Production" period: "PT10M" log_level: "info" -sentinelone: - base_url: "https://your-sentinelone-console.sentinelone.net" - api_key: "your-sentinelone-api-token" +template: + key: "your-value" time_window: "PT1H" expectation_batch_size: 50 - enable_deep_visibility_search: false ``` #### Environment Variables ```bash export OPENAEV_URL="https://your-openaev-instance.com" export OPENAEV_TOKEN="your-openaev-token" -export COLLECTOR_ID="sentinelone--your-unique-uuid" -export SENTINELONE_BASE_URL="https://your-sentinelone-console.sentinelone.net" -export SENTINELONE_API_KEY="your-sentinelone-api-token" -export SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH="false" +export COLLECTOR_ID="template--your-unique-uuid" +export TEMPLATE_KEY="value" ``` ## Deployment @@ -112,7 +102,7 @@ export SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH="false" 1. **Clone and Install Dependencies**: ```bash git clone - cd sentinelone + cd template poetry install --extras local ``` @@ -126,28 +116,27 @@ export SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH="false" poetry run python -m src # Or direct execution after installation - SentinelOneCollector + TemplateCollector ``` ### Docker Deployment ```bash # Build the container -docker build -t openaev-sentinelone-collector . +docker build -t openaev-template-collector . # Run with environment variables docker run -d \ -e OPENAEV_URL="https://your-openaev-instance.com" \ -e OPENAEV_TOKEN="your-token" \ - -e COLLECTOR_ID="sentinelone--your-uuid" \ - -e SENTINELONE_BASE_URL="https://your-console.sentinelone.net" \ - -e SENTINELONE_API_KEY="your-api-key" \ - openaev-sentinelone-collector + -e COLLECTOR_ID="template--your-uuid" \ + -e TEMPLATE_KEY="your-value" \ + openaev-template-collector # Or run with configuration file mounted docker run -d \ -v /path/to/config.yml:/app/src/config.yml:ro \ - openaev-sentinelone-collector + openaev-template-collector ``` ## Behavior @@ -155,46 +144,17 @@ docker run -d \ ### Supported Signature Types The collector supports the following OpenAEV signature types: - -- **`parent_process_name`**: Process names to match against threat event data -- **`target_hostname_address`**: Target hostnames to filter threat queries -- **`end_date`**: End time for the threat search query (ISO 8601 format) +- **change_me**: detail of the supported signature ### Processing Flow 1. **Expectation Retrieval**: Fetches pending expectations from OpenAEV 2. **Batch Creation**: Groups expectations into configurable batches for processing 3. **Time Window Determination**: Extracts time windows from expectations or uses default configuration -4. **Threat Fetching**: Queries SentinelOne Threats API for the determined time window -5. **Event Correlation**: Fetches threat events for each identified threat -6. **Expectation Matching**: Matches threat data and events against expectation criteria using detection helper +4. **Data Fetching**: Fetch data for the determined time window +6. **Expectation Matching**: Matches data against expectation criteria using detection helper 7. **Result Reporting**: Updates expectation status in OpenAEV -8. **Trace Creation**: Creates detailed traces linking back to SentinelOne console - -### Threat Matching Logic - -The collector validates expectations by: - -1. **Threat Data Conversion**: Converts SentinelOne threat objects to OpenAEV-compatible format -2. **Process Name Extraction**: Extracts parent process names from threat events, focusing on `oaev-implant-*` prefixed processes -3. **Signature Matching**: Uses OpenAEV detection helper to match extracted data against expectation signatures -4. **Static vs Dynamic Threats**: Handles both static threat indicators and dynamic threats with associated events - -### Deep Visibility Integration - -The collector's behavior varies based on the Deep Visibility feature availability: - -- **Deep Visibility Enabled** (`enable_deep_visibility_search: true`): - - Requires SentinelOne Complete license - - Provides comprehensive threat detection including static engine alerts - - Supports full range of static threat indicators - -- **Deep Visibility Disabled** (`enable_deep_visibility_search: false`, default): - - Compatible with Core and Control licenses - - Focuses on behavioral threat detection - - Maintains full functionality for dynamic threats and behavioral analysis - -**Important**: When Deep Visibility is disabled, static engine-based expectations may not be fully validated, but all behavioral-based detections will continue to work normally. +8. **Trace Creation**: Creates detailed traces ### Batch Processing @@ -202,80 +162,19 @@ The collector implements efficient batch processing to handle large volumes of e 1. **Configurable Batch Size**: Processes expectations in batches based on `expectation_batch_size` configuration 2. **Time Window Optimization**: Extracts and consolidates time windows across batch expectations -3. **Bulk Threat Fetching**: Fetches threats for the entire time window rather than individual queries -4. **Parallel Event Processing**: Efficiently correlates threat events across the batch - -## API Requirements - -### SentinelOne API Permissions - -Your SentinelOne API token requires the following permissions: - -- **Threats**: Read access to query threat information -- **Threat Events**: Read access to retrieve threat event details -- **Console Access**: General API access to the Management Console -- **Deep Visibility** (Optional): Required when `enable_deep_visibility_search` is enabled, needs Complete license - -### API Endpoints Used - -- `GET /web/api/v2.1/threats`: Query threat information using time-based filters -- `GET /web/api/v2.1/threat-events`: Retrieve detailed threat event information -- **Deep Visibility endpoints** (when enabled): - - `POST /web/api/v2.1/dv/init-query`: Initialize Deep Visibility query with SHA1 hashes and time range - - `GET /web/api/v2.1/dv/query-status`: Poll query status and progress until completion - - `GET /web/api/v2.1/dv/events`: Retrieve Deep Visibility events for completed query - -### Rate Limiting - -The collector respects SentinelOne's API rate limits by: -- Processing expectations in configurable batches -- Consolidating time windows to minimize API calls -- Adjusting query complexity based on Deep Visibility availability +3. **Bulk Data Fetching**: Fetches data for the entire time window rather than individual queries ## Troubleshooting ### Common Issues -#### No Threats Found -- **Symptom**: Collector reports no matching threats despite expecting them -- **Causes**: - - Threat ingestion delay in SentinelOne - - Incorrect process names or hostnames in expectations - - Time window too narrow for threat detection -- **Solutions**: - - Verify process names match threat event data - - Extend `sentinelone.time_window` for broader searches - -#### API Authentication Errors -- **Symptom**: HTTP 401/403 errors in logs -- **Causes**: - - Invalid or expired API token - - Insufficient API permissions -- **Solutions**: - - Verify API token in SentinelOne console - -#### Connection Timeouts -- **Symptom**: HTTP timeout errors or connection failures -- **Causes**: - - Network connectivity issues - - SentinelOne console unavailability - - Incorrect base URL -- **Solutions**: - - Verify network connectivity to SentinelOne - - Check `sentinelone.base_url` configuration - - Review firewall and proxy settings - -#### Deep Visibility Issues -- **Symptom**: Static threats not being detected or matched +#### Type of common issue +- **Symptom**: main symptom for this common issue - **Causes**: - - Deep Visibility disabled in configuration - - Insufficient SentinelOne license (Core/Control instead of Complete) - - Deep Visibility feature not properly configured in SentinelOne + - common cause for this issue (1) + - common cause for this issue (2) - **Solutions**: - - Set `sentinelone.enable_deep_visibility_search: true` if you have Complete license - - Verify your SentinelOne license includes Deep Visibility features - - Focus on behavioral expectations when using Core/Control licenses - - Check SentinelOne console for Deep Visibility configuration status + - solution(s) ### Logging @@ -293,34 +192,28 @@ collector: ``` #### Key Log Patterns -- `[SentinelOneClientAPI]`: API communication and responses -- `[SentinelOneExpectationService]`: Batch expectation processing logic -- `[SentinelOneThreatFetcher]`: Threat data fetching operations -- `[SentinelOneThreatEventsFetcher]`: Threat events fetching operations +- `[TemplateExpectationService]`: Batch expectation processing logic +- `[TemplateDataFetcher]`: Data fetching operations - `[CollectorExpectationManager]`: High-level processing flow -- `[SentinelOneTraceService]`: Trace creation and submission +- `[TemplateTraceService]`: Trace creation and submission ### Performance Tuning #### For High-Volume Environments - Reduce `collector.period` for more frequent processing -- Increase `sentinelone.expectation_batch_size` for better throughput -- Monitor API rate limits and ingestion patterns in your environment +- Increase `template.expectation_batch_size` for better throughput #### For Low-Latency Requirements - Use shorter time windows in expectations for faster queries - Reduce `collector.period` for more frequent collection cycles -- Monitor API rate limits and ingestion patterns accordingly ## Architecture The collector uses a modular, service-provider architecture: - **Collector Core**: Main daemon handling scheduling and coordination -- **Expectation Service**: Batch processing and threat correlation logic -- **Threat Fetcher**: Dedicated service for fetching threat data -- **Threat Events Fetcher**: Service for retrieving threat event details -- **Client API**: SentinelOne API communication layer +- **Expectation Service**: Batch processing and data correlation logic +- **Data Fetcher**: Dedicated service for fetching data - **Trace Service**: Trace creation and submission - **Configuration System**: Hierarchical configuration management diff --git a/template/docker-compose.yml b/template/docker-compose.yml index 57477ea3..1d99b138 100644 --- a/template/docker-compose.yml +++ b/template/docker-compose.yml @@ -1,11 +1,10 @@ version: "3" services: - collector-sentinelone: - image: openaev/collector-sentinelone:rolling + collector-template: + image: openaev/collector-template:rolling environment: - OPENAEV_URL=http://localhost - OPENAEV_TOKEN=ChangeMe - COLLECTOR_ID=ChangeMe - - SENTINELONE_BASE_URL=https://change.me - - SENTINELONE_API_KEY=ChangeMe + - TEMPLATE_KEY=ChangeMe restart: always diff --git a/template/manifest-metadata.json b/template/manifest-metadata.json index 1cdfd0d6..2dd84de6 100644 --- a/template/manifest-metadata.json +++ b/template/manifest-metadata.json @@ -1,18 +1,18 @@ { - "title": "SentinelOne", - "slug": "openaev_sentinelone", - "description": "Collect responses from SentinelOne", - "short_description": "Collect responses from SentinelOne", + "title": "Template", + "slug": "openaev_template", + "description": "Template project for OAEV collectors", + "short_description": "Collector template", "use_cases": ["Security response"], "verified": true, "last_verified_date": "", "playbook_supported": false, "max_confidence_level": 80, "support_version": "", - "subscription_link": "https://www.sentinelone.com/", + "subscription_link": "", "source_code": "", "manager_supported": true, "container_version": "rolling", - "container_image": "openaev/collector-sentinelone", + "container_image": "openaev/collector-template", "container_type": "COLLECTOR" -} \ No newline at end of file +} diff --git a/template/pyproject.toml b/template/pyproject.toml index b42e197b..432c36d9 100644 --- a/template/pyproject.toml +++ b/template/pyproject.toml @@ -6,9 +6,9 @@ build-backend = "poetry.core.masonry.api" packages = [{ include = "src" }, { include = "tests" }] [project] -name = "SentinelOneCollector" +name = "TemplateCollector" version = "2.3.2" -description = "Collector for SentinelOne EDR." +description = "Collector template for OAEV." readme = "README.md" requires-python = ">=3.11,<3.14" @@ -41,7 +41,7 @@ pytest = "^8.4.1" polyfactory = "^2.22.2" [project.scripts] -SentinelOneCollector = "src.__main__:main" +TemplateCollector = "src.__main__:main" [tool.pytest.ini_options] testpaths = ["./tests"] @@ -124,4 +124,4 @@ plugins = ["pydantic.mypy"] [tool.cmw] install-command = "poetry install --extras local" config-dump-command = "poetry run python -m src --dump-config-schema" -icon-path = "src/img/sentinelone-logo.png" +icon-path = "src/img/change-me-logo.png" diff --git a/template/src/__main__.py b/template/src/__main__.py index f025e84b..29e5b88d 100644 --- a/template/src/__main__.py +++ b/template/src/__main__.py @@ -15,7 +15,7 @@ def main() -> None: logger = logging.getLogger(__name__) try: - logger.info(f"{LOG_PREFIX} Starting SentinelOne collector...") + logger.info(f"{LOG_PREFIX} Starting Template collector...") collector = Collector() collector.start() except KeyboardInterrupt: diff --git a/template/src/collector/collector.py b/template/src/collector/collector.py index 442b3683..b1a99403 100644 --- a/template/src/collector/collector.py +++ b/template/src/collector/collector.py @@ -4,9 +4,9 @@ from pyoaev.daemons import CollectorDaemon # type: ignore[import-untyped] from pyoaev.helpers import OpenAEVDetectionHelper # type: ignore[import-untyped] -from src.services.expectation_service import SentinelOneExpectationService -from src.services.trace_service import SentinelOneTraceService -from src.services.utils import SentinelOneConfig +from src.services.expectation_service import TemplateExpectationService +from src.services.trace_service import TemplateTraceService +from src.services.utils import TemplateConfig from .exception import ( CollectorConfigError, @@ -33,17 +33,17 @@ def __init__(self) -> None: """ try: - self.config = SentinelOneConfig() + self.config = TemplateConfig() self.config_instance = self.config.load super().__init__( configuration=self.config_instance.to_daemon_config(), callback=self._process_callback, - collector_type="openaev_sentinelone", + collector_type="openaev_template", ) self.logger.info( # type: ignore[has-type] - f"{LOG_PREFIX} SentinelOne Collector initialized successfully" + f"{LOG_PREFIX} Template Collector initialized successfully" ) except Exception as err: @@ -58,7 +58,7 @@ def __init__(self) -> None: def _setup(self) -> None: """Set up the collector. - Initializes SentinelOne services, expectation handler, expectation manager, + Initializes Template services, expectation handler, expectation manager, and OpenAEV detection helper. Sets up the collector for processing expectations. Raises: @@ -70,17 +70,15 @@ def _setup(self) -> None: super()._setup() - self.logger.debug(f"{LOG_PREFIX} Initializing SentinelOne services...") + self.logger.debug(f"{LOG_PREFIX} Initializing Template services...") - self.sentinelone_service = SentinelOneExpectationService( + self.template_service = TemplateExpectationService( config=self.config_instance ) - self.trace_service = SentinelOneTraceService(self.config_instance) + self.trace_service = TemplateTraceService(self.config_instance) - self.expectation_handler = GenericExpectationHandler( - self.sentinelone_service - ) + self.expectation_handler = GenericExpectationHandler(self.template_service) self.expectation_manager = GenericExpectationManager( oaev_api=self.api, @@ -89,7 +87,7 @@ def _setup(self) -> None: trace_service=self.trace_service, ) - supported_signatures = self.sentinelone_service.get_supported_signatures() + supported_signatures = self.template_service.get_supported_signatures() self.oaev_detection_helper = OpenAEVDetectionHelper( logger=self.logger, relevant_signatures_types=supported_signatures, @@ -100,7 +98,7 @@ def _setup(self) -> None: f"{LOG_PREFIX} Supported signatures: {[sig.value for sig in supported_signatures]}" ) - service_info = self.sentinelone_service.get_service_info() + service_info = self.template_service.get_service_info() self.logger.debug(f"{LOG_PREFIX} Service info: {service_info}") except Exception as err: @@ -121,7 +119,7 @@ def _process_callback(self) -> None: try: self.logger.info(f"{LOG_PREFIX} Starting processing cycle...") self.logger.debug( - f"{LOG_PREFIX} Processing expectations using SentinelOne services" + f"{LOG_PREFIX} Processing expectations using Template services" ) results = self.expectation_manager.process_expectations( diff --git a/template/src/config.yml.sample b/template/src/config.yml.sample index 2a162707..c9a0b9bd 100644 --- a/template/src/config.yml.sample +++ b/template/src/config.yml.sample @@ -5,6 +5,5 @@ openaev: collector: id: "ChangeMe" -sentinelone: - base_url: "http://change.me" - api_key: "ChangeMe" +template: + key: "value" diff --git a/template/src/img/sentinelone-logo.png b/template/src/img/sentinelone-logo.png deleted file mode 100644 index ddd6d518dc23eb95c33d0c5c842671b1cd4669df..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 63856 zcmZsE2_Tef+y8W0Po-T`Y45ZgLMxp_TC_=&WfEgbN9rI&s99=Gowkv7X|tqcHzb6f zk&2Qe`#u~IF-*ja<*5I4-_J8c@Ap5?_xjR2b1&ERyMEVlFI_a-u~}v0w2?F#O=atr zjk{>HAzd`uu$3c*!QZ6C>@9=;H^_b0=JkWhvKF6$Uw%EZ#oC=l8#NLAZxHS7{V6nB zx9Zl7>nx7nmsDk6kF-8<{)CF^4E^>;uEE236JsKnoRT{a2MxY4(**#t zjgG2+@Z2{hlkajVHaj!3l&fR#r*M0S=eBUu{DmO~(O%J=oVmqo<|WSUxYEG${QT!H zG#bCD>s40Z_JVu*0o=|Xya)*YN9j9{KBwkg4jj%j38kXOX`z z(Rm{vCJvdUXz%_#^9=Fe{@>T!JTD-Y_cyodmit9w`AB8+t6nE(h)0~h(m0rGuXXvY zQnj7^-Idx)e^GvF$g$lfVVWsF4GT2(CPw#v%&VKgnL1Q47Jhq{rtFvguItAeNrp~P zj=XlFiDc-c{$IyiN`_8W{@Ol~DN&u>b-{ofmTkbU*LLfaJ-z6GHxtoiq*mQ1~0NS8w=$1$$e%}Hl<3#A&D8{rg}f8@iWtt=66{sRPr|KaNOqh@!a9{Gbtrs^>$nk7+|ox zyHIk6u)k6>uWQ&SQE0Gq|BWC1r!G;NoN_*-w6|k1k+J-SDO2Cwto0gqNcG-iuWts4 z+^RPVek?atbg*jpJ$q>Z!^<_SE?c{7glL58)yh=f5BEMw`26#+Hj)_{UFEl+g?-8L zHkah2?~2d%uUag#fWL3-RE+M(;@s^9%S~K^O3mBGbLru7*#*&$1(fJxk|%Nktsi+M za5oGM6kMrE-9mUt`{bR?`IK@DQ#-tgr9)Zwt_xk#y<}`xrUGq2CNgg#Za(Yezni{h zjOUIGllLgPZ2EjB%W=3sUv*YGV>>~ifZt;+AaXzC+dR@Wl!SMU5Y;vj0df|T{jGx< zyxD@>9R_+Pt}=;vToOBZD9h%$&?Mb2T5Ln-MtaTbG1x}piN9+%o%4mMpE^e#yhZe1 ztxrv4in!wVET*XSA*H+v@8fUex?ChW>Be6Z=5?1(0vxe@3inOTcO!c-z0V43Oa^Cx2&g|zpCz(wv6Zf*hoF6CpQWPRua{pwU?&BKo+1>5lt4^B|NgEXE*gK^+LtJ`YJE3()UHp+`Dnt5PX0R`%M25qt3K0Dbr~1n#9S^MyBeXt21viI;Dvk z?rBz-;Oy0@8RFZgO*80yMGN7}n+rM33Y(v;em(=HQZEp$Ob-=TXvnEy&TXr#>pa9B zj6N(92({ot&R!yEb4TB0(m-o=RbC`Y=wFzc)n@6TwNBBsPFuf*%P-&JBuTwWgvOK( z_imrrDIgZ1&O3|dLFe-ZbiQ|SOokmhzGL#0#MgGeWrfyyckLO^Z6e~|5-NG}Ygy9I zv*wV7x5|I>4ecET9CewMc1#V-k?VovdbWx1kzcE)q_FCiUYtC`h|Oqr(-t8?a5T|N z2i;s-`Z#{#HTtAa;xnDSiB5(9p~8ae@vA|nT_&>AXv!R4@W4<{rqgZO>nC&7MYo0o zvZcXpiuu4mbP+HD1wsc9$|x%-WdXMFlQ^#L^VN_B)eQh^^j)o06L~*$6hu{+{kW9# zsj1aptGjljXvB5X3`Sy&xm49m8cm$Cp$2C*3v~5rD9bs^&jvv9*JKdK7B^P~Z24@= zSGawKr2>^rHFq+(yr#lY02_a+26w}^{m>%;YFCg@EsBmlXtv3W$Z-k$@!HXz!spf-@41A zyWUNJVZLkbXN>l?|LTkjIy{l(ZcHyZoWZyOgEG3#84Juo$$3g3s4K*Xtp^y>j;M2r zuhrNL2$~ipXN%K3zbL`ou-^vE=>b5pugonRWNS$HuPTA7rJC!-_DQ2*jJs$d(sdjzJdmTUU8l*EUrWHc@Qpb2q8g*ATh3 z(j@u$U+A1>(+nV!v|4XOCQZc7ezG^rnm(DME?R||F|Af(CrD#%7BRP(5RqJM`0JVg zuESk14d}>wUKh}LiiYSoo@nZB%n-{-KYtDQnPvS**9^Q@?G{nxX<@5CxB_P4{hK@@ z9F)A)i}D^<@43V!eZPDI>_(Lpt;$-Ar|S?cw{mmsBhBmTnLc|de7QDTv)gTyh^-;I z4!G35UJ0(OnM2Cf_Vk+@!ZBFMfP=>%*l_vFBE^RSh$osBQ{0WFuVF$??Q;_`BUoCZ zaI42hm?pj!H>s$i(Lg)ra65u9d9hf?AX$KK{%kVmOHRJcnzC`6&@JdSVT(XGgq&L; zM?R!Fr}tr&Fty|Jsg2mGUYu<8CW<(>fx5!G%453P0XqR~Vd+5L;jOvDb-62EOsK?p z8GtIy+Y)TrjObXC;13?cJd=W@fh0k@#Q!c6c_wi8kHEcG3ABZHoDmW7R^n93NhWC* zd|4x<>}<`hrtq%0BxV7V!AeZ>6G=`HZMiPIEYJ=l+Yu`4G-yY_c9DUJD<9$^Xy8}T zYIRW9qC|Dk1x1US!Ukyl-QKiP@41Mq&^y^!Xr@HTJSwh^JF)BphhGXDgx&yq zJE&+yOJUx%?Jw(E5H(IVUEn%)9A_s`glR^}jtnnr46< z)D*uB_6Tz3Z5h#B!QhmDHwO5Q%ui>tbRPqtDm1`(L|bs*0H#T~Qkix53rIR7njk)DII-R}_bF>G8JD#)$>`)*GIzolNuM_|(HB(Te~hD0B?8~fH1dZ_!(ov)$g>c@3gB<9nYIJ8@V}t`-!9N(V-BZ zMbhQ(kcWo1_h;x{#(*y53~Gj8;qUrp14E2fhbNOb)ptXZwn+b)FaMx{AOu4HRh2bH zVa6)O5GT2sboBj2$$g_Ecp~u>@I+|bzGiKR>cP!45$l_X1bJi;4n+82lF6L?#%TT! z^$@kGPbmq2*l&?Z9RJ|xiOAgmZ`2E<&nYdd+LAuW@He~rTn=J_K@@5dY}=T-s7VK~ zU9Z;IOyqe}u4#!5IDiU9HjWpNe%`89cLNrXej=m9sq)Dsq%<>wQCM0DaR(Z`QFNEq zd`i`(gjZCJP30*}qeryHc;kSLdg}v<+5j?kgQ*nh{tf!V>N^MzM%O8hw9oMRRUx$| zVt5IMv$0zHBFcSh#%Y^~mh--BlJFd%g3#`Ztr-OUS|I(Gn)k{*($42BPtq9}&eDkU z6W4}>B6v{f2E$gZM!pntP9f)EAx9%w-a(vd&(_8 zc`b<1OE^5NHp65XF?P1dQY^Ti7>M&p0x_4KjU)s1n>mmpxZe8ty*CtJ0^1uy=F`z< zPvV$=+!v5f`ZE+>!@KM)xDAsi*A*((NEBn!xa#M(@RSZ@tnGCiZ@;k+9U`HEQ)jiQfTKga z30TZy3mF}RYrA)M?}Agt$XILsgJff^jUv}05nLbH^`RfZ5Dh1_iM_skPb5>17bk%D zyeLzJc#Oe?=nxx)xzs_}?_*Z^_u0FeV-iL7Unh`GSE#IzQ>4(siz9Xc|8*Ne$U}KG=#o7P-|eZlrgpx`AUH!^r1rPB@Zo8u0*IT~q~AArU2r!DHkX z1E{?q+NGi-K(uLwSA>$FfL!>MsSA6)vq`9!4F+=Yhr|*{6vGdhA}dX40ZmN=M=BIz zPiVGGxLpjqz;Q4}wtGl%DpTA3uCBMBL+7=)2Pn*!l$nam85g~qfUIc24-<` z2LjK=njI3}DMUUfrW5@~VZw*m<^p*#LR_fWDG`YB@LKRllJRwcB2bo1p}AxpDR|?ycXisRk^v#*b!>F)3{*ry zF)V5FbP`7VIH>!i-3G_hA9BU1i}wEp&Z1MG&9f8l!VCfJi}xlc$xarB=#nYssFzAAElNlODaVq=J*KUxZa8Ev|MR-*8c+>x8mY5}?Rvhk_Y0YeKh4@EK@Pad zH`H}Bj(Cl(VlNyZa7AG#3L6k72(PlG;@&SsoCRxJ^6#acM->RN$kK+xIr=)txGcCQ z)J6LM@Q@{vPWfdM0Wq9{`aR-JC7^JMam?HN&D&ch46q{bzT|`pGa_%y+EIXGG3=uu z@id1f$(Y@gV64Mgjw%Xz-qr1p^c_)POVPNVUtKf!9;Ja=S^=G&@}SUVHU=2ynZlk7 zRIBGB6ew)U7X=-d`5j_ek{L)p)F2=!@Iq6Yn?>?ui6K#fu{F+{v2g8OWq^i8p}jdH z3E7Zh%q3>HR=__FJUUA=Uu`xnQeav5J*=I;mEUoEO!U1>s=X-dauA?m5d;+DPiO?<~;$kY5yvF@>q zq!UzGJ@21~D477l@Yu>!Rc>Snq0^mB224{jAD8yF6;-KiP>?=2HHpA=po&AqY8FtH zNwdWkAaP78RfCTe27vfR;i-YN0H9QTCMfWgJ(@Qp;aF?cUk1Pt%r%RQpc1!k*n3w@ zIuk_*C7qJxXRv%TMM$~qL<+jvCz$BlPat)V{glINp_-6+g$Tc~p8Z+mUQ02ohEtv*7b0@#lji7OiXY2a`IaanBcR)u|l z@uk*gYjn9~6C<#Pg88^*13R3*S4w*IuS6Yxfzcd_95DRt0rsGPXcKeIhvmskYfnNVK#2hp)A;J5%-ls5N)ktf%P! z8bfY=EGWzv!T^m<_HB-Bm{R--_CeZRCiiOV6wl0)^Pq=u=S|`oPVfFWbSH(JyJ0A` z{$J$e$_!3$aP^J<973JRF&C0IpHoUzk&7ep^^Q-CjFR_w>H_+ot%2bi+yCpg5hm(7 z1Uu~d8{AKb8h)k;NB2MvcO;m$t^4<*QenxIR3T0UAnRDi%> zWqAF6Xvn$gR z_c9o{VTz%Z$KQ%!th+8f_P_L8_94+dp`zQW1Etyg`?lT$s=142**|)=K4raP+Rw?j zKp~^UiT9)}pS4_SyNw|=w!PJBpF2Mvo?Vo+c^ZofkQ*MzrjE-i-kZ1_0BmceYsDq~ zvD@v6YRQ(8w`L_0J5T+m-!vLpo)O^v2tAZ>b{dO1{#dmO zG12}3FmiUVDYG)L*8u$;83VDA6sB@nO)Gh`cjTkUKf59y3A7(!Po0T!^8ZK=i1^;~ zi9fe?M{EADfKXuX7fYSJ_5HS@hti9lW&4bU*f%3Cqi<*}d#m%rr!xK;@Y9r+hvU57 z@slf$AN4=xAJw=zyf#}4S%j!fnHrb9k;U=;3*sT|%zE`YU{nZtsLA-x?LLP>4AM?B zu5uFNT%6U$R%RmE-Wc|N1!0XBT62B#eH9X(sZLd+B0B?^&h(0md9AftF4ZWs<0qC* z;%NDz=PXm*5+@S-RI8Ve&KRliUg?6wi{90*;awf54#fDPXDCa9e0Pv)^)~DTkv-@; zkF9t`FM9>AuHL%;z+JIqFsxH~hrr`7lx~d0Lw{P!aWlg%2V2nO#Ir8{swaH~-*c^T zu-WQEAGa5zj_1VOLXQbklk6r)Al;+vW`^_diu5wiBl23G#$KJg;X#Xm{_OvC`vOnd zfA!?79b&p<{ca{3Kd?5vLwlNDfq#d72Q{@{`8TezOpo#u&<)>JEYF9 z|4b`&Pd|nq(k%YafG`1vLI?_qcRwe;?ZH)@SU%&xdWo`@nas0#(*@ zitZ$PQj22}oHmyxY@GE`{m>ATf@W2YmOuIUvR1>yMW5AyB#vvOQFR5KXcMP`@~87$ z%Wd-HD#2sJ{9&19?5L6*iU-Cd^ieEy6>?-z`cAp!@$~X|RI4|e*Jq!bQ9wON2~H1c z^>$DQwkf`q;NViWEpc()@WPiT;~JVylrM4XJO+<*O_TKnOgiVbqM{4sj3vAEiZACp z+kep@$RdBqv_&#o119yLv6pLz6lLMESe+;pr&m_b_rbf9fYqw3*`rEKRRfzt(Su1u zQC9%}buxj~QnI%;<`>!7OajBoce!qJcgq7uyU1O+P zt7Ck5lul$$`EHbN|B*~C6Fxh)Y07r+UuN)pT!bmVLbJOar;d0)j225y!bhS!WX5V# z%K4D1#mu0EtN!HC3&=)WQ(g2%4%ePy5Pu;Ox!`&18jeB7s6;>h?76|g!! zEDzI(@QDzsJ>PlJVDF+st4s>iev{d5o~N=1WFH34K{BIsF&1Uxgf-W|Qd$-&3+L_W zHXu}ata&~BUMMu2+Vkvc3iXg0P&kHo6BqM}LLEz~{=s{w?*C#XfXsKr2VaYfhHXvo zOsm}%fzp93?MxDCp5g3Vtpwwi{9#_d=biT~)0Ww8V^R+(98|+ctWfqyC*0((8xisk zqOBJVo^F_C4(pLlhweLt3vv?*;aMR`umBjV{no8TE^Tlu2*)es4`pHc+d3Y&0nkcd zxlrq|;>p`&sY*8oBXJGF;&k3=Q#d)n$Yp1z*eLNJNon=*nlf9H?B{nQx@=9D@OY=d zA1%0Qu3T)mWw5@lS2!kNL(4~hy_=02k{|6{3Ayb9h+5s6sArnHttICVq>N$=RL3*r zK{{b7f1QbtKXLh}<#N4yfAHA(SBDjw$2G9`RCI!@*iG>4`49eBaxg;|HpT#-NK8BJ zT6&i!RmQJv$!(G?%sc-Kc5kiC&9`|Lv}uYpu+^vtc>cOSDgvY3hE$>KslJP8MS;vh zT3->M|E6Bc& z*KK(5a9Q5@?4!FS$&WLq+KeCO2SaPP+jMuDgL8P-o|vvosCOgZ5H617xqiOIkv%5L zB;wzCmdrfl4DbXNvG{Z;rt);IQ4u^VKNwjeqDrcToi50fZ3>E?u5>Z6F(>Gax49ZU zqU|03{)-T5{VXCSBBCppX=%xgiHM09bk0v8SkV zsrfNaHD0y6gI#OoJ{wuyQLpXT-UV-N8%G*D{j>CQ{NuRsHzs z?=IW!8@0~XtxPT;M)v8l`sihk}#J&yV;eP?USwB3LC$#tz57%^2tD!ZQeNb`_! z`Sm7Zx0^J|=6Yk~McBYmV$DWdXL5^>e%(FLe=6=CmTmsl?fSOZCwV>sj(6N#e$DgG zIK3yvy+^y}ZGn%Akw`knhp@I)~da<(r>q4>4^O~ z^K~+JyI<67o`CygwndZQJn(d?%cgsI2e8`q>7XqbUcdKbh*#}5$!M5{-c8O-foC$$ z$Fi*oG8p709knAMER+Qk2eoyQ=@&&yyWc|5>(JBWw<&KoO>xnBvUJluZ)a<|HNvD= z;EGn_z|iuno~pVS+3Y0f#;s??f{oRGY<-9}wfH0&U5<;W@FvQ*$J%jxZE)=)EhLzoU!O|5F52tC`+a9pwYQ>XJCkuX27p>M1IgyIH zpcCwKa9;TZq&VR-pHf`=RC}3NntjUOvMu=oBI1~M^qAN>Mn-qRiW};Whx#%crD_qT|?M>TL%G=5zA&@Td^NC4G&TnR{g;Bru{{9NJ#w)U;cNPD#jyyi!9 zAXBbS%Ll_ggiylR15fz@;nNIQam0C$BOOW7^8xc9z5MLv)|ZJ!)lih`T}(jz^j(Nm zM#HXV{Mh_}Utm{OBv~4T5t<*@m;d>rqdy)eot8r&HcX0!)> zc+cumxf#0Ku@%UIkP63^pOFRWA3yh_XlGJmg{ExV^HLb5GZR0}Y4D2$x6^{#O$MX! zLtWO{e4PArXb~r_&d%H>Rx8hIx5U>Y!LrS0n&*5bY>gFWYIB2GX?NKkX5wd_PwI88By~}-x@Zj< zaF_9Qf>aI;F6UzrM=&{p=CF{0xrQXZFT5uhhu1LBNfJ8UDOiyoM+ib{4A{Tnb z%cy*y?3669&+&&3tX#FEX%WwYrF$eHP(N#ny~fBN%0Sx(lWS7o$xN=9xx9T_N(~Y~ zJ%g;RLI^2~?{i*Tf$SQL(>X1mr6j~h_mq~PF;!c(C%HDFYpx1$J?5FZXbQs?s5M68 z@TCWgW>sA~MI~jtr(!5P&%i22IWr21io6p>tZD3m4%k(8BGYcGpRFqEWO~E(W_Z*Z zu1D5tG{cWk$FpXH)TTyJ6{o-)$8Ti_r9kbD-&88OeyTh#d!fzKlQ~Em-1$s*q>W@? zCEJq9DQASmPKToeLWMx=$ONg3BMTrLrLd8%M`j0s3UPE}bz-R+r^EMHo@HA-oeMQA zc#dkNsEeMUAM3SJ#zu5$Q5k=d3>_qR$yNAj{SxT`XJK}@!uqC^t5eZ-C*W|p8PIy=?QjIuk-;cNKfXHXUZtiV;N$qI>^L?3PyQ!9%~;q2aoJkDO*1AHHe(HrebfxY$n;=j zG^8}AsO_f}v#We|q9xtl9Rz(}H7W!nBX_JW^O7;gcZ$d4$wR+&^cYn?F7@tUt=g9I zH3DtJquaqFUPZT4Wm%&i4s`J-GTC;lDDzD8cR?{FxVyxvp(k_O6B~{9OMjS185L>V zKTg+Ze0m2mSTV#nr9Gm{-u46p5zk|uFFm(6nj!uV)}nNKuQO2%wV4t|OnkFsb!rUW z_E$yc{dkj5(_3*enQqkXd+ZCT5kd=|qgn*g2>jTiO?+AYE$iu*?{>)Dd~sfV@55ti zW%OQa9a|{vt1dY$ZHODnnpy(Gf3$9QegXqI;>cR}7LOL6r6ZgO*@5sbos;9;%MWe- zZ%~4#{60>DfNb4m_w^lfRbj0w?S~^DZFbw1(nX4eMQ72gSJ7EWEbwC}y_z_+&*gR_ ze9JBIzNTY#nH$uelWY$sJ7&XifNc}F4z@f??zMCKwld)TOOZnG(j`T@nM5dD{We3toh6;RnTAoFJ)B~J5b{Cs zR(m0lFKTA-!G#amR}DG=?3dX-~p8I%HBP*VsRHlNw9zjqm|?#?|ugf*417u4`zcfRUw zP{YWmTKA!zLoY9#5jik0kr!%1HKvG7jOGY6l6xE`I zkBrH2{OSqMW%pmvIN-<7{|iEb9IN;u=arpy;WnB{m1<6r{-&)fcS*^mc;WNN zSe%7G3^MY~cy7$gvC}U<$Pn|!c#FF9`bD$?!h-{$8Ks9Zia4Grtm^%8idH>^ip z!w)%6r=h#~WpT>fAATEY|S2fmg-&onms0)7MK=(Q66nHDML;7(||E z1RMQ7=-)z1mxY}9txKT_df*Cdp(d9{{C?O+VIO{pdCBiq^_iPx((mL=VW8z<(mc|x06Yk`>3BMn1-izsiqecYmbm#p@F_Zev zl{;%!WXsguImig@&?2q&aRS9;D9~VDBSRQLu4;GbweHgz?GGND(2$8XN|^ko;g1nq zpg5nZfdm#V-&VhUVM^SN959q!Sn%qaP_H|o*b%~zG>H#3fY(?wwXiem(9%XeX$&D8xcH*(nub)PWa(&2c%dGFC;;|Rl z2OSJPh+0dRvTBbRv-$Kmtzg z`vE8CLnWYaV09E8de4Dr%WfiNI}={ND~RPjCKm5_INpYSM0vYk5L_vJwF3-fs}0OE zKE(O=PP;oc&~NcA46(Dt93HM|B0lp#zGXG|7XRkwtLYYPaImOgp6tj~y*zpDDD3*_ z`!?g)Om7=dtVW7B^^pQ#ZR$tnaQx1HZAg_aLkz=lO9qz?XGy zssP&DHy|T3!#g2hTbUoV6{VgTh(hCB2#p(73v<_L7}$?>{DcCCqU1m)j_$gke`!-%V>aOrU#tj!-Lk@27n9@RP%_^c zWi4%?_HPQGYdnR3RCN#J@5~0()rJ`*A)6)t&UF^Rb}9eV-w{M$jSFfoTVdjn!QyPC zqTzdKH5$?h(bznqCLi-F z32(6OJrHe7i>`I|iTO5-_pd<6SudG0LL<-5GjBb?cx8uGg@I;=fCdL}+M_XNH{nLV%x@r>M>G-l75k3oUMhiOLC>?Ff`!6?oR($0 zBhT^4FPYhN<2<=3oM;n>cGU&=Sk!JJcRhx>BbdS*C+Ss{4trq#m2g%;53-F-G4zE8c;gh5NM znU8>7-~e$U=T0-)c0}&Ao_nbb&Dl7Cob%^hu(iPSO6dmo%8Q~?wm0XKBS>5wNBKTGONM=KH7D&x2#c2Q zz>5jkaVG%0ivZq#F&m25(LlrezdIn-&|9Q%SnCpsIk45_4?QGj*gzU0FY~SG6~6DE zdIeTDIrZ$5$*uX%=F9jK!F@`#oM)?{uR(4DQ-*OL?G%SAX@x+vVuu_({1|hQz*PET z@YY_(yNNlHGiNrED?8qCrwTbft+PP<+V%w6Er}5_Wu7jY@@aC)TI}=%GXAx$nsvaU zvhI+U8-35h5w4%{D9ycPOXn!N&Olw;qpsn{1^t?K$z;T#&_ibHPU*^64gxilDWQsp z&DZ`?71nPe)~{O2brZ>;JEfC&^Ic|k+I@fc7!hvm=Pcx$VKbVWwzm>c@d!|55KYIF z}hPj*|}-)*F_+XU1t>`QqPQ*MG#Y&N17B0-3i_chd!Ni_UYVnHNPJ1s?1K4Eb_ zA;o!lrU<^Xs3KTfp1D(6zt$dAyXGIhPa+Y*srA&pZid*y1~@7u8p7oiCKKop;ePPQ zh8-P!ee02Brn!;|KouA`Sgl8Gqhwtgf8XY-uh+k1+{PKcByPCHw(_M=QUSdS|8seW z6Df>b@mU;sAVYbF45c19&U&pBgw{>Zw0rDlXhFr6nBe8Z)~<=!2}}sb^@dizk&T4X z^Y5p%&!gj=0C~|Ozr)yn%a4ul`jW%?O~(KHY%=fAoA>!PVe)rXD=-AmK+UDh-AX2^ zLQHfR;_;GJdv%ePzWvx2b%RZfs)dt1TJIodkXX0FWKRK5ixN7a^H8A(=MKcmlJejjoM_+~*Ba z9We!ZB(u!BF>)P>LZBRdyc>yfS{UG&wlK~HH$P=y^Eg&VL1QOsz6Ukm<4>UG@1HdR zvpI=@gxm?)!Ah3Yh58OuFx!5K0aLKFqP#!tzHdmQ4eSA(-2?l}uiQYBUs^H$Dh_k! zz9oVwl_^dbEWh$6$jxX=>YsdYy*5h!p$HaNrpv3F{$gaD#DAN& zB4epL@T(SOWx_3#bt*P64%&aGbnv*-E)Y=&|3_4X94{%-OErUv9T~kww#Yp_s%_4{ z7rI$8Xhg=Yjy|_r;+Iwc_-b_}6jEV4K_}>>KKg@7=2pNQ8@E) zcxQu77LSY888AXYgsB53BWI;kCfH)SA(Y{(4~ z?fof_jJq?`Aq+FTF?}-d;PuMQn+MF}CGbP_QCJ(qd+_j~d=Pv!u?~py#h)n9_f6Q` zPYmYLmaCjzC1i|}Zv!{070qY9-yDwc+_=vioUoDGEH$(=L-jLiA#wy{tONi}RA}TSUqo@sq?`(6X_@L>MLEqeQfjoxwV+-6N3 z*pbFh9R(wksTnPq!0Sf2;5iI=lBfnH>uw}@1F8`>N2Ne&e2tkuv@Z-oH^3%8K3=QG z-uX)98>4};QW_ZonjjLihms1~lDQ>N5xQv2_Fgu|8|?nP5p2=#--cG5Q7=ZfG$_#t zTS{id4W~`r!m$qm0oSipN31;_+cSM#>&o8;&SDf3f(+U?ejiZ$I#}8H6ch#rI`H;% zm=kQ0oGZUfMG54F<&|)A3Vh+}$vhzSQ`=#I*g{H#iAOEK)@ZNP9fJKnk|t@V-2s6I z9`rN0-Sl)nlh*!&M5yV=CnzpHwL>zY+2HBQM3DF0iGXQ_H* zH!$2g>M#i+EWKT=(L<_#OrJAwni?pxqy?>q`pK@#=atZb)3@qf1~ZHlxVo95Q-$Vvs4CvMlV_Hy5A^$50VFSvFGo#oa5JwD$ zg?6O}K$2gG7?V~rTUs>%aVG8q+%-CN0Ouwiv4y6nE~}hJbR2aMN;fqbtv5Q(8q{us1j9Wg0mJFXbRt_g z@Ce5?4G}EcPm{v*gLdcZpe^FnKfIU8){%=oMpvnmGhRoPXIv5S34_O<0+%vhMx&`- z6ZM8l|3LJOeDP?-4oOE}&U~ep00)ke3%DGtOtWXM}G#0l?W0vVGA<%u+Ajc3kYATey78NYRzRpRvX~DDe`4N&v{djmFjA$bt$v zc-dPba&0BLum<+7;4C1#>!k%(*Iv33?1}IENZS3y$rOq>|6}K;PfI^7?6;TslwlQX{argHHQ~OFGxFZkv zWTOL1dB2gS3*kx>R@0(O;8K#v6=d>!5nL&7yRrf~Ygh`xc5dG0V~1c!oAe>E5m(D_ z)ImtH}IS%DYdLa<=PIK#U}#^A=Lg(^#oPzDHQyP}TJ z_<&1UdANECM+XC*{7#w*ilG_(^$cvQS5c(Z_YXjhMm*Q)&O=+F`?YNe7hw$;n#N-k z&ZM13Zx2qM%oFZ7xq{;%1i!BEii0}E!{Lh07;g}3xpyISacdrSKfY{goQ9s2~biCAdI1PHJ~Jy<{VSZZ=S8tJ*FdMIf{&oE=jAOQE`~D085}PDDXH9Q+)6 zPdTGPj$;@s^eg%^%}COcoispsK8-fz#jXzCsas;h0o*)MY36WnZ3J$vX3+Cx-QQ^g zUeb3%cSTU7HVsu};kJ+EPX<_-^l#4c zoH)_m@Uzkx4uluEyol6(mPDPQ5u{!eDi|(=m-67ID3~OJAbU@xoku2$)!-^FQi-e= zq}?98&>xjpmT*FWdP$-n=?CS-G}?-{kQtZ3)+Qw&;Na-PI`MI(=$GNB6X>tKy)Jhc z(17wNu@+vl!OrVjsBO0;R~yuqE{_ zNSlq61DSZRl%t?fczwrp&L?Fur50Wb2Q{Q9BA^`^W*5M9?a8m>#QTk*=+Tb_Rhv!9 zpx!Hw6v>N&&!1AXKukFe(IJ&2h|Z%cd_)nXWNGw=6jE`JtsAVcGEv?Tw2}tCfWH{{ zNUBBB3p(SOt*)>j>6R?8@b8zA0U3JQ-Q{}0(P1SL>Cy5Jzbju|Oak9Cv9?p3JhXbZBL9YT#fpUeqFen6w{9Zf_9aT;sS`PA_ep&R z9fyQQFTR6~D~3o_BIk8%bkA?kts_9Uc zpZLGQ$Y^NO>h^RHtGYi(GD4Nm`k*Cmry@%RPf<<;DSdn1WzoUoD00c6)H|?68e^2E zcl>BO+zB<9DjLI>6_?UjlI%zu{b!nz$jvC|uEfzWjrQVVe-y7luI#Fd>;Y_0?1O=> z8Vo?QQ2=e!BFcyyRI8&)dsUJ2Y4`8I1v;>*gUWiO%4bs=65xs)9B^OXZ>s5Vok?nn6w(fa<7jyCR6NMrA3xTl0m9i!3u zO>R#K$uFU{*zHLmtL>kU}zN9;oL_c0>K20@41q(Eq4|WReB^@GnYuGi}KxXopd440$}5 zd!*ug_%*5s0ot7m>T5wMBJ36nm*1sqUK)l1GQt=OrZjuhjik^oGbkxToP$G2#Fvhg z0->sEN@|y|Zz8&=d9)8YCS`Lqc* zzP&-2P|y~*Hcm$YaolTUJW1I!i*!YLHf<9AQm5ie+j=M>nTPBKU139+m0>3^q>y6B zc(S<&R18X6fF-sf<3<)E^ot9XLXiX$Li#=$ZMzQDtP`B@gO)S!^2%83W@)kM35}dJ z-A$6VbCfJhFx6@ArtIkjZKf-ZS0PSM#^eols*E3KM}}Nk$bu@pLJQ#WBy)v3Fz_4{ z8ecW|6u%Hld6+G^ve4Be zFz`8*+OOO#^~E$O8cz9sn#J31!?8vnR$;&eaWG}iEjaiCrVk>$Wl=FF1jiWCMOh?o z8=gV+Hb^YW*+M%!8O;oF+6R(7XhHh|0=b9qt!dap%bs8eHUB`0%zt)>v@sq><#<Ci_jYX_ zv*utV)9450_sldA+oCu!orM-U7=I$~_G1Jo|F>@+X5A~;49x6&X*->*qkz?t^a9o0h%Iq2^n)u|;=R!Pgq)5cyHdZb%@^_klZjpclW zi(F4#6)!C4wPpuyk^Xq!w;Y^9%^Fe_aMf(v*;`!4iHgL#D1L$LO$OMe6&{D;L(s{C0B1vo_ok-sQ=DTFOVsOGJB*>4WUuB5R zsf%6>9moY9XyhttBm-GE|G;s$-ged_uYXD_u98<_Wsu1r8lHn6cQ^_pkkSx~5V}{! zaX934>1Mx!0(%w{NREgyf~~k!vnFzA)ApH5h!Y81#~U2eYPhgf`dEJP%?)ihUQMI< zem&UUV~!&*yiSN6;eKd)Fx;fDzh1l(E`{TDSK%9|`2tuo%r&tK)Gs099}unA9dcH1 zrB_)0C5%DK0xF;ktB);8jn$5GfV(xZrKww}8&~-6V40k?8=(Re=eigreyh=x)=c|N zre#N79jNlPvThW*xfjZR>9~XU;tLU`)^*I&)O~0O%i=*d_63YiWxEFC0|qfuyr}Xb%b4)uY%e^{tMuy zSJ%o5M3F{R|E1m*nU5PajL?-RW%X(>kkqgmpz9y&B!j{N;4aLn{~;9l`!P96l~sL>46s2X0ATmd)fK}ufoP1JK3 zK#Ffq3j2o{4CDqCqY23l2U5o(dbxD7ku ze#V?4j*T>{E)pq!A-W0AaHXxc$Z=s{UlC-Fs z2E#7Fn{Y8@A;)*kX<;p?b7R;aJMT3F-CKX8;&YOKC-r?O*0gNjqsq0!YnTVny}AFn zus2_TBHNgBuU?VtFuu*z&#Il=IaT7#y$UR{t8#Y(Zjmn0;xAZa$q&LN^Kx0Oz*VI<@8q37q}|t7I25_+4J5U zLRT2dkm_HryFOBcK|$=846!?5kIQ$U_!zt0_}`@nS_Vnca&pfGxRi}w{gV;Ju6PBv zoyjX25&;0heAtyEFQ!Zi{euH&nPG%z?{(>D5~^@t@VB=xqchn`^KWYZV1aP%L}^<6e*hN=bDIO%|T{`oYZPvu?cA& z7hoym%URUo?_I-4D&8$1w7JXRZX=Y(L7AGog%yAd|5^jN4l+VC`nszr3X2Y&LpRR* z!`$JntVQ@fUjDW}7USNBrI%ng2iu_}(I_F1?@cDjW;=p<-w#Qp@r^$jC#3(*SU7!* z-pex+r`_+Fr24P)$a>9VPF2#Enw#`*I({E1SarsHR^8~o{xRgoE&Z&%<(~|rr;8Vj z&M}%j&gb`gHvX2r{!Bd$uLG@7@BDk2PWq;=^;}g}yM0MJb^UMG*Vm-2FTC!5h#hXR zuR9jaNM+gAgmWU@A9Jc$SNgob1-IOu1>+qiY8xi{bj4qVb4o`tD8oh{OX|#J*aeVN zkm}!RxnUN~@Q;|4!XK6?X=!OE`W6l|7+kHjSrordy8oPh&@X-GP|4V}@o#C4w<9>c zMeqA^nooKBc=y1yW4vbj)GnPVU86aMD|^PG)fVxGCgQ^bDf`>!$PyLsCmFj9I(!U* zjSL2}t}K@fD(c$u?M`pL%Mv^aJ)!(_Yfr~$c<=0;Sv{L(eGCr#+qE;+%k3&mY?72) z+WUUkD0~-HjWi_|*qhk$p{?ubE=iX(!CKSdN>VL(}I5aN*I#tGR(wrrOWEIjG zeI$^fWV2@eow$33EsulKku#QF4@(=Kta2aGKfGcjrYB zXh3E@IaqE&0ozrlH6a&$czfGk}5jU?!o$gat>k-Vv!aVTG zE=A=$5Bx;+8`5UV1ijJYluAgMkp}G>y=dWWb zoL5&k*~3MxM^Pu)5#rP%HwyIBdS)(%yL%=*{P;k4;qP34&;BwDAGo~kkFO@5TjLJ6 z^)5xD**Q|J=LtEQVihDb06}x7*tyav5Q$;Q+0i`fFNTw2r7atwEJJm|!JQtRLo&7_($dS>dPVeEhO zz;)r^zZqJg5OZS2!K8hiEBEThwxWd&5-5KK(D|q2$esu`85&s8>-WYyVxWy zXsYo1reRc&~UbN+HqtAL;Qj*8dht?`bSS<6;#fEZ6T7i!(RlE z{D}t3_>(A^*`+m_SB)S_8&{I>c_QbF&J<+>jcn;lUUSFcP$U=Hxa{s&`9&4V2mvyW z#ZF)}1`vGJ0zeFoE+ew9kX=LD$4MKd(L9Bm(u!k>9XrHdSg|zPr`<{@Kp;2rllbfd zX@&4H?kQHknZh=bnu}viuS|l-n`X;Q!&F9!(k4P)J{XY%jh|iNtPl_A@)p&HpL_HP zp0*O>=v%|0TtI$03X2@F8bS1>e zWyGA}qB(DnU=h!0h{X4NIF9nVj>BsY;S?>`xzyN;`-9l$+K8F(f5#ZKfPS9n9`Mnf zA`TOQlD24PNo>+8uo+5m+68RajE{0ASUt_~gB3-JI7Hx7R$+zku2=Qu}VA;qG<6H$W7C ziv-lBmz0xKKvDoL_FcUaomko70FY`DI!vv#3Ua}YUV%@(Ipe4CR9F*ODYwf$D9s-6 z!=g^H;DPiO8pxsw&!2|5DO#>o84W=W;vw2i$KC<*F68V}xDeXAW&KP7P1-$U3!9NB z4Fk^2P}m&O@+hVhSwElKa-bj7u*~a?VG)CdaKXXrr%uX>IP}&k#SkvJ%76Z}7yu!a z1RNF=!XU%LmGXu@P|rCP|v-{>lMX)W4eKdM6O`M{K`7Yt1Dzuk^jQ4 zwKvr3CdR}ci}nxHB>XcZ{zs)^99Zxh-%TB;e86=w8K==U?iBw7!K>+<2KWbn3GmrdWHDqEj=dfD-b0nJtPiZ*lB(Cy_YrK-@Wg0N z#T0y_21VQ6D}ZKK4-*|*y9U&U1odujX+<$G6iL^}EFS?!m*}@vsdXXatjmo}YWbTS z9cE}^3xs#VH_s5zXPcfQ<_xd{pkE(|l><8Q2%k@potyrDde{1+l{uy7`=Y3%=U6-o*b4suqUaw-*=T^7`%5c|UZ4RM?NSClFw8Jz!2hCkip-OWh)Dc_C@_u? zkH^}aBbE+Bg)GG+=BmK$g@3<60T%7m5{b{s|I>d<5NL4L|D)^67D78Y1B8m&GD2PB^5Q0_=0ym-*P!U07 zQ%HhPfv`m&5EhZ&%(*uqF7H3#qq+B;^LwiV2aw zC9n#J$CBtMGTqlh1)smGi{n}btCw%qg+psx1Qw%5fOf{v{=8~N^8#~C21H2@&jhhP zgdau-mROmC=Wg9laoFY^5k7*GepTU1l@l@Voc}VIx>q|mQTTl;5Qy_qQ-p2Z1(C}D!7=6njI%BnUY$Ubd>S{$LBX8d zPFX~(vWqz2f#qhskZYNYuAR@PmqFZ?CukRl4D2#uS0?du;sIvvt4RTh;DRafBHx|O zolaSCta2Y1Z~m>J0U7VY8_u)Uwe~11>@YG-jEd!5(jgu2f^vAI=sK!IO2v#!$oP}6 z=gE%po9}AHMeC{aA+A;qe`h`=0tW>tVBl+avv;swA~;6~8W=`behmPSvq)$4!H5zTfvy*&gABI?tDO(xpC7& z-oEQf6IQr|-h_jmymI#KTOle0IM|Z1MA42d53RH`1{UQ~0`+$te?#EoPDKM-7ry5a zyQ`xJRDJguYF3wlcJ82Q5+N;D)#=0}>D`XH5F?{0Kh>8@`+78aE+N#Gbot+qBzbo0 zIYZ&!L;>HF73qXFCQ*FHl+VRE!{42S$5l0ky;&o8Z`%Y~Kk$YNC}^>gJh@f!G)mrsrTD<0TyM{q^)Na!4~h14nJ#1IPpYj*W31iyYG>>dVXQ0`MY<1+ejS} zT-!I$kW*($CmxqQsqg9^aubk7MIec-2Z}wt-<)OSdW!uBvVnOlNhCCg2Hw?SemvpG zB|rDS=ew))cbDElHRQLo$~{%A@M0<9NfS$7y?eX$d6bYgU+e3-PRiFnaoPy6BJ+MD zyp{-{0800st?JLIP%eevM83ZD&B>62`XeONBwse#^rMV9@1o^8{D{lY04ppcU1pj0 zLW9mKQP<58lJKi-HmwIe2(=oQQI;v;w`_0d`{$ZAqreLW7}3NDOwnKLxy}AsWODtx z6U>L?lJ3EpWmUEJJ|yX7$${@8i4_Nlo^uBC%6Z@XccH-fi*$(ONs(u`((ei0N^9OT zR3_xn+cvb%{^cm2U|ekRzS(W6<@BM35;Dq!*WV}d|1>vyC3-HbnxG7qT-EQ9Ax-8n zUDe;aki^N$b?=$X2SASl?MN$1kvxmnoM8Js|LPzkq21Q{0^$M~!GSdogCz!M*hh7a zKF|?pK?zdX10(E;GOww#ignRXO=L1j#=lv7uZBLPN6W6{=w5>Pj`aY*pHY)mo^H&e zs)T!MHp`tlNHGM&w0e0jYk$#jfJck?%UES&6n8RNQIad*R62 zv9pn~WPJO77#k8Rn_Y|y-X(R8YJu6?5G@fzA`{>(Ax+G5MUh$oEwVRx4~f_^6P*xb z`Ln%r+k2d51u4L`-gqlI5QlQ7zbl#b9$!jUpB@^(_l_rV(t-hEBhpjTNjXh(Tc?2A&caZK?Kbh(U)-91I>@H~)%b3~UrKnU_1B9Y&ubsRo( zAngqYr*^>4;%I48Wg63NR`ePMxV{y+@X_H+CVufA_oq7`F^J~nQ;gnsyKGz&^_e>6 zBHx;%(+y8zm46#3j;|qGLY9;$%%m%&1dpa02fKD?3;rP=(VpA489jEyDm(s{-Q{B1 z^+V1SDib`kIvTY7^~)Oa>ZXPMU|%zz@%ZGoZ4TQ6_Yg4;5;6adRX(pHLJU3`HStv= zxO|A#;vrFmO1r98P2Z-kf2#m!gD50BP1;KHr&xNw-g8iQM6K&IO~W86Lhkb*HBdJX zXru8r;Cz|WDG&Oho5D9QVoWjHLcK(r_Fe5$GxK z8}ILa+v5jA>qjMmBwNxLKko}^D&OEeA+wRH9*j}P>+)$*F!&4%*W($Al&B3pgJun8 zzN0<-jhc^D%eWnQwl#XT2A8F-${qHtIV3YtVH$7QBDt>qx#QG9CxK3Z$_zver5C`$ zEL+|~f2LodOUE_#ZyOOdnj|sby@wtCNiU*~0GAM1#@ntU>ts4|!p~?T%(kk2?kF`3 zeqS!YsFZcK^F8MMTT3$DmxYiMtV`gFnh79u`n9m5_+bO*yP6use{8%eoQ z3YKK7=F#9Q|217v$%FwcU^d!_%H>_)|B(<8efM zwD{_+Jw=0a(L9(Y=c~895wP)bBuC?*+c(DIRl$9SMK9-KVB}y}Bce@-^2oQJq<$!K$O*s)?!eCMrJ+HrXsA_LS1e9)};SrRs5NS8u!%L1Ayx zJmgV|R)~h~j0tPBC0PBCv;lTk#{cOZ2(l~7Oa&e))h+CaABz51@e<>dj@EAQ9Y8Z7M^p^GBeRWlRztiT%jgr&)E^)cP1w~}u*zun3> zgjEPZ_Ij(m0~_AV1?~Y98KYNlecxu5^xJe;=pfxaa{-e|;EXm3U)_G1p#`kF z4Ez05q((Gmb**KDlT|QG)jCav*44|NYtb9MRW|pnt`?c}hQ`YRN4fgVK#cl3@Pl2c zkcX*5*Y!Hd1U|kj*G3|!@6R?-WM>A>?UL+`iV_Y6+o6h%g%8@E37L5Hd9k91-!vHU za_Hq0ydx_&daE~s|G`?C%;%wk&e*O)G;0XtzkM0v^VZc@8B+V!)vDPRy)rvmC4{)r z(E_EaUJlx_Gw+23dpjLA#9B=osi8?xC#5dBzu6aoZCAlVa4~wj7p#|i)V*0o z!-iKho1Wvo^d2=fnFs55QXp<*p68dKt83r^FKH*6db(80w+Ko9me%zAVTX18A0qt+ z%>S>fca#hA8Y$0mK_b&7GWEif*hG2%)+p*I%GX+l+q$h#S|ns_LG0fvx}pPv9N1e1 zkt*-C-G`v8&~J1S1H1{7JnrOH$nQ@w2ivHy;6#8he@gTgS*75wp4W8Q$a}@m8qaGo zTNmO4mmbe3Q0r!!&sHm?7G1Bmqa>H4y*iQv(GS&=4mJeHZ=w>-s%GefSX@TO*p&5R z-ji4B(8Z#4)YJ`)L2}P_C%WH;>+rjSdF8@T;nOm~=4_cYx};uk znKln_e)VDH`zt3(VG$C)@@o<{*xE1{qK4~pSJ4i1V0iSFqg}-XIuxI%?bx4PTE}9h z_NV{Nxz85yUpC*>euFzR?9N+SZCmnMU9o?Np5D#PKAOT9UQ7vkUKZ9D6|TJ4$Gxa8 zH!faWY-itP^(o}yKaBz$<_(4cDE)9_C+ug;_MM=3?Kk`7xhm3y^NQXPSg~9C%&!qb z%fdd(mfgQjx0NmplKqWt2$Mf?Cok`QirA0d#W%R7NNuMjnw8B_GB*!0{<~0nfCP3S zngS>`ddKVaZ<+@;M8U?*ik%f1D#E!Zu9Ryr;I|Uo?!eZyyDBo;=`TwL4K)Nqk%KR6b>791N1_A(T0& z;0xOkU4HiTFoRXNtR{t@4V!n@r$W}XctY#wx4nbijXb;*W!FP@4~ZT%T5HIoN8 zu?V(Hx%aX;f4pTiIN^dS{djuwA$ps|Aq`98eA_8=wE=4%T9?hKU~bEBly6hCk~aci z6DT8FrZg1HZ~Mha@SHANW23iu%OhJ;eHStDgAEPe3jH}I!ixM&wC1-yW$+8j@~7#| zn|}@!JcV7mP-K1Qsm)zvcdR>XjK0at_oAb`qoAd{SW3UYof3sQ8Q;7Rlh3Lmrowg z5&7_DH}`i|?BD-E@T&VM?otK|9R*9-bomHv@FldXJqV_@GSj1(9OtADZ!@vy!h~#2 zC!-sh;;eNAW=^))evA>7J4 ztgd&acT=ZpQXRju4K}rQs`V!d6+sqsd zIOs$AoGGSfV&s}7juu!01jsg6b{!E8k$4$MfVNnpi7u#!{V0COmj&_oE7pIO>`(KfRzGjP|AmbA_P&Uk4!=Os0rzn@;P z^!k!u88^eX_`=RuI8|Yb+iZ6Q1~^F`hupZdxjLgcO?@p#v(5U45Q&<*i0=d_tXxja z@z@S_aK1F_87lO!?6zSxFLQpDDe3M_u-$<@+3+(^S?roL@n57JuyP$;<4rLmJE@ar z#Wce{v+kk7F@{1rTye>J5=EA`hQ-s73?ft#?5m$({N&sZB4yUani(%!c2;GO?Swez zLubH{vz2mm9=LYLmm#o`)_zP(kTO(X4h>PVQDazLPF;T=XM14V8j==8S(q$WuJ1`< ztM0LTFMX*PDvUaU>)rTBlfBn^79cHzx0Lg3p~$;*H?=OzlFY<(XLaQ+*u%U34_e9% z*xj&>mu1*C_md1^(@gpNk(`%;f_`L<`++rEL=P#jYQSa`*v5D!ui0TIuVS105B-?q zg<0a~S>l7RChh$LZ&!8^teu{OZdxQb79?W4{%by%sDiS~2?E?WSf5Od zNrvrpZHMgixkEZ-hx+;z*g*L7Z+}RRVJnqz-o^@?-ee9SW6!QY$0KkMT%Rk|IX{7H zjQ15SM#ExG+>0|r>vp8tmNS!N%U9{k2ldoFBj~oAaE=bcf3^I)}Ze1kC zGVrF&OJqS$@Wf0|-4nYII}`3g_l~T{6;%1QstmzEfd2{7{{H0#09RI)OL8`BKvqC< zf6`2nTigy!1ip6b*{EK_Ae=wk^rDabyFhynD9+#SvtbTGR-^1K(> zocQBQb2h^g#tt|(fV!Av(Hq!_z~P>v6Amo$sfKp}O9EZM8;+thWD9zP{q{OgOR(i-%Oc5xx0h*rv`R_r5B|a$ca8w{& zCR~VJ_>`3G6_vv3!&8|@>>z>az<1}218z%J?4&S5Ct-$8_d?07U%`IVT1!2M{jhnWtfg3MPr!fNaH7sk5NU!vU-aZwE34 zEPOt=|ILfz9r$E-14Q9HXRNxOF>^z}%&8?U!*z)vcpfVT0ICLa(DKcNWDhCgQ(t2^ zyma0Yjy1r`UjRw)l4i0igsjLP19~$st@(pgkh#7e_5)}xvW}B&lkzrW%TI5VH7L~~ z=B)gM6TU!(JmVU(BScT{dJ$~K4*+$;Z#S9DXg`J9T$z@;-mhXfIs$f}SH=NxdIyeK zZnqcf+7u3V@#|INS!_NASiRz0wA z=cvx57Hef%q1*!-aFGG%rMSU?hpx-rUxmRdC?*@g2FZ9I-Znx4^bRELz0?z#03n-I z1In`A2@iK$4SS<_L@NB)w?aMlP`h7zb{zK6-VuDyzU@>Tzgsp;;MA>A~R_{}`LvukgCOkMB zKezn_eSRYn4zi&FfzEb)GlKwWh~zOncNY9UDq3=4QT8@sVo`S}OH4>(Wn&iK4|YT? z;gFL!8(uuG?c)SZFt~v7;T0+t>O^|vtOJntO^u#(j)aUcS<+}N8A%KPeIa!s*&m>Hb9*2p z&`o0yKJH#Qkpr|EO+1+cDAl2-jRleBV8F?ePpu^l#1haGXc|fKyq*Giv-Pq*HD+_Gmh$OjjsTx-1v;uN7pxB#d_F`XtL3B{(+3*T)&*8XdCG&B$;^ex zz&j!ETn1=;w$%@unTXd@g7}^qAc{_P!BjrxQE)TX-$a#gD#kOgD#P|=0meMNl^5-r z3tG1lw*c^NC5oGV2{z-k|J1PvgB~XR|UO`Ld%sHI}c%l$=czmIx z##)j|h7p}7>t4`6#kIs>3@ESz!CeTyTt$S1Q$lbk)~3~z#6rDMCm~J0z%*^Xo5bvu zLO64Yi$2e_OhRwCv?fKYAb?*mygM|x8xkyD@@o(a*xnw9b9(FQ(V%AFaFZn;kKtZ3 z;lkzxkk;LchW^6GJjS3w3nd?GCi0nwa*q+PeI+gdA4u}Q(enV!a7F}491{bH-f%2t zR_jJraKQc(rhgGfO5Q#jlJ`E<_9}BQ0RRAI1+T+c50RS|D^a^kSZ{lCO-i}M(_JVf zXauYML=;gXSVSKd+c_zbs0rTPurY^SM|ztuP1qg@Te=)@%d6gQ%N>&R81EM(`&{n6 zezm)XC<)F;#Gy%F(I(wi(bsrim1{|0{jomR)kMBpGga8vj0Y4jKF&V`RJbe5j=3bj{dB)OElm$&zLhB;P?xfHge&>u+PyHZ4 z5WKB|i>FKl>b1KC?PtM02R=P(1Pd)z8SG|5h>wU7nB=IzNq#hVmTuT(yXI70H?VNb z4e-vL7LCTCG3bdHI{$HgFeNpl)h%iddaxN5k@Z2dsmlsclF?#0OeKfdV`O zL3EU{@b#&XYQ3A82lnvjkI%OL4P z6P|#{eM^owN#jkhLVAg~IcDFK-Qlx%EokZmBQTJGUmY@$downK-LZG!WS0}!@w#pT zL~rquN32MVY^&RLtS7@lC~Bm>({44@I|zM)*%sYjF?mIFJ(SnYm;z|~JW|M1T)|w- zdPG9(BI%?%P4@^+v+UrV4n*6)E+jSP+QXvM@h0kM-}AAgp!qPI zJ#aIob*iU^Y(Qicb$>(mWiXJ>#)6SPG_eeOG_wg|Ai*g+l-H-70UQOQ4^I5n>ZNRl zV4yXN=OCu8(p#gxrTHHrCJS=JQtkAnUq&D}A!#2v4;EiFBpE%zZKW`ZCefJE*1|d5 zqh!1Be@ejTyYBZGg3PqixC>3>U0THCr;(r^HE%t*#>dZpuWRXtm02D4w}u~5fby78 zo=K;y%5DZmldSLY{R}52*yL%{2vK&27S?yQN2{`q&U-A}=8ati+9~1PXLM9R1*T%( z;ErN3**-mFszsM%&mXND`~$S%2Z&)tFOxOMek{6FOWAlhTh`7%8)s%tGAq5B*fg93 zPLC8Jyz+&nI0R9Lwf1mZLvmDdHLT&%6oi(ob7apdonv6IlR34ukFLp#o_HxMwY;0m z1z0Q-4`3!t?=H);18T-^95m_%Ah&46Z=|friOvI?N;PzaSVfYgD?NED$AQMHR=_ZP zG%*2wYfP&(Nb33IP&yi7Sv;)0Hak2PUc}P#Fiq?yOCDNF3P=W}S13IBG1uc4yt23pi9!(D32B|Q9w2mI#FS;tf1X> zLnSfiW3FG@UJ*q+6loOF>jgP*ewTP3+DA<9`P%T}eqDw&iLIOMS;GFg7Ypbr=ScXlOz(yF9);EW5^Rj=RCL1C2ZAm81T4MJ!^U3-OYpSMl1EaLb zlK+^|?BEuCV)5LS>S z(kn--i(NX5N&mfEhlj}Lbgb;4`h7P;JiE!(!STPLA}sASDG-h&mPyZGoSf`u-yER) z_f_z4Iw_$V*+1o!>zvADH8GYs{~f%AZ6I4F$Jb%+G2q@#*F6 zIs=L4Q9+y5O{{OqntDz3z^OKD@Tu_VH5U;H5gYB+_zoe><^gC3xYD-x5MPHv2Ud73 zU~#BYSFNd}WA#fD`;AL@XM+HEP&K3fzl!Pdsq2$k*a7_^J+n zM2TF?g4c@kqTE|`^u9vb8Q1_t&bq_j7zr{4wES+eGCyGC23T>3cW%-3&PIv@9NJ3n25AsuRJckKLw<5i zWr0Q=uwacW6hh~*5GvOAz?B5?{BSQ(gZO|}>|jy%Y_j2$h7ZV~a_=L&zw{3yfzl#( zpYbJ)21V6Kk{CNK7>=*DKy^QoRDf0{K6{l?zxccsJm*#lA)%3=M!V0=-c)F*&v!j- zEI2OvNNYd9xw&8UVfxJV=eeX(fjHP0=?}B}agp+u{>`v>BO9WuvipMM zA5djl1aM!mV-U}PA&l17YMON7431zTDX*<6GRRt3+!u=rIQi=S^EKxCnTCb!)ozHS zZ$iBOGhrLInz+Q*vn;xf8d0M&M>EP7oexz)?@<$KfnCIll#zjdfG~e_G*_jFql^#r zWRB6ADso?r7}iDTsG>n5SC;sy-F_cPwJh}sECG21@P4nd|1vmW5n~IkkqK8Ym^`7_ zE_PQZ+TCN#djLnDPi-+4tPh;}u--0K3uFidiNS)c4#<@fzhGyMJIPc=1`+>D$)KbXb8o;`=0L2edcp zbDPN?n8QoZVWw zw_x9a%^t}3KGk;Zt}(=ly)K8}^OowUOi4M=hZ z=M80WV7}|$$_=1ESSr?F`D6&+71jx|D~k;N^AOVXIexGV`RVXOlaZZ=@?flW8xF&y7Fw9d$B*GI;TdX1+VC%7;kjCV#L6?^pT_($>VT$8 zO4GQ>5^KY^Ok~(mu}bN-3Kg*Mm~5n7FeQ|a)H?5qHp8jyof!C_^kTSIWUXVyAHa~V z**P0;akR?%GC`dSeXjzCudiq#ATXtX9P!h4;wuEKWmI|RIBAi=&ILdiPm4ozG~i*` zgV~-MOuG4PL7wpXxX$ylRUd4>w5=lJ6|p^&RPk-R18*o*G-tCUmrtu*gF&`9@mbO-pcEy?OV1AK_@kvZ=I4IbVXjH%Iz zH|MqQF%;I832L!jtjQetZKIQ9mDSVioWtSh>#~!o@lO$zc*6y15xvqt>YO4iXK;D7 zJQ?>gtOQSsHw>8H^$14#KNIea=I5*RQsvc0m^K}bD9_I z6X8hiI1+r|NY72=nwXn=MeWUdxz4ih+F7}&55SF;P0EF=YRK5H59@;h*@r_}FQe$u zT5i_ZT6Djj+L*
H&s2@S3KVz0{bmGEKS7Q4Mc|7Nm0#q@>rLe1eeZ`ZaLo$!Gu zho80V&_=3L-Q(5*`+;2nP5xt-LREag+L?D6`1|YIcYI{u=s;@6oc1u=m#{mN#6@Up za_{5?j!IOw3*h4yyJLm{S-G$XHB!!kA1qaxns7PrfqdHi=N!JS`LfV_zrcIjfOl_$ z*F!=#QZ{w6WFp3&;S4C7?ApWFZwz)jFK{>x1|QS+4hPVG)P#dg!@-8*VDzo->vGuS zC5`B`qJLO&2r!(DWTj-Wqt(i9sP$b>I zY!h`%o42o|$RG#ao>pwp+PXLnvfF}vPw-y&z>3veC)IP)!jZA z20z~g(cL(5F12l?aGGq^WJxA`p!-4c)*(3i0u}TYN6P9>u>HjO{*}JKuLCM^C|B8R zylpnQjkgFoc?OcUJcjm}KDSm^7_WmLPr(n4adnv09qdHhdkF2ap64l6Q+NBz5yL#h zFnqvMxT8(v*R_yue|0yBI8L&vdzR|kA>hapc_TqW)&p~=oXB$cz!GdnY)z%~4dhBK z`<^EPa`B_-dB&+cQLOQYYwY1|K5grMdkZq}=GFf7z+#WIeLLf#`Y8N3wE!Oi zANY`soHkEg13x1M=1t(0j~tBf#_r|8SOHMPy*mnMQ{gJgI!KYoY|TGP1Kp6K%aS{I~$29OWSK%;u(lEJ=K6k)}jLU685&lf!Pc^S!_LziqBUsgq>U5PSb{89c!b{w(sp z_LsWX>hH*rH$=vzm*K6ThH`0&+C5~OrCna;?-T7qSue(4=-w**?=Y(zr?x#7-M!(0 zPxP%QC*`=zRqi$$6x0#N)3U@p_f9_+1YeeZKO>k5RnwOf8C``r!{ld!v{Q^zM{q7F zsMmQ-U->I5+HBxeycfG$!FYbFJ3jY)X#P70x!ikfsHiLqH8f1V*kEwbbFBesWYFg> zFzDKlCxX1A?v62E3Fb9>?zVPIKQTDVaQFe`?nj0~%`7dcNY(Vk%1(zh?r@hxvEDJ} z!>#<{*XB+k)E+Cxyv$QXyQ@&UE<|$ahUzAV)^S6HDo&ESLS|qM>Pi@S75;T_&;Nn! zci=TLk+PwJdgVBfmilko#;QfbxJd$>!9>Z<9R5}W2AUP|5pTN1Uo>#de+9*FUVZM_ z=UW?|G^Pr+$R&;gQNP!it2Y6@yvM_X3UJ!u6+&2A&(YC$SZlx~GWA%T`bu`9`lv-> zuzgZ-e+s`q`ro$F^r(rFBN@NfB<^8@+N^Tm?c~ws-J0?3{8@@|XJk$Li8sg$CXS`fQ8 zb>32Yxxh0fg-oK{o2DkHzp-7ywK}%~VCi2BjvzN~lDUG8Z9{AF)8YV_{9y!`L7__N zn-GhJpF7{|ZuQrxqs7y~R<5A#rf4I932#y;ugU{&Pe6M=KMkHCfqp$Kl>Q1&NbGE^ z>Kl~+_Bwg+B>2&>Vu5=DwLT`67iZ_toD{GWH*}Ma^7lW!4+qP>{m!uicVR845VNi8!-H&rVAM4e(wBi3ZXbyG5!VtX1eGsJB|2 zSII&_ZGckiXBpWXk-^GP-pM{xn+UY7M(JIJqeGRYS!(eih+v|!&vqx&o*+m+Yq|-L z-gfH~K>7@oiJ~OMsm)Dtjks4Xs{2>-!{8#TMquz_+|yZ4ukr(xeC`i{Ix1Q6~o$>FpnwoUUpicpDI<4|K#r^ zK6b|$d~A+_(t|0a%~b7a>`q}1$S!n8Kn75M3Haq94yF+N+E_~_gZbT?7ThYnhWE`@ zpwH)IESeU>N~Li z@huG**Dq$vX0e9@;DS<%`xiizfIPCC^(6;xpq2deO*Rk2KXpV;6WVdrFY)G|hRG|!lTLY013*n#2!Jv*fV%Z z0;7QN5GvJbElQDms&yJjVTlh~4^o)&!^4uhmm8on9QH|A9|0EmVbCJu$Eal2cr)OP zL^dbKsoaHa1{xiY%J5C{J%P%Y4$6p{%YYFr62UoaZ4m*ec8ri!{wAv|DdGoKpw_SO z%>aXuh(Wx`+9xvXLiZPBi4b|z#Whbhg0DaoE@PxEWAwQC+{_s?=+s^w4C(k&2F&n*Nv zKLI$4o}F8@ylot$eNIy3a~G#EV7{N@(Ysljs}p@^o-P+WcI0N1cC|!JYjzdR72}wz zPzgtpd=pWaS)c^L<~Jbusk_Z^fTY0~z_#a}sY_a#rmLjJ0u!qZh=c2=T4wxdjX&90|wn)d&VgU?1yJau84 zgH)U1x(ek$yeaRzwk;U!1mnJ86=wAqtf`^eJ3?ju9DR1L2 zt`fqs&-dV-Mi?qa4~B}JPXHP|a|H0iXw6Ce-!E$Gy#0KlB)>IH#vYl&?i#Zr!Tcx) zmF+70on2+M-0HYhbX8;NAGma=$l^q7(wv_reT#6+3~Z7;xBxXFNm>NTyu|| zs_f|MKWH#8KL1**NIe>@P-^ro3bwB$0V;@JP&A*9>xK$HF&2XnLMmPO77#Pa0cEW= z0P#e>@&%J5<=L`7+QjMFJV+UFT%-=5 z6$3ZcN^m-Sf>24*U){3T?Kcvm*s8~Z^^%vXY+g42KCr5JEFh@_Y+JYgd0JpKMu)9z zORz~o)m~K1&RWET%!Dt)Z8nZvWx3QP1n+dSebqo6$p%WdryG(B=cvrq{v|<0NR@{fn|AtipD!WTC5Md?%JeQuFJ!LY>QJV^WoYAaG9eod)gWf2Wn`dUkY(Bx81 z5Vq9BT_|x{P1G}frI!-k&&75Mk)-E=CYvfCw_4C7iNsJ~DbIX(e+HodSEK|SyE<@I zcMAZfGNILYLxGamlMw&>fPy672H;0VgpxcFTr@x%CU&KGC|cuyN&zIxq_kBerjsO@ zUP>v)FrX#Pwq#yA*uV*8xJRlO5vVsI7y6^=(wuDVcen?cEBI@ z%~T~@ZWKU*OsUOFFoHfBp&GUEm(13IR-u8+MGfZj89*aA5F-o}NIqx+@ym?F0ezc> z22kO)03jbJHf>tz*&wCjt7MDvsEzbI{PdT@+#eWz4}yHc9+gwB7;^TRyP!fK?r@a< z7Ggng(2O`Z3L;dX2+6W2HiHA13x5=dU;v4vbv_Znz>@kX&%8_4(p^;EcL{aB2Z7+d zff6^nx_7)d4Q4HYr^q^LW$pSnakq?*CV>)b*7FiPjzNg}Y`Kt~_E8@E4oFK|5xeGF z{(aZ-f%CKmU%=n37QX13N$NUdw4Qs*9dWVU_Yf)%f|m-85wjk=ok4*io*lo*lLrw> z{dxFYZJ^2az4{W?s3m;Jm%vcp)Q^b2fA^u333PIWvg z%b)Wv7&K%gsN@B_BV`DUgO?tPZ{h(5-vJIhQHziHv0loxg~l5D)eLIhMNT@7}NkHuKU2W*TrMG(vxvQgK8J`Xi&Zt zD&G<2aQK$7p@7xm-O$&hPaOZ$QTE`R%_*@eP29@N`M+e&AFov`#5w;qO#aZ^XKRcX z@BXvrH+gwaNB8Z-ihFB_ImEy;L_a<$F-|QTDje`ti%o~psvn(Jd=6FrJE-3FIH*3o z>h~v3!PK6<2@^u>{do`L!T>clJa?sy8@Yo7wx992J+-n#QUAkLBWW*4LbaJ#9kiB) z!gD;1*UV`CNafqm6p%u+a4-zCP=Q)7hiR1-3ycYL7u_}m1+vuN{sCd01t^v`sxNqn z_-aQhid}bYA%MPYQ1#nXvNdqTC&Q|09TNh_AohgAl6_}@0=;t!tnIToYg#_86!UV6 zG}YX$OE~vgIQNTcnFWf=>i3R^5Ly-D6)GB#qU48VVNhNR?nCUmesm^WYhM5Tpu@*M z7Hm5z5NjNso@$7-^e9ZV(q~0zVTyZFEr?8pr zB_~U)3qQ(9< z1%_jJ2wKt z`^mT?VAPW^B2y|oXrMff2*g&jHK#ZeYaX_iMxZr!eVntoqH>LriD!=m`$Wmm);M?s z+;VFZwSqW9g8T%gqP}%MK$e^UW>wy|+fq^K&kMHqXT2DXPQM;T1II%eJk4LZs+JSO zgz%FX1lk^_6)$#%Treqrl6`ixWx3I{0Nd+E4-x%Hc`8oAy1JZcjA!N zlO$o*5~%g;?6|hqA#$GLtg*K6g2*)nZ)%_*h5hsSLgk*}>RzHE_m-Qem(cLMQ{Icf z^LLmf!70}Ct4|!c=k)1YPQu^BJqJKdvf=(bI@l=G;{bIp>_@E-_A3{K@*Ke)7b81; z8xmD_*C@I#z&yE{dHUp;Ql>LoVMjGdZ(+1rk6TOs0JWOWSlDLPbz{34#>+lTL_7(9 z6Zh=J@#etsQR3(d{&+IpSRC(%P~Nf}b}Ee6xzzx=Av1y{GNf{No3F;2c)P+!G{D4( z#UxJru~t!rapH{Uuo$nU;ucjAB;*UECU^mVw_2lO(Uz z<8#;tz@abm@v6d4y2JJOu+}p^If3}(%4FZ&=#!61O<=emrN#mM##2ZnTW&N3 z5?bs6z-%oLJthv=k~rY?C&U5%a%h&qO;1y^x5_0L2E+IpbO+1yfiz4wkeD89ZL+Tb z2P<4)A_t5|H>x%@K;dZoH4M0D-vXq9t8sRJ$zT=fwYr(0u2{Jg6q6sY?yv4%dCV|w z3eM24<=C@<+{dvf}o(|M?jkw{ez!#$a(#p#g zKL8r4+@Cx%Frn(tyX>2V^InOf-;@b#v?EuqRaoN4t}y#WX-;uypUPGhr0Ho3Aa4Do zJL)bfaxO+89Q6c_YU7^nj8RB&BBLZ_f?czsD6((On!aNf)FYQq5+$IAhJxyHq1j*{ z`Smywv@a~9t)&UT^E_Q3sC>fTas(9Zd@&tcG5u7xAPV_lDW$o^7r;QCD^}j<6zPn2jTsfp_cUfHz~+(N^7GEW~`iXzzh!- zGxa2Z;<&1d1o8gP7ek;J{Xd{`duiAMhQKmSot!S>C?H`NShfFSPsPH7?(cXjv!icp z$b$@sSeR}!0YLlwzaVJkW8jb4xm~c9UdGa!o?Bxh1cM1lPy3796+fd>E!IJh8f-{- zA8IoZwYkte-EETOfJbS{1nG5oMSppKZ7{~W`gM%~Q!#6ehrugPBVE+sW|XIbgVu8+ z)^j(deuS1zS3IY`_bHS>*ZvC@JgsZN7yhDL&GZUzdJ{brn-aRcg@2FeijR*5s&D6C z7<;m9gbSedB>aOX@aM_0BB!>4*|~SKeG@WHLHX!`?)luC>h+t|?0nzOve)0r86)fO zpc{;=s3TZx*+U2w+Rhzqr*IKl=ZI@swpN^&ya(QT5sAQ_d+ZC&x3KhKP;8aFXB7A~ zCo4<+DviCrdeQJ6OwP}P(u@0DfRTtWD=chV+yZ!U+m=^&1ZMczL~yA&7u1aks`Ka= zf03GW+w-B4CK&;`!tAfHR^FpDhtlyp1oYzX&shT0j(|tHK4_W%w@*cnY=N`eLl`r{ zs1ONmt#7hhxdL9gwPl8#MX&#DVKMZtzar+}MY;|itCXdL1Hi>;8z)JwStSbO*N4d; zpwV|+&WKeySmn<9t-;Q~^+6&IGztgW0X$QK7iOT%O?In!tLce(Bw5%y5tQZ63nrE$ zs2n00Sn)FkIZ#OLUkT8fxzQT%WV!IO$q*&t`z(zDQjJIZ7mhyJ{>&LH_ba(G!@N6C zc{hmV^)I;*y>D0RGk@c9b61#_bD?@`!?ut@B-8CjN0EuaZOV;|<~l7!0{KmB29{Gi zk5Hvc4m)5|#fh0>uE191W^(}Mwd{d4&P(qC{F1OhY^J_$AORlOm!ag&We|$R&@k?! z01xE{g89H2PWT42q&yY#>L4tZ&J84DLQ@$2^6V(JFFix>*?s@~)ji$vnn0d5On$_c zj}fhSIZ*hAPhq%2OsN{0NlisiF@~c;TiQVv&*(zBz<0<|lOg-~Ka ztDZU;q)f*$_Dy@q{EaJP5mA%G^$@X~jDWuXTSU+0v7pHaqcJHt!z&^1)cPqa`7`kH z%S)Eh*;zjXxb{vWR-76-a7}sCi?@)zzsYW=0p^?dbg5y$K-;qVG!X2lxds!w%R$N$ zCm38nX*q_S6WMl8+Bd{1sA)&|vuZEJ(z=7lVc9L&vuh9w&8Wc)f&A_Z6S>L_q)hS^LP8%o^AiI@ zk}RO3kfm<-Q5M|3Bd%hC6|I%I>YbE<(u9oXmx1`10ce<}MFXre0YLHE(!DAih7!pC zNkF<0AuBn05|Rs$+-S$~&r>64+p@SyO8nr>G59`Ii1N@d!ZpTPG`)f{IpKQo=VTCI z@R-g5RI&`M{=)Kj;Kgkj>K!8!AtFW01k2j{7k+T!rHeBSQ+G#Y68kL|TC4ov)u)KL z_-#-pbOy#=_01$WfVB+_n+_CzyGCb;Z67ab+c@tRwdD91GMUqjKqPvCWV_Q;`_GYP z7ODNpESMyTp7Yr1?j*@kemhd8u}QABKP-W{b8_MiX;ICu?z49k|D9wKD^EmqrzB4avoOTt)mC!1|eWaD$TGkyBOEc<#%p1t_Qh` z+n(LI#*^Rqe0tgE5goN!a#2c$Kl4P)7S4Z!~yIY$N^LD^&x14JYmJICN>02_#h5_>RT&l8( z>d|lTTZaW0Q-`QPaI>`GnaZDCA*lh`rMg zihW(?0d03w^$&?dYUTx~NXpmALsZl8~`mjvO@(F#1gEXr7dMug>^4 z_GST9*KCX?eYVLTr8jDY$THv%AG6NyL{&7H=r+q*HOsM#BY~g5} zj~Q7|8+Syoh_nDm)UnDD-}40j7^_&l6tw#Pz%4eyb{A-jzl8apZh@E~&r$b~!0H@XcJ!c`qx@JNvfq!$UL1TfyLHNp z!RyW4k)Z){p%8oc92+}98d10Dippw5piD7jKn8; zKZatYeI4kZ`bzsm(sgAfukJ$Km&ExEYp&$`HhgCLrB?k${=BDy>Nj6W=l+P2jP;J( ztXR^0!XUDzOxF*2amYrTm;Z!~AF_DxCP9kp9y!e0#9}JcMWF> zS(vQoj7-2&OU(!KE?vPmM{s22{Qb9>b~{=vF+m-zC7i>!qnU|BoNg&CQm0C}mb(T` zr+l8KZrnDoSlH82Iw&UZ+Z=X(1gLUslC(S;$|{0Z(kPYn3}WjuHv$RYK1*X=I4Gq) zN^9dm`>|FLoi87NfPH%okRi@FvTNguTq=6dGN$Ld|3W(8ynGe6vOnht>E?VvN;oi1 znHA{*OJ?Tq@5727v?RvBk}PjHL56b=6pN@MH9^@YWS<|FG{_jGw8fWG(WDUyP7H|- zHUjdad~ny7_Oi9Kko^#&L5EFtxUB=g7CCd!-K{5+PNj3iOcb$45&*0iyqMO@^;#L- z-z4LTuh52W6L=5e!C%LC-_dxvS)NTMO1#$terr?9bfLYC4b`#E#fh=X!_pWlatQ`Z z&Y;rxIrrH8$kN6te7Mc8TO)(>7%R~oS)~cm!u?*7<;bq)u5+M(ec~O3+eF^ z>>OY*)47)Onvmj8^tmxUOdFMG^qd8zsIks0SSBKKM->1OW)$u!G-e)D3}woGzw_!{ zxzsDtY=Fi5K=}EbK?Dg@j8I8WO*0&#rop&Gp%9%v(U*tce$~}#N-ue!p`vKK#e4Am z8_2#Jt?%|86+~*4YrcxWGF}D^0Y+&6wO@mjwi-}Q2XMmXGelcNG4MUy%q1J4&s7p! zLYIPWa3bA?Eu`3;c8-YuS5nch&?tCUF2F4uJ9K2$k<;MsIl|$`US6Y#p}pWX94m@n zpfLmcJVC`+H)V%3rqyBpbj^IjMCzkVc!w-DOl~+#u7Vf;!l>6kJZ2KmE=$%1s8Con zL2xt#Ncz~Y(+Y?g`hJ}H3zdv7Di###KJM-qy6SN*K5c`xB|3%^vn(@B0Q;1|TA4r_ zzY>$wOLQOd8Ew@FOZ8q2q!UGI4x<65wnKzpI7ckuL(}L!S^NVXr8CQ>5ZX{jGn^Q8 zjx9hPJRGhOdsnCI3k|MeYaC{`v5JkfU*ojk`%D@)k|Xffc6vLp-i_^${b|Tk+hK80 zqiOznIx-ZeLlc|v(Wm}AKr}i$H#bYX4?^67R%F5vW2MQ3{Qa2&jjiFv%(a){)ixTK z@)p3QSJ26v?clXP8h;hV$TFBx4wy{kNm}MfiGyoO4zTj45lDQD`97mh`v8%a`@sXu z=_nboGEvm}x)T-&LL_JD&21E{UXIY{MsA|9Wke&%H3%}gOYTi4LVVVDG`451g4W7h zW;@b)1(m7^%90slFjJ0oUQdk4JR@0_LAXCS$NFm62t>U6*($!H-1mDkn&6r*RVFdb zWv4bfCSph2kV?{{jxxb8AxzMzlI1f)O&=opnKng)rDk!Tr-dSy%rQ+buvjPpgP z7XofZ(Q8NJ!UVu0^EGW{Y=tO0)7brIh}dW3n?)fK2OGMwp8B%1L?C98apkWyQP_0` z*G_KvTWU4v8D-2B`NO-@kB|^CcFTR_6m$shXCw~BZX$qUO93!t=DTf7J_YAY45zCbKc}$?%#>RBq$b%^40=bQ=wvPIg))yUy6rOkaLMYsizuJv1h$ zBn9Qc_}nSd{fiho=@zyf&;@S(r9MrT`k<3JygS(vi!4Z&36On>m^)mRoy-&uucU>% zybFw1NNaQln2zDD;EDw!dgyc;5%1c&UHx}Cvl0r8at-xLr$bD{%v~|9usKN8Ku3Go z&;C40>x#dU&UaeEJw} zc@PgxZZh9iU4Xd-wMdkae2!mtop#4c%b6hK)4X(pt zoXsz0R+h;|SQ4$A(>?QoVe_91ekUx3clbu_Nu2Q#QbI8!q%4qhBw>cr$v&ek;L*4&*3UfQ&miRP5o0eep*KIk)OR>4 z#}UkmT^a9NDZYr5cQ){v+D6Kr!69TsOd}ETKI!JYudA^-05G>SP$k5|vo)kFVryk9>cJ&WO&%e$ag(X~%s(`C>86 za3jW23MSw1tcgVMrUv4C&i=_u{cc0?aDwrSQ1Z3mMNP;VyJd+ngM?g?6h*%=VWc>f z8qsSn>wit}Ke<9~JOyunVAGbq@q?8EV@2~VL>kiaMkS>7?o|uu4Q@cA3G?HcY3TD< zXYtRnqDVpvqltEYVx|W@GKNxd%7{sh)8uZf^CX}k*9qGup2u}(j1gY~TsUTAu>tx^ zWgWRmN-u*Vj@=q+rAuKk90U7JS>hZ-vfM@UP&ZpURHm%LpS7FFAH|qqY*_IVqApm; zb0fatF`rHZW4Dyhyw;qnd_m*lj7D9nVA3yM$4MiRI-38ebR%(Dc~^`2h?^Bon?cvC zJ>VEuNO%%!lpWD(LoCbLLAk?jYQV(*F_|U8Hxt`Ykv0IhIKs{N*O(zA>N+2{=AoXc zhe0rZdWnAj0K)~HBff!RFme`Q;7LT&OO7(ID4tHpsfsqhD!L|xxM72e22KWMBk4(A zd7SnM=k&!80>h2A=%eqKT9EQe^1-046NeGtdH@2W(vJH=$`MgyJy!s{Wq`o;MLAkU5u26j3ywm*>Ejt+@D)Hb*!( zYRUWO6UY&c*QR? zML&RPCXaY#67(a;YB|8mIl}iUom5ba&o#3xK1-KK@H?pCHE>RhmOCODyC%=knP5iT z34`uXyr^%dI$KOMw*#wi8;w^xZ&Ph;=v$}G##V<}x3^vU4(P+(-a@kS*e(6`dkOQE zHGr(*>l&?=#L?6U39y?6Me)DdSM(pImya=l?!{ZGv*_9!)886cZ+Qse7N^2yQnz!& zWU}@E88a8QI!UYz=qNYV*#tN592F4!c(`fQHzL;9Z$Jp=nQEZo0ZuG&W0na}S#f)? zgv%X)uW;Kz7nAV6&%P@(gf)jCQJItGDbMy|MB-)o@S61H3pKG}EEGXXW#G>9>Rw}5 zkqqV4!4_Bso4}=Cw;CWKM8)FHAf6@=jdiXkK7LBlW*{lhYBfC*L%4g(jF+eA#uHdl z)sBuWdL|AF;(b488z}bsr936wwO9_X3!lqGp0UnNdEE%wtGI5&-g0A!Q&j(UuDU zWItt#gmhts3hA{U6nuy133-flEaMZqnfD`Q^xPE=$no5w+H0hsxz+=BbjYFVAmjWlMfd0L|iTQpr ztgxB11?0%xpurg954skge1X$2bHObeU~gtxZR+>Cf5U=gN+S}lo z8#t5)nm7E%9R2?Mf$wjiEfy6|uh$=e?U5L7;K9!R(3%>UTe2{J0krV}43X1Pi7%uR5Xw_S z<$iI2F2sT;*Nr~9a6Q8EOS?j)esKZs$AC)T-Nxk{--s}6pDKUtL1^0u=+;P8zS>3v z_ogauts{cF)l1?R6WrmGvaT1ucx?e~6@tcVY}@wnAyg-qpI-RB4Z*+tO3e8_FGBma zyR)3{cOm4fZb&)b=R)wUu1<2k--O`rTA~Wvtc1}2WfJg^=e>xK@boS_d!hU&JlrQ; zdjq`eyb11CmAjiPo$vP`_+Q@uH_Rl#L(hu$?aud`5z0efY43jj=dgdd s{oiBT%lH2{zW2WlFvz0T|K~rdq<8cCh6hc+ Configuration: """Convert the nested configuration to the flat format expected by BaseDaemon. @@ -139,17 +139,11 @@ def to_daemon_config(self) -> Configuration: "data": int(self.collector.period.total_seconds()) }, # type: ignore[union-attr] "collector_icon_filepath": {"data": self.collector.icon_filepath}, - # SentinelOne configuration (flattened) - "sentinelone_base_url": {"data": str(self.sentinelone.base_url)}, - "sentinelone_api_key": { - "data": self.sentinelone.api_key.get_secret_value() - }, - "sentinelone_time_window": {"data": self.sentinelone.time_window}, - "sentinelone_expectation_batch_size": { - "data": self.sentinelone.expectation_batch_size - }, - "sentinelone_enable_deep_visibility_search": { - "data": self.sentinelone.enable_deep_visibility_search + # Template configuration (flattened) + "template_key": {"data": self.template.key}, + "template_time_window": {"data": self.template.time_window}, + "template_expectation_batch_size": { + "data": self.template.expectation_batch_size }, }, config_base_model=self, diff --git a/template/src/models/configs/sentinelone_configs.py b/template/src/models/configs/sentinelone_configs.py deleted file mode 100644 index 904fff71..00000000 --- a/template/src/models/configs/sentinelone_configs.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Configuration for SentinelOne integration.""" - -from datetime import timedelta - -from pydantic import Field, SecretStr -from src.models.configs import ConfigBaseSettings - - -class _ConfigLoaderSentinelOne(ConfigBaseSettings): - """SentinelOne API configuration settings. - - Contains connection details, timing parameters, and retry settings - for SentinelOne API integration. - """ - - base_url: str | None = Field( - alias="SENTINELONE_BASE_URL", - default="https://api.sentinelone.com", - description="URL for the SentinelOne API.", - ) - api_key: SecretStr = Field( - alias="SENTINELONE_API_KEY", - description="API Key for the SentinelOne API.", - ) - time_window: timedelta = Field( - alias="SENTINELONE_TIME_WINDOW", - default=timedelta(hours=1), - description="Time window for SentinelOne threat searches when no date signatures are provided (ISO 8601 format).", - ) - expectation_batch_size: int = Field( - alias="SENTINELONE_EXPECTATION_BATCH_SIZE", - default=50, - description="Number of expectations to process in each batch for batch-based processing.", - ) - enable_deep_visibility_search: bool = Field( - alias="SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH", - default=False, - description="Enable deep visibility search for SentinelOne threat searches.", - ) diff --git a/template/src/models/configs/template_configs.py b/template/src/models/configs/template_configs.py new file mode 100644 index 00000000..f4570648 --- /dev/null +++ b/template/src/models/configs/template_configs.py @@ -0,0 +1,30 @@ +"""Configuration for Template integration.""" + +from datetime import timedelta + +from pydantic import Field +from src.models.configs import ConfigBaseSettings + + +class _ConfigLoaderTemplate(ConfigBaseSettings): + """Template API configuration settings. + + Contains connection details, timing parameters, and retry settings + for Template API integration. + """ + + key: str | None = Field( + alias="TEMPLATE_KEY", + default="value", + description="key value example for configuration.", + ) + time_window: timedelta = Field( + alias="TEMPLATE_TIME_WINDOW", + default=timedelta(hours=1), + description="Time window for Template threat searches when no date signatures are provided (ISO 8601 format).", + ) + expectation_batch_size: int = Field( + alias="TEMPLATE_EXPECTATION_BATCH_SIZE", + default=50, + description="Number of expectations to process in each batch for batch-based processing.", + ) diff --git a/template/src/services/client_api.py b/template/src/services/client_api.py deleted file mode 100644 index dfc4aa7d..00000000 --- a/template/src/services/client_api.py +++ /dev/null @@ -1,71 +0,0 @@ -"""SentinelOne API client for session management and core HTTP functionality.""" - -import logging -from datetime import timedelta - -import requests # type: ignore[import-untyped] - -from ..models.configs.config_loader import ConfigLoader -from .exception import SentinelOneSessionError - -LOG_PREFIX = "[SentinelOneClientAPI]" - - -class SentinelOneClientAPI: - """SentinelOne API client for managing HTTP sessions and core functionality.""" - - def __init__(self, config: ConfigLoader) -> None: - """Initialize SentinelOne API client. - - Args: - config: Configuration loader with SentinelOne settings. - - Raises: - SentinelOneValidationError: If configuration is invalid. - SentinelOneSessionError: If session setup fails. - - """ - self.logger: logging.Logger = logging.getLogger(__name__) - self.config: ConfigLoader = config - - self.base_url: str = str(config.sentinelone.base_url).rstrip("/") - self.api_key: str = config.sentinelone.api_key.get_secret_value() - - self.time_window: timedelta = config.sentinelone.time_window - - try: - self.session: requests.Session = self._create_session() - except Exception as e: - raise SentinelOneSessionError(f"Failed to create session: {e}") from e - - self.logger.debug( - f"{LOG_PREFIX} Initializing SentinelOne API client components..." - ) - - self.logger.info( - f"{LOG_PREFIX} SentinelOne API client initialized successfully" - ) - - def _create_session(self) -> requests.Session: - """Create and configure HTTP session for SentinelOne API. - - Returns: - Configured requests Session object. - - Raises: - SentinelOneSessionError: If session configuration fails. - - """ - try: - session = requests.Session() - session.headers.update( - { - "Authorization": f"ApiToken {self.api_key}", - "Content-Type": "application/json", - "Accept": "application/json", - } - ) - - return session - except Exception as e: - raise SentinelOneSessionError(f"Failed to configure session: {e}") from e diff --git a/template/src/services/converter.py b/template/src/services/converter.py index b6c77d19..1f42d005 100644 --- a/template/src/services/converter.py +++ b/template/src/services/converter.py @@ -1,117 +1,103 @@ -"""SentinelOne Data Converter to OAEV format.""" +"""Template Data Converter to OAEV format.""" import logging from typing import Any -from .exception import SentinelOneDataConversionError, SentinelOneValidationError -from .model_threat import SentinelOneThreat +from .exception import TemplateDataConversionError, TemplateValidationError +from .model_data import TemplateData -LOG_PREFIX = "[SentinelOneConverter]" +LOG_PREFIX = "[TemplateConverter]" -class SentinelOneConverter: - """Converter for SentinelOne threat data to OAEV format.""" +class TemplateConverter: + """Converter for Template data to OAEV format.""" def __init__(self) -> None: - """Initialize the SentinelOne data converter.""" + """Initialize the Template data converter.""" self.logger = logging.getLogger(__name__) - self.logger.debug(f"{LOG_PREFIX} SentinelOne converter initialized") + self.logger.debug(f"{LOG_PREFIX} Template converter initialized") - def convert_threats_to_oaev( - self, threats: list[SentinelOneThreat] - ) -> list[dict[str, Any]]: - """Convert SentinelOne threat data to OAEV format. + def convert_data_to_oaev(self, data: list[TemplateData]) -> list[dict[str, Any]]: + """Convert Template data to OAEV format. Args: - threats: List of SentinelOneThreat objects. + data: List of TemplateData objects. Returns: List of OAEV data dictionaries. Raises: - SentinelOneValidationError: If data format is invalid. - SentinelOneDataConversionError: If conversion fails. + TemplateValidationError: If data format is invalid. + TemplateDataConversionError: If conversion fails. """ - if not threats: - self.logger.debug(f"{LOG_PREFIX} No threats to convert") + if not data: + self.logger.debug(f"{LOG_PREFIX} No data to convert") return [] - if not isinstance(threats, list): - raise SentinelOneValidationError("threats must be a list") + if not isinstance(data, list): + raise TemplateValidationError("data must be a list") try: self.logger.debug( - f"{LOG_PREFIX} Converting {len(threats)} threats to OAEV format" + f"{LOG_PREFIX} Converting {len(data)} data to OAEV format" ) oaev_data_list = [] converted_count = 0 - for i, threat in enumerate(threats, 1): - if not isinstance(threat, SentinelOneThreat): + for i, single_data in enumerate(data, 1): + if not isinstance(single_data, TemplateData): self.logger.warning( - f"{LOG_PREFIX} Item {i} is not a SentinelOneThreat: {type(threat)}" + f"{LOG_PREFIX} Item {i} is not a TemplateData: {type(single_data)}" ) continue try: - oaev_data = self._convert_threat_to_oaev(threat) + oaev_data = self._convert_data_to_oaev(single_data) if oaev_data: oaev_data_list.append(oaev_data) converted_count += 1 self.logger.debug( - f"{LOG_PREFIX} Converted threat {i}/{len(threats)}: {threat.threat_id}" + f"{LOG_PREFIX} Converted data {i}/{len(data)}" ) except Exception as e: - self.logger.warning( - f"{LOG_PREFIX} Failed to convert threat {i}: {e}" - ) + self.logger.warning(f"{LOG_PREFIX} Failed to convert data {i}: {e}") self.logger.info( - f"{LOG_PREFIX} Conversion completed: {converted_count} threats -> {len(oaev_data_list)} OAEV items" + f"{LOG_PREFIX} Conversion completed: {converted_count} data -> {len(oaev_data_list)} OAEV items" ) + return oaev_data_list except Exception as e: - raise SentinelOneDataConversionError( - f"Failed to convert threats to OAEV format: {e}" + raise TemplateDataConversionError( + f"Failed to convert data to OAEV format: {e}" ) from e - def _convert_threat_to_oaev(self, threat: SentinelOneThreat) -> dict[str, Any]: - """Convert a single threat to OAEV format. + def _convert_data_to_oaev(self, data: TemplateData) -> dict[str, Any]: + """Convert a single data to OAEV format. Args: - threat: SentinelOneThreat object to convert. + data: TemplateData object to convert. Returns: OAEV formatted data dictionary. Raises: - SentinelOneValidationError: If threat data is invalid. + TemplateValidationError: If data is invalid. """ - if not threat.threat_id: - raise SentinelOneValidationError("Threat must have a threat_id") - try: - oaev_data = { - "threat_id": {"type": "fuzzy", "data": [threat.threat_id], "score": 95} - } - - if threat.hostname: - oaev_data["target_hostname_address"] = { - "type": "fuzzy", - "data": [threat.hostname], - "score": 95, - } + oaev_data = {"change-me-key": "change-me-value"} + # oaev_data to update according to the custom data object for your collector self.logger.debug( - f"{LOG_PREFIX} Successfully converted threat {threat.threat_id} to OAEV format" + f"{LOG_PREFIX} Successfully converted data to OAEV format" ) return oaev_data except Exception as e: - raise SentinelOneDataConversionError( - f"Error converting threat {threat.threat_id} to OAEV: {e}" + raise TemplateDataConversionError( + f"Error converting data to OAEV: {e}" ) from e diff --git a/template/src/services/exception.py b/template/src/services/exception.py index 6593a8b6..12686b44 100644 --- a/template/src/services/exception.py +++ b/template/src/services/exception.py @@ -1,43 +1,31 @@ -"""SentinelOne Service Exceptions.""" +"""Template Service Exceptions.""" -class SentinelOneServiceError(Exception): - """Base exception for all SentinelOne service errors.""" +class TemplateServiceError(Exception): + """Base exception for all Template service errors.""" pass -class SentinelOneExpectationError(SentinelOneServiceError): +class TemplateExpectationError(TemplateServiceError): """Raised when there's an error processing expectations.""" pass -class SentinelOneDataConversionError(SentinelOneServiceError): +class TemplateDataConversionError(TemplateServiceError): """Raised when there's an error converting data.""" pass -class SentinelOneAPIError(SentinelOneServiceError): - """Raised when there's an error with SentinelOne API operations.""" +class TemplateFetcherError(TemplateServiceError): + """Raised when there's an error with Template fetcher operations.""" pass -class SentinelOneNetworkError(SentinelOneServiceError): - """Raised when there's a network connectivity error.""" - - pass - - -class SentinelOneSessionError(SentinelOneServiceError): - """Raised when there's an error with session management.""" - - pass - - -class SentinelOneValidationError(SentinelOneServiceError): +class TemplateValidationError(TemplateServiceError): """Raised when input validation fails.""" pass diff --git a/template/src/services/expectation_service.py b/template/src/services/expectation_service.py index 320e3191..a1214c14 100644 --- a/template/src/services/expectation_service.py +++ b/template/src/services/expectation_service.py @@ -1,4 +1,4 @@ -"""SentinelOne Expectation Service with batch-based processing.""" +"""Template Expectation Service with batch-based processing.""" import logging from datetime import datetime, timezone @@ -11,16 +11,13 @@ ) from pyoaev.signatures.types import SignatureTypes -from .client_api import SentinelOneClientAPI -from .converter import SentinelOneConverter -from .exception import SentinelOneAPIError, SentinelOneExpectationError -from .fetcher_deep_visibility import FetcherDeepVisibility -from .fetcher_threat import FetcherThreat -from .fetcher_threat_events import FetcherThreatEvents -from .model_threat import SentinelOneThreat +from .converter import TemplateConverter +from .exception import TemplateExpectationError, TemplateFetcherError +from .fetcher_data import FetcherData +from .model_data import TemplateData from .utils import SignatureExtractor, TraceBuilder -LOG_PREFIX = "[SentinelOneExpectationService]" +LOG_PREFIX = "[TemplateExpectationService]" class ExpectationResult(BaseModel): @@ -40,38 +37,29 @@ class ExpectationResult(BaseModel): ) -class SentinelOneExpectationService: - """Service for processing SentinelOne expectations in batches.""" +class TemplateExpectationService: + """Service for processing Template expectations in batches.""" def __init__( self, config: Any | None = None, ) -> None: - """Initialize the SentinelOne expectation service. + """Initialize the Template expectation service. Args: config: Configuration loader for alternative initialization. Raises: - SentinelOneValidationError: If required parameters are None. + TemplateValidationError: If required parameters are None. """ self.logger: logging.Logger = logging.getLogger(__name__) - self.client_api: SentinelOneClientAPI = SentinelOneClientAPI(config) - self.converter: SentinelOneConverter = SentinelOneConverter() - self.batch_size: int = config.sentinelone.expectation_batch_size - self.enable_deep_visibility_search = ( - config.sentinelone.enable_deep_visibility_search - ) + self.converter: TemplateConverter = TemplateConverter() + self.batch_size: int = config.template.expectation_batch_size + self.time_window = config.template.time_window - self.threat_fetcher: FetcherThreat = FetcherThreat(self.client_api) - self.threat_events_fetcher: FetcherThreatEvents = FetcherThreatEvents( - self.client_api - ) - self.deep_visibility_fetcher: FetcherDeepVisibility = FetcherDeepVisibility( - self.client_api - ) + self.data_fetcher: FetcherData = FetcherData() self.logger.info( f"{LOG_PREFIX} Service initialized with batch size: {self.batch_size}" @@ -107,7 +95,7 @@ def handle_batch_expectations( - skipped_count: Number of expectations skipped due to missing end_date Raises: - SentinelOneExpectationError: If batch processing fails. + TemplateExpectationError: If batch processing fails. """ if not expectations: @@ -146,7 +134,7 @@ def handle_batch_expectations( ) error_results = [ self._create_error_result_object( - SentinelOneExpectationError(f"Batch processing error: {e}"), + TemplateExpectationError(f"Batch processing error: {e}"), expectation, ) for expectation in batch @@ -163,7 +151,7 @@ def handle_batch_expectations( return all_results, skipped_count except Exception as e: - raise SentinelOneExpectationError( + raise TemplateExpectationError( f"Error in handle_batch_expectations: {e}" ) from e @@ -236,135 +224,20 @@ def _process_expectation_batch( f"{LOG_PREFIX} Batch {batch_idx}: Found {len(process_names)} unique process names" ) - threats = self._fetch_threats_for_time_window(batch) + data = self._fetch_data_for_time_window(batch) self.logger.info( - f"{LOG_PREFIX} Batch {batch_idx}: Fetched {len(threats)} threats from time window" - ) - - static_threats = [threat for threat in threats if threat.is_static] - non_static_threats = [threat for threat in threats if not threat.is_static] - - self.logger.debug( - f"{LOG_PREFIX} Batch {batch_idx}: Processing {len(static_threats)} static threats " - f"and {len(non_static_threats)} non-static threats" + f"{LOG_PREFIX} Batch {batch_idx}: Fetched {len(data)} data from time window" ) - threat_events = {} - for threat in non_static_threats: - try: - events = self.threat_events_fetcher.fetch_events_for_threat( - threat, process_names - ) - if events: - threat_events[threat.threat_id] = events - self.logger.debug( - f"{LOG_PREFIX} Batch {batch_idx}: Non-static threat {threat.threat_id} has {len(events)} threat events" - ) - else: - self.logger.debug( - f"{LOG_PREFIX} Batch {batch_idx}: Non-static threat {threat.threat_id} - no threat events found" - ) - except Exception as e: - self.logger.error( - f"{LOG_PREFIX} Batch {batch_idx}: Error fetching threat events for non-static threat {threat.threat_id}: {e}" - ) - - if static_threats and self.enable_deep_visibility_search: - try: - sha1_to_threat = {} - unique_sha1s = [] - - for threat in static_threats: - if threat.sha1: - if threat.sha1 not in sha1_to_threat: - sha1_to_threat[threat.sha1] = threat - unique_sha1s.append(threat.sha1) - else: - self.logger.debug( - f"{LOG_PREFIX} Batch {batch_idx}: Static threat {threat.threat_id} - no SHA1 available" - ) - - if unique_sha1s: - end_time = self._extract_end_date_from_batch(batch) - if end_time is None: - end_time = datetime.now(timezone.utc) - start_time = end_time - self.client_api.time_window - - self.logger.debug( - f"{LOG_PREFIX} Batch {batch_idx}: Fetching DV events for {len(unique_sha1s)} unique SHA1s (from {len(static_threats)} static threats) in single query for time window: {start_time} to {end_time}" - ) - - sha1_to_events = ( - self.deep_visibility_fetcher.fetch_events_for_batch_sha1( - unique_sha1s, start_time, end_time - ) - ) - - for sha1, events in sha1_to_events.items(): - if sha1 in sha1_to_threat: - threat = sha1_to_threat[sha1] - if events: - threat_events[threat.threat_id] = events - self.logger.debug( - f"{LOG_PREFIX} Batch {batch_idx}: Static threat {threat.threat_id} has {len(events)} DV events" - ) - else: - self.logger.debug( - f"{LOG_PREFIX} Batch {batch_idx}: Static threat {threat.threat_id} - no DV events found" - ) - - self.logger.info( - f"{LOG_PREFIX} Batch {batch_idx}: Processed {len(static_threats)} static threats with single DV query for {len(unique_sha1s)} unique SHA1s" - ) - else: - self.logger.debug( - f"{LOG_PREFIX} Batch {batch_idx}: No valid SHA1s found for static threats" - ) - - except Exception as e: - self.logger.error( - f"{LOG_PREFIX} Batch {batch_idx}: Error fetching DV events for static threats batch: {e}" - ) - - results = self._match_threats_to_expectations( - batch, threats, threat_events, detection_helper - ) + results = self._match_data_to_expectations(batch, data, detection_helper) return results except Exception as e: - raise SentinelOneExpectationError( + raise TemplateExpectationError( f"Error processing batch {batch_idx}: {e}" ) from e - def _extract_hostnames_from_batch( - self, batch: list[DetectionExpectation | PreventionExpectation] - ) -> list[str]: - """Extract unique hostnames from a batch of expectations. - - Args: - batch: Batch of expectations. - - Returns: - List of unique hostnames. - - """ - return SignatureExtractor.extract_hostnames(batch) - - def _extract_process_names_from_batch( - self, batch: list[DetectionExpectation | PreventionExpectation] - ) -> list[str]: - """Extract unique parent process names from a batch of expectations. - - Args: - batch: Batch of expectations. - - Returns: - List of unique parent process names. - - """ - return SignatureExtractor.extract_process_names(batch) - def _extract_end_date_from_batch( self, batch: list[DetectionExpectation | PreventionExpectation] | None = None ) -> datetime | None: @@ -384,19 +257,19 @@ def _extract_end_date_from_batch( ) return end_date - def _fetch_threats_for_time_window( + def _fetch_data_for_time_window( self, batch: list[DetectionExpectation | PreventionExpectation] | None = None - ) -> list[SentinelOneThreat]: - """Fetch all threats from the configured time window or date signatures. + ) -> list[TemplateData]: + """Fetch alldata from the configured time window or date signatures. Args: batch: Optional batch of expectations to extract date filters from. Returns: - List of SentinelOneThreat objects from the time window. + List of TemplateData objects from the time window. Raises: - SentinelOneAPIError: If API call fails. + TemplateFetcherError: If fetcher fails. """ try: @@ -405,36 +278,34 @@ def _fetch_threats_for_time_window( if end_time is None: end_time = datetime.now(timezone.utc) - start_time = end_time - self.client_api.time_window + start_time = end_time - self.time_window self.logger.debug( - f"{LOG_PREFIX} Delegating threat fetching to FetcherThreat for time window: {start_time} to {end_time}" + f"{LOG_PREFIX} Delegating data fetching to FetcherData for time window: {start_time} to {end_time}" ) - return self.threat_fetcher.fetch_threats_for_time_window( + return self.data_fetcher.fetch_data_for_time_window( start_time=start_time, end_time=end_time, limit=1000, ) except Exception as e: - raise SentinelOneAPIError( - f"Error fetching threats for time window: {e}" + raise TemplateFetcherError( + f"Error fetching data for time window: {e}" ) from e - def _match_threats_to_expectations( + def _match_data_to_expectations( self, batch: list[DetectionExpectation | PreventionExpectation], - threats: list[SentinelOneThreat], - threat_events: dict[str, list[dict[str, Any]]], + data: list[TemplateData], detection_helper: Any, ) -> list[ExpectationResult]: - """Match threats and events to expectations and create results. + """Match data to expectations and create results. Args: batch: Batch of expectations. - threats: List of filtered threats. - threat_events: Dictionary mapping threat IDs to their events. + data: List of filtered data. detection_helper: OpenAEV detection helper. Returns: @@ -445,44 +316,30 @@ def _match_threats_to_expectations( for expectation in batch: try: - matched = False traces = [] - for threat in threats: - events = threat_events.get(threat.threat_id, []) - - if self._expectation_matches_threat_data( - expectation, threat, events, detection_helper + for single_data in data: + if self._expectation_matches_data( + expectation, single_data, detection_helper ): - base_url = self.client_api.base_url if self.client_api else "" - trace = TraceBuilder.create_threat_trace( - threat, base_url, events - ) + trace = TraceBuilder.create_data_trace(data) traces.append(trace) if isinstance(expectation, PreventionExpectation): - # breakpoint() - if threat.is_mitigated: - matched = True - self.logger.debug( - f"{LOG_PREFIX} Prevention expectation {expectation.inject_expectation_id}: " - f"threat {threat.threat_id} matched signature and is mitigated -> expectation satisfied" - ) - break self.logger.debug( f"{LOG_PREFIX} Prevention expectation {expectation.inject_expectation_id}: " - f"threat {threat.threat_id} matched signature but not mitigated -> continuing search" + f"data {data} matched signature and is mitigated -> expectation satisfied" ) + break else: - matched = True self.logger.debug( f"{LOG_PREFIX} Detection expectation {expectation.inject_expectation_id}: " - f"threat {threat.threat_id} matched signature -> expectation satisfied" + f"data {data} matched signature -> expectation satisfied" ) break result_dict = { - "is_valid": matched, + "is_valid": True, "traces": traces, "expectation_type": ( "detection" @@ -496,7 +353,7 @@ def _match_threats_to_expectations( self.logger.debug( f"{LOG_PREFIX} Expectation {expectation.inject_expectation_id}: " - f"matched={matched}, traces={len(traces)}" + f"matched=true, traces={len(traces)}" ) except Exception as e: @@ -504,25 +361,23 @@ def _match_threats_to_expectations( f"{LOG_PREFIX} Error matching expectation {expectation.inject_expectation_id}: {e}" ) error_result = self._create_error_result_object( - SentinelOneExpectationError(f"Matching error: {e}"), expectation + TemplateExpectationError(f"Matching error: {e}"), expectation ) results.append(error_result) return results - def _expectation_matches_threat_data( + def _expectation_matches_data( self, expectation: DetectionExpectation | PreventionExpectation, - threat: SentinelOneThreat, - events: list[dict[str, Any]], + data: TemplateData, detection_helper: Any, ) -> bool: - """Check if an expectation matches the given threat and events using converter and detection helper. + """Check if an expectation matches the given data using converter and detection helper. Args: expectation: The expectation to match. - threat: The threat data. - events: List of events for the threat. + data: The data. detection_helper: OpenAEV detection helper for matching. Returns: @@ -530,52 +385,16 @@ def _expectation_matches_threat_data( """ try: - oaev_data_list = self.converter.convert_threats_to_oaev([threat]) + oaev_data_list = self.converter.convert_data_to_oaev([data]) if not oaev_data_list: self.logger.debug( - f"{LOG_PREFIX} No OAEV data generated for threat {threat.threat_id}" + f"{LOG_PREFIX} No OAEV data generated for data {data}" ) return False oaev_data = oaev_data_list[0] - if events: - if threat.is_static: - parent_process_names = ( - SentinelOneThreat.get_parent_process_name_from_dv_events(events) - ) - else: - parent_process_names = ( - SentinelOneThreat.get_parent_process_name_from_events(events) - ) - oaev_implant_names = [ - name - for name in parent_process_names - if name.startswith("oaev-implant-") - ] - - if threat.is_static: - event_source = "DV events (parentProcessName + processName)" - else: - event_source = "threat events" - - self.logger.debug( - f"{LOG_PREFIX} Threat {threat.threat_id}: Found {len(parent_process_names)} " - f"process names from {event_source}, {len(oaev_implant_names)} with oaev-implant- prefix" - ) - - if oaev_implant_names: - # breakpoint() - oaev_data["parent_process_name"] = { - "type": "fuzzy", - "data": oaev_implant_names, - "score": 95, - } - self.logger.debug( - f"{LOG_PREFIX} Added oaev-implant parent processes to OAEV for {threat.threat_id}: {oaev_implant_names}" - ) - supported_signatures = self.get_supported_signatures() self.logger.debug( f"{LOG_PREFIX} Supported signature types: {[s.value for s in supported_signatures]}" @@ -627,12 +446,12 @@ def _expectation_matches_threat_data( if not match_result: self.logger.debug( - f"{LOG_PREFIX} {sig_type} signature failed for threat {threat.threat_id}" + f"{LOG_PREFIX} {sig_type} signature failed for data {data}" ) return False self.logger.debug( - f"{LOG_PREFIX} All signatures matched for expectation {expectation.inject_expectation_id} vs threat {threat.threat_id}" + f"{LOG_PREFIX} All signatures matched for expectation {expectation.inject_expectation_id} vs data {data}" ) return True @@ -696,7 +515,7 @@ def get_service_info(self) -> dict[str, Any]: """ return { - "service_name": "SentinelOneExpectationService", + "service_name": "TemplateExpectationService", "batch_size": self.batch_size, "supported_signatures": self.get_supported_signatures(), "flow_type": "batch_based", diff --git a/template/src/services/fetcher_data.py b/template/src/services/fetcher_data.py new file mode 100644 index 00000000..5cfb8ca8 --- /dev/null +++ b/template/src/services/fetcher_data.py @@ -0,0 +1,69 @@ +"""Template Data Fetcher.""" + +import logging +from datetime import datetime + +from .exception import ( + TemplateFetcherError, + TemplateValidationError, +) +from .model_data import TemplateData + +LOG_PREFIX = "[TemplateDataFetcher]" + + +class FetcherData: + """Fetcher for Template data using time-window based queries.""" + + def __init__(self) -> None: + """Initialize the Threat fetcher.""" + self.logger = logging.getLogger(__name__) + self.logger.debug(f"{LOG_PREFIX} Data fetcher initialized") + + def fetch_data_for_time_window( + self, + start_time: datetime, + end_time: datetime, + limit: int = 1000, + ) -> list[TemplateData]: + """Fetch all data for a given time window. + + Args: + start_time: Start time as datetime object. + end_time: End time as datetime object. + limit: Maximum number of threats to fetch. + + Returns: + List of TemplateData objects. + + Raises: + TemplateFetcherError: If fetcher fails. + TemplateValidationError: If parameters are invalid. + + """ + if not isinstance(start_time, datetime) or not isinstance(end_time, datetime): + raise TemplateValidationError( + "start_time and end_time must be datetime objects" + ) + + if start_time >= end_time: + raise TemplateValidationError("start_time must be before end_time") + + if limit <= 0: + raise TemplateValidationError("limit must be positive") + + try: + self.logger.debug( + f"{LOG_PREFIX} Fetching data for time window: {start_time} to {end_time}" + ) + + data = [TemplateData()] + # to fill with the relevant data according to your collector + + self.logger.info(f"{LOG_PREFIX} Fetched {len(data)} data for time window") + return data + + except Exception as e: + raise TemplateFetcherError( + f"Error fetching data for time window: {e}" + ) from e diff --git a/template/src/services/fetcher_deep_visibility.py b/template/src/services/fetcher_deep_visibility.py deleted file mode 100644 index ef8a6060..00000000 --- a/template/src/services/fetcher_deep_visibility.py +++ /dev/null @@ -1,533 +0,0 @@ -"""SentinelOne Deep Visibility Fetcher for static threat analysis.""" - -import logging -import re -import time -from datetime import datetime, timedelta, timezone -from typing import Any - -from requests import ConnectionError, RequestException, Timeout - -from .client_api import SentinelOneClientAPI -from .exception import ( - SentinelOneAPIError, - SentinelOneNetworkError, - SentinelOneValidationError, -) - -LOG_PREFIX = "[FetcherDeepVisibility]" -REQUEST_TIMEOUT_SECONDS = 30 -MAX_STATUS_POLL_ATTEMPTS = 30 - - -class FetcherDeepVisibility: - """Fetcher for SentinelOne Deep Visibility data for static threats.""" - - def __init__(self, client_api: SentinelOneClientAPI): - """Initialize the Deep Visibility fetcher. - - Args: - client_api: SentinelOne API client instance. - - """ - self.client_api = client_api - self.logger = logging.getLogger(__name__) - - def fetch_events_for_sha1( - self, sha1: str, start_time: datetime = None, end_time: datetime = None - ) -> list[dict[str, Any]]: - """Fetch Deep Visibility events for a specific SHA1. - - Args: - sha1: SHA1 hash to search for. - start_time: Start time for the search (optional). - end_time: End time for the search (optional). - - Returns: - List of event dictionaries compatible with threat events. - - Raises: - SentinelOneValidationError: If SHA1 is invalid. - SentinelOneAPIError: If API call fails. - SentinelOneNetworkError: If network error occurs. - - """ - if not sha1 or not isinstance(sha1, str): - raise SentinelOneValidationError("SHA1 must be a non-empty string") - - try: - self.logger.debug(f"{LOG_PREFIX} Fetching DV events for SHA1: {sha1}") - - query_response = self._init_dv_query([sha1], start_time, end_time) - - all_events = self._execute_query(query_response) - - events = [event for event in all_events if event.get("fileSha1") == sha1] - - self.logger.info( - f"{LOG_PREFIX} Successfully fetched {len(events)} DV events for SHA1: {sha1}" - ) - return events - - except ( - SentinelOneValidationError, - SentinelOneAPIError, - SentinelOneNetworkError, - ): - raise - except Exception as e: - raise SentinelOneAPIError( - f"Unexpected error fetching DV events for SHA1 {sha1}: {e}" - ) from e - - def fetch_events_for_batch_sha1( - self, - sha1_list: list[str], - start_time: datetime = None, - end_time: datetime = None, - ) -> dict[str, list[dict[str, Any]]]: - """Fetch Deep Visibility events for multiple SHA1s in a single query. - - Args: - sha1_list: List of SHA1 hashes to search for. - start_time: Start time for the search (optional). - end_time: End time for the search (optional). - - Returns: - Dictionary mapping SHA1 to list of event dictionaries. - - Raises: - SentinelOneValidationError: If SHA1 list is invalid. - SentinelOneAPIError: If API call fails. - SentinelOneNetworkError: If network error occurs. - - """ - if not sha1_list or not isinstance(sha1_list, list): - raise SentinelOneValidationError("sha1_list must be a non-empty list") - - valid_sha1s = [sha1 for sha1 in sha1_list if sha1 and isinstance(sha1, str)] - - if not valid_sha1s: - self.logger.debug(f"{LOG_PREFIX} No valid SHA1s provided") - return {} - - try: - self.logger.debug( - f"{LOG_PREFIX} Fetching DV events for {len(valid_sha1s)} SHA1s in batch" - ) - - query_response = self._init_dv_query(valid_sha1s, start_time, end_time) - - all_events = self._execute_query(query_response) - - sha1_to_events = {} - for sha1 in valid_sha1s: - sha1_to_events[sha1] = [] - - for event in all_events: - file_sha1 = event.get("fileSha1") - if file_sha1 in sha1_to_events: - sha1_to_events[file_sha1].append(event) - - total_events = sum(len(events) for events in sha1_to_events.values()) - self.logger.info( - f"{LOG_PREFIX} Successfully fetched {total_events} total DV events for {len(valid_sha1s)} SHA1s" - ) - return sha1_to_events - - except ( - SentinelOneValidationError, - SentinelOneAPIError, - SentinelOneNetworkError, - ): - raise - except Exception as e: - raise SentinelOneAPIError( - f"Unexpected error fetching DV events for batch SHA1s: {e}" - ) from e - - def _init_dv_query( - self, - sha1_list: list[str], - start_time: datetime = None, - end_time: datetime = None, - ) -> Any: - """Initialize Deep Visibility query for SHA1s. - - Args: - sha1_list: List of SHA1 hashes to search for. - start_time: Start time for the search (optional). - end_time: End time for the search (optional). - - Returns: - Query response object. - - Raises: - SentinelOneAPIError: If API call fails. - SentinelOneNetworkError: If network error occurs. - - """ - try: - endpoint = f"{self.client_api.base_url}/web/api/v2.1/dv/init-query" - - if len(sha1_list) == 1: - query_string = f'tgtFileSha1 = "{sha1_list[0]}"' - else: - sha1_values = '","'.join(sha1_list) - query_string = f'tgtFileSha1 in ("{sha1_values}")' - - if end_time is None: - end_time = datetime.now(timezone.utc) - if start_time is None: - start_time = end_time - self.client_api.time_window - - body = { - "query": query_string, - "fromDate": self._format_timestamp_for_api(start_time), - "toDate": self._format_timestamp_for_api(end_time), - } - - self.logger.debug(f"{LOG_PREFIX} Making POST request to: {endpoint}") - self.logger.debug(f"{LOG_PREFIX} DV Query: {query_string}") - self.logger.debug(f"{LOG_PREFIX} Full body payload: {body}") - - response = self.client_api.session.post( - endpoint, json=body, timeout=REQUEST_TIMEOUT_SECONDS - ) - - if response.status_code == 200: - json_data = response.json() - - class InitQueryResponse: - def __init__(self, data: dict): - self.data = InitData(data.get("data", {})) - - class InitData: - def __init__(self, data: dict): - self.query_id = data.get("queryId") - - return InitQueryResponse(json_data) - else: - error_detail = self._parse_error_response(response) - - retention_days = self._extract_retention_days(error_detail) - if retention_days and len(sha1_list) > 0: - self.logger.warning( - f"{LOG_PREFIX} DV retention limit ({retention_days} days) exceeded. Adjusting time window and retrying..." - ) - - self.logger.info( - f"{LOG_PREFIX} Waiting 60 seconds due to DV API rate limit before retry..." - ) - time.sleep(60) - - adjusted_end_time = end_time or datetime.now(timezone.utc) - adjusted_start_time = adjusted_end_time - timedelta( - days=retention_days - 1 - ) - - self.logger.debug( - f"{LOG_PREFIX} Retrying with adjusted time window: {adjusted_start_time} to {adjusted_end_time}" - ) - - return self._init_dv_query( - sha1_list, adjusted_start_time, adjusted_end_time - ) - - raise SentinelOneAPIError( - f"DV init query failed with status {response.status_code}: {error_detail}" - ) - - except SentinelOneAPIError: - raise - except (ConnectionError, Timeout) as e: - raise SentinelOneNetworkError( - f"Network error making DV init query: {e}" - ) from e - except RequestException as e: - raise SentinelOneAPIError( - f"HTTP request failed for DV init query: {e}" - ) from e - except Exception as e: - raise SentinelOneAPIError( - f"Unexpected error making DV init query: {e}" - ) from e - - def _execute_query(self, query_response: Any) -> list[dict[str, Any]]: - """Execute the Deep Visibility query. - - Args: - query_response: Response from query initialization. - - Returns: - List of event dictionaries. - - Raises: - SentinelOneValidationError: If query response is invalid. - - """ - if not query_response or not hasattr(query_response, "data"): - raise SentinelOneValidationError("Invalid query response, cannot execute") - - query_id = query_response.data.query_id - if not query_id: - raise SentinelOneValidationError("No query ID available, cannot execute") - - try: - self.logger.debug(f"{LOG_PREFIX} Executing DV query with ID: {query_id}") - - self._wait_for_query_completion(query_id) - - return self._make_real_events_query(query_id) - - except ( - SentinelOneValidationError, - SentinelOneAPIError, - SentinelOneNetworkError, - ): - raise - except Exception as e: - raise SentinelOneAPIError(f"Error executing DV query: {e}") from e - - def _wait_for_query_completion(self, query_id: str) -> None: - """Wait for DV query to complete processing. - - Args: - query_id: Query identifier to check status for. - - Raises: - SentinelOneAPIError: If API call fails. - SentinelOneNetworkError: If network error occurs. - - """ - if not query_id: - raise SentinelOneValidationError("query_id cannot be empty") - - attempt = 0 - while attempt < MAX_STATUS_POLL_ATTEMPTS: - try: - endpoint = f"{self.client_api.base_url}/web/api/v2.1/dv/query-status" - params = {"queryId": query_id} - - self.logger.debug( - f"{LOG_PREFIX} Checking query status for ID: {query_id}" - ) - - response = self.client_api.session.get( - endpoint, params=params, timeout=REQUEST_TIMEOUT_SECONDS - ) - - if response.status_code == 200: - json_data = response.json() - data = json_data.get("data", {}) - - progress_status = data.get("progressStatus", 0) - response_state = data.get("responseState", "") - - self.logger.debug( - f"{LOG_PREFIX} Query status: {response_state}, Progress: {progress_status}%" - ) - - if response_state == "FINISHED" or progress_status >= 100: - self.logger.info( - f"{LOG_PREFIX} Query {query_id} completed (Status: {response_state}, Progress: {progress_status}%)" - ) - return - - wait_time = self._calculate_wait_time(progress_status, attempt) - - self.logger.debug( - f"{LOG_PREFIX} Query still processing (Progress: {progress_status}%), waiting {wait_time}s before next check" - ) - time.sleep(wait_time) - - attempt += 1 - else: - error_detail = self._parse_error_response(response) - raise SentinelOneAPIError( - f"Query status check failed with status {response.status_code}: {error_detail}" - ) - - except (SentinelOneValidationError, SentinelOneAPIError): - raise - except (ConnectionError, Timeout) as e: - raise SentinelOneNetworkError( - f"Network error checking query status: {e}" - ) from e - except RequestException as e: - raise SentinelOneAPIError( - f"HTTP request failed for query status: {e}" - ) from e - except Exception as e: - raise SentinelOneAPIError( - f"Unexpected error checking query status: {e}" - ) from e - - raise SentinelOneAPIError( - f"Query {query_id} did not complete within {MAX_STATUS_POLL_ATTEMPTS} attempts" - ) - - def _calculate_wait_time(self, progress_status: int, attempt: int) -> int: - """Calculate optimal wait time based on query progress and attempt number. - - Args: - progress_status: Current progress percentage (0-100). - attempt: Current attempt number. - - Returns: - Wait time in seconds. - - """ - if progress_status < 10: - base_wait = 10 - elif progress_status < 50: - base_wait = 5 - elif progress_status < 90: - base_wait = 3 - else: - base_wait = 2 - - backoff = min(attempt * 2, 10) - - return base_wait + backoff - - def _make_real_events_query(self, query_id: str) -> list[dict[str, Any]]: - """Make real API call to fetch Deep Visibility events. - - Args: - query_id: Query identifier from initialization. - - Returns: - List of event dictionaries. - - Raises: - SentinelOneValidationError: If query_id is empty. - SentinelOneAPIError: If API call fails. - SentinelOneNetworkError: If network error occurs. - - """ - if not query_id: - raise SentinelOneValidationError("query_id cannot be empty") - - try: - endpoint = f"{self.client_api.base_url}/web/api/v2.1/dv/events" - params = {"queryId": query_id} - - self.logger.debug(f"{LOG_PREFIX} Making GET request to: {endpoint}") - self.logger.debug(f"{LOG_PREFIX} Query parameters: {params}") - - response = self.client_api.session.get( - endpoint, params=params, timeout=REQUEST_TIMEOUT_SECONDS - ) - - if response.status_code == 200: - json_data = response.json() - events_data = json_data.get("data", []) - - self.logger.info( - f"{LOG_PREFIX} Retrieved {len(events_data)} Deep Visibility events from API" - ) - - self.logger.debug( - f"{LOG_PREFIX} Returning {len(events_data)} DV events as dict format" - ) - return events_data - else: - error_detail = self._parse_error_response(response) - raise SentinelOneAPIError( - f"DV events query failed with status {response.status_code}: {error_detail}" - ) - - except (SentinelOneValidationError, SentinelOneAPIError): - raise - except (ConnectionError, Timeout) as e: - raise SentinelOneNetworkError( - f"Network error making DV events query: {e}" - ) from e - except RequestException as e: - raise SentinelOneAPIError( - f"HTTP request failed for DV events query: {e}" - ) from e - except Exception as e: - raise SentinelOneAPIError( - f"Unexpected error making DV events query: {e}" - ) from e - - def _format_timestamp_for_api(self, dt: datetime) -> str: - """Format datetime object for SentinelOne API. - - SentinelOne API expects timestamps in format: 2018-02-27T04:49:26.257525Z - - Args: - dt: Datetime object to format (should be timezone-aware) - - Returns: - String formatted timestamp for SentinelOne API - - """ - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - elif dt.tzinfo != timezone.utc: - dt = dt.astimezone(timezone.utc) - - return dt.replace(tzinfo=None).isoformat() + "Z" - - def _parse_error_response(self, response: Any) -> str: - """Parse error response to extract detailed error information. - - Args: - response: HTTP response object. - - Returns: - Detailed error message string. - - """ - try: - if hasattr(response, "json"): - error_data = response.json() - errors = error_data.get("errors", []) - if errors: - error_messages = [] - for error in errors: - detail = error.get("detail", "") - title = error.get("title", "") - code = error.get("code", "") - - error_msg = f"Code {code}: {title}" - if detail: - error_msg += f" - {detail}" - error_messages.append(error_msg) - - return "; ".join(error_messages) - - return getattr(response, "text", str(response)) - - except Exception as e: - return f"Error parsing response: {e}" - - def _extract_retention_days(self, error_detail: str) -> int | None: - """Extract retention period in days from error message. - - Args: - error_detail: Error detail string from API response. - - Returns: - Number of retention days if found, None otherwise. - - """ - try: - match = re.search( - r"retains data for (\d+) days?", error_detail, re.IGNORECASE - ) - if match: - return int(match.group(1)) - - match = re.search(r"retention.*?(\d+)\s*days?", error_detail, re.IGNORECASE) - if match: - return int(match.group(1)) - - return None - - except Exception as e: - self.logger.debug(f"{LOG_PREFIX} Error extracting retention days: {e}") - return None diff --git a/template/src/services/fetcher_threat.py b/template/src/services/fetcher_threat.py deleted file mode 100644 index 080e5c97..00000000 --- a/template/src/services/fetcher_threat.py +++ /dev/null @@ -1,141 +0,0 @@ -"""SentinelOne Threat Fetcher.""" - -import logging -from datetime import datetime, timezone -from typing import TYPE_CHECKING - -from requests.exceptions import ( # type: ignore[import-untyped] - ConnectionError, - RequestException, - Timeout, -) - -from .exception import ( - SentinelOneAPIError, - SentinelOneNetworkError, - SentinelOneValidationError, -) -from .model_threat import SentinelOneThreat, SentinelOneThreatsResponse - -if TYPE_CHECKING: - from .client_api import SentinelOneClientAPI - -LOG_PREFIX = "[SentinelOneThreatFetcher]" - - -class FetcherThreat: - """Fetcher for SentinelOne threat data using time-window based queries.""" - - def __init__(self, client_api: "SentinelOneClientAPI") -> None: - """Initialize the Threat fetcher. - - Args: - client_api: SentinelOne API client instance. - - Raises: - SentinelOneValidationError: If client_api is None. - - """ - if client_api is None: - raise SentinelOneValidationError("client_api cannot be None") - - self.logger = logging.getLogger(__name__) - self.client_api = client_api - self.logger.debug(f"{LOG_PREFIX} Threat fetcher initialized") - - def fetch_threats_for_time_window( - self, - start_time: datetime, - end_time: datetime, - limit: int = 1000, - ) -> list[SentinelOneThreat]: - """Fetch all threats for a given time window. - - Args: - start_time: Start time as datetime object. - end_time: End time as datetime object. - limit: Maximum number of threats to fetch. - - Returns: - List of SentinelOneThreat objects. - - Raises: - SentinelOneAPIError: If API call fails. - SentinelOneValidationError: If parameters are invalid. - - """ - if not isinstance(start_time, datetime) or not isinstance(end_time, datetime): - raise SentinelOneValidationError( - "start_time and end_time must be datetime objects" - ) - - if start_time >= end_time: - raise SentinelOneValidationError("start_time must be before end_time") - - if limit <= 0: - raise SentinelOneValidationError("limit must be positive") - - try: - start_time_str = self._format_timestamp_for_api(start_time) - end_time_str = self._format_timestamp_for_api(end_time) - - endpoint = f"{self.client_api.base_url}/web/api/v2.1/threats" - params = { - "createdAt__gte": start_time_str, - "createdAt__lt": end_time_str, - "sortOrder": "desc", - "limit": limit, - } - - self.logger.debug( - f"{LOG_PREFIX} Fetching threats for time window: {start_time_str} to {end_time_str}" - ) - - response = self.client_api.session.get(endpoint, params=params) - response.raise_for_status() - - json_data = response.json() - threats_data = json_data.get("data", []) - - response_wrapper = {"data": threats_data} - threats_response = SentinelOneThreatsResponse.from_raw_response( - response_wrapper - ) - threats = threats_response.data - - self.logger.info( - f"{LOG_PREFIX} Fetched {len(threats)} threats for time window" - ) - return threats - - except (ConnectionError, Timeout) as e: - raise SentinelOneNetworkError( - f"Network error fetching threats for time window: {e}" - ) from e - except RequestException as e: - raise SentinelOneAPIError( - f"HTTP request failed fetching threats for time window: {e}" - ) from e - except Exception as e: - raise SentinelOneAPIError( - f"Error fetching threats for time window: {e}" - ) from e - - def _format_timestamp_for_api(self, dt: datetime) -> str: - """Format datetime object for SentinelOne API. - - SentinelOne API expects timestamps in format: 2018-02-27T04:49:26.257525Z - - Args: - dt: Datetime object to format (should be timezone-aware) - - Returns: - String formatted timestamp for SentinelOne API - - """ - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - elif dt.tzinfo != timezone.utc: - dt = dt.astimezone(timezone.utc) - - return dt.replace(tzinfo=None).isoformat() + "Z" diff --git a/template/src/services/fetcher_threat_events.py b/template/src/services/fetcher_threat_events.py deleted file mode 100644 index 759dd863..00000000 --- a/template/src/services/fetcher_threat_events.py +++ /dev/null @@ -1,139 +0,0 @@ -"""SentinelOne Threat Events Fetcher.""" - -import logging -from typing import TYPE_CHECKING, Any - -from requests.exceptions import ( # type: ignore[import-untyped] - ConnectionError, - RequestException, - Timeout, -) - -from .exception import ( - SentinelOneAPIError, - SentinelOneNetworkError, - SentinelOneValidationError, -) -from .model_threat import SentinelOneThreat - -if TYPE_CHECKING: - from .client_api import SentinelOneClientAPI - -LOG_PREFIX = "[SentinelOneThreatEventsFetcher]" - - -class FetcherThreatEvents: - """Fetcher for SentinelOne threat events using API queries.""" - - def __init__(self, client_api: "SentinelOneClientAPI") -> None: - """Initialize the Threat Events fetcher. - - Args: - client_api: SentinelOne API client instance. - - Raises: - SentinelOneValidationError: If client_api is None. - - """ - if client_api is None: - raise SentinelOneValidationError("client_api cannot be None") - - self.logger = logging.getLogger(__name__) - self.client_api = client_api - self.logger.debug(f"{LOG_PREFIX} Threat events fetcher initialized") - - def fetch_events_for_threat( - self, - threat: SentinelOneThreat, - process_names: list[str] | None = None, - limit: int = 100, - ) -> list[dict[str, Any]]: - """Fetch events for a specific threat filtered by process names. - - Args: - threat: The threat to fetch events for. - process_names: Optional list of process names to filter by. - limit: Maximum number of events to fetch per process. - - Returns: - List of event dictionaries. - - Raises: - SentinelOneValidationError: If parameters are invalid. - SentinelOneAPIError: If API call fails. - - """ - if not isinstance(threat, SentinelOneThreat): - raise SentinelOneValidationError( - "threat must be a SentinelOneThreat instance" - ) - - if not threat.threat_id: - raise SentinelOneValidationError("threat must have a threat_id") - - if limit <= 0: - raise SentinelOneValidationError("limit must be positive") - - try: - self.logger.debug( - f"{LOG_PREFIX} Fetching events for threat {threat.threat_id}, {threat}" - ) - - all_events = self._fetch_all_events_for_threat(threat, limit) - - self.logger.info( - f"{LOG_PREFIX} Fetched {len(all_events)} total events for threat {threat.threat_id}" - ) - return all_events - - except (SentinelOneValidationError, SentinelOneAPIError): - raise - except Exception as e: - raise SentinelOneAPIError( - f"Unexpected error fetching events for threat {threat.threat_id}: {e}" - ) from e - - def _fetch_all_events_for_threat( - self, threat: SentinelOneThreat, limit: int - ) -> list[dict[str, Any]]: - """Fetch all events for a threat without process name filtering. - - Args: - threat: The threat to fetch events for. - limit: Maximum number of events to fetch. - - Returns: - List of event dictionaries. - - """ - try: - endpoint = f"{self.client_api.base_url}/web/api/v2.1/threats/{threat.threat_id}/explore/events" - params = {"limit": limit} - - self.logger.debug( - f"{LOG_PREFIX} Making API call to fetch events for threat {threat.threat_id}" - ) - - response = self.client_api.session.get(endpoint, params=params) - response.raise_for_status() - - json_data = response.json() - events = json_data.get("data", []) - - self.logger.debug( - f"{LOG_PREFIX} Retrieved {len(events)} events for threat {threat.threat_id}" - ) - return events - - except (ConnectionError, Timeout) as e: - raise SentinelOneNetworkError( - f"Network error fetching events for threat {threat.threat_id}: {e}" - ) from e - except RequestException as e: - raise SentinelOneAPIError( - f"HTTP request failed for threat {threat.threat_id}: {e}" - ) from e - except Exception as e: - raise SentinelOneAPIError( - f"Error fetching events for threat {threat.threat_id}: {e}" - ) from e diff --git a/template/src/services/model_data.py b/template/src/services/model_data.py new file mode 100644 index 00000000..cb17ca6f --- /dev/null +++ b/template/src/services/model_data.py @@ -0,0 +1,15 @@ +"""Template Data Models.""" + +from typing import Optional + +from pydantic import BaseModel, Field + + +class TemplateData(BaseModel): + """Template data model.""" + + key: Optional[str] = Field(None, description="Example key value") + + def __str__(self) -> str: + """Detaield representation with key debugging information.""" + return f"TemplateData(key='{self.value}'" diff --git a/template/src/services/model_threat.py b/template/src/services/model_threat.py deleted file mode 100644 index e451873d..00000000 --- a/template/src/services/model_threat.py +++ /dev/null @@ -1,132 +0,0 @@ -"""SentinelOne Threat Models.""" - -from typing import Any, Optional - -from pydantic import BaseModel, Field, PrivateAttr - - -class SentinelOneThreat(BaseModel): - """SentinelOne threat model.""" - - threat_id: str = Field(..., description="Unique identifier for the threat") - hostname: Optional[str] = Field(None, description="Agent computer name") - is_mitigated: bool = Field(False, description="Whether threat has been mitigated") - is_static: bool = Field(False, description="Whether threat is static") - sha1: Optional[str] = Field(None, description="SHA1 hash of the threat file") - _raw: Optional[dict[str, Any]] = PrivateAttr(default=None) - - def __str__(self) -> str: - """Detaield representation with key debugging information.""" - return ( - f"SentinelOneThreat(threat_id='{self.threat_id}', " - f"hostname='{self.hostname}', is_mitigated={self.is_mitigated}, is_static={self.is_static}, sha1='{self.sha1}')" - ) - - @staticmethod - def get_parent_process_name_from_events(events: list[dict]) -> list[str]: - """Extract parent process names from threat events data. - - Args: - events: List of event dictionaries from threat_events endpoint. - - Returns: - List of unique parent process names found in events. - - """ - if not events: - return [] - - parent_process_names = set() - for event in events: - parent_process_name = event.get("parentProcessName") - if parent_process_name: - parent_process_names.add(parent_process_name) - - return list(parent_process_names) - - @staticmethod - def get_parent_process_name_from_dv_events(events: list[dict]) -> list[str]: - """Extract parent process names from Deep Visibility events data. - - Checks both parentProcessName and processName fields since OAEV implants - can appear in either field. - - Args: - events: List of Deep Visibility event dictionaries. - - Returns: - List of unique process names found in events (both parent and process names). - - """ - if not events: - return [] - - process_names = set() - for event in events: - parent_process_name = event.get("parentProcessName") - if parent_process_name: - process_names.add(parent_process_name) - - process_name = event.get("processName") - if process_name: - process_names.add(process_name) - - return list(process_names) - - -class SentinelOneThreatsResponse(BaseModel): - """Response from threats endpoint.""" - - data: list[SentinelOneThreat] = Field( - default_factory=list, description="List of SentinelOne threats" - ) - - @classmethod - def from_raw_response( - cls, response_data: dict[str, Any] - ) -> "SentinelOneThreatsResponse": - """Create from raw API response. - - Args: - response_data: Raw response data from the threats API. - - Returns: - SentinelOneThreatsResponse instance with parsed threats. - - """ - threats = [] - raw_threats = response_data.get("data", []) - - for raw_threat in raw_threats: - threat_info = raw_threat.get("threatInfo", {}) - threat_id = threat_info.get("threatId") - - if threat_id: - agent_realtime_info = raw_threat.get("agentRealtimeInfo", {}) - hostname = agent_realtime_info.get("agentComputerName") - - mitigation_status = raw_threat.get("mitigationStatus", []) - is_mitigated = False - if isinstance(mitigation_status, list): - is_mitigated = any( - status.get("status") == "success" - for status in mitigation_status - if isinstance(status, dict) - ) - - detection_type = threat_info.get("detectionType", "").lower() - is_static = detection_type == "static" - - sha1 = threat_info.get("sha1") - - threat = SentinelOneThreat( - threat_id=threat_id, - hostname=hostname, - is_mitigated=is_mitigated, - is_static=is_static, - sha1=sha1, - ) - threat._raw = raw_threat - threats.append(threat) - - return cls(data=threats) diff --git a/template/src/services/trace_service.py b/template/src/services/trace_service.py index 99af6a72..d7d912fe 100644 --- a/template/src/services/trace_service.py +++ b/template/src/services/trace_service.py @@ -1,4 +1,4 @@ -"""SentinelOne Trace Service Provider.""" +"""Template Trace Service Provider.""" import logging from datetime import UTC, datetime @@ -6,34 +6,34 @@ from ..collector.models import ExpectationResult, ExpectationTrace from ..models.configs.config_loader import ConfigLoader -from .exception import SentinelOneDataConversionError, SentinelOneValidationError +from .exception import TemplateDataConversionError, TemplateValidationError -LOG_PREFIX = "[SentinelOneTraceService]" +LOG_PREFIX = "[TemplateTraceService]" -class SentinelOneTraceService: - """SentinelOne-specific trace service provider. +class TemplateTraceService: + """Template-specific trace service provider. This service extracts trace information from expectation processing results and converts them into OpenAEV expectation traces using proper Pydantic models. """ def __init__(self, config: ConfigLoader | None = None) -> None: - """Initialize the SentinelOne trace service. + """Initialize the Template trace service. Args: config: Configuration loader instance for trace service settings. Raises: - SentinelOneValidationError: If config is None. + TemplateValidationError: If config is None. """ if config is None: - raise SentinelOneValidationError("Config is required for trace service") + raise TemplateValidationError("Config is required for trace service") self.logger = logging.getLogger(__name__) self.config = config - self.logger.debug(f"{LOG_PREFIX} SentinelOne trace service initialized") + self.logger.debug(f"{LOG_PREFIX} Template trace service initialized") def create_traces_from_results( self, results: list[ExpectationResult], collector_id: str @@ -48,15 +48,15 @@ def create_traces_from_results( List of ExpectationTrace models for OpenAEV. Raises: - SentinelOneValidationError: If inputs are invalid. - SentinelOneDataConversionError: If trace creation fails. + TemplateValidationError: If inputs are invalid. + TemplateDataConversionError: If trace creation fails. """ if not collector_id: - raise SentinelOneValidationError("collector_id cannot be empty") + raise TemplateValidationError("collector_id cannot be empty") if not isinstance(results, list): - raise SentinelOneValidationError("results must be a list") + raise TemplateValidationError("results must be a list") try: valid_results = [r for r in results if r.is_valid and r.matched_alerts] @@ -100,7 +100,7 @@ def create_traces_from_results( f"{LOG_PREFIX} Trace creation returned None for expectation {expectation_id}" ) except Exception as e: - raise SentinelOneDataConversionError( + raise TemplateDataConversionError( f"Error creating trace for expectation {expectation_id}: {e}" ) from e @@ -109,10 +109,10 @@ def create_traces_from_results( ) return traces - except SentinelOneDataConversionError: + except TemplateDataConversionError: raise except Exception as e: - raise SentinelOneDataConversionError( + raise TemplateDataConversionError( f"Unexpected error creating traces from results: {e}" ) from e @@ -130,18 +130,18 @@ def _create_expectation_trace( ExpectationTrace model for OpenAEV. Raises: - SentinelOneValidationError: If inputs are invalid. - SentinelOneDataConversionError: If trace creation fails. + TemplateValidationError: If inputs are invalid. + TemplateDataConversionError: If trace creation fails. """ if not expectation_id: - raise SentinelOneValidationError("expectation_id cannot be empty") + raise TemplateValidationError("expectation_id cannot be empty") if not collector_id: - raise SentinelOneValidationError("collector_id cannot be empty") + raise TemplateValidationError("collector_id cannot be empty") if not result.matched_alerts: - raise SentinelOneValidationError( + raise TemplateValidationError( "result must have matched_alerts for trace creation" ) @@ -151,7 +151,7 @@ def _create_expectation_trace( f"{LOG_PREFIX} Processing matching data with {len(matching_data)} fields" ) - alert_name = matching_data.get("alert_name", "SentinelOne Alert") + alert_name = matching_data.get("alert_name", "Template Alert") trace_link = matching_data.get("alert_link", "") self.logger.debug(f"{LOG_PREFIX} Using trace builder URL: {trace_link}") @@ -173,10 +173,10 @@ def _create_expectation_trace( ) return trace - except SentinelOneValidationError: + except TemplateValidationError: raise except Exception as e: - raise SentinelOneDataConversionError( + raise TemplateDataConversionError( f"Error creating expectation trace: {e}" ) from e @@ -188,11 +188,11 @@ def get_service_info(self) -> dict[str, Any]: """ info = { - "service_type": "sentinelone_trace", - "supported_result_types": ["SentinelOne processing results"], + "service_type": "template_trace", + "supported_result_types": ["Template processing results"], "creates_detection_traces": True, "creates_prevention_traces": True, - "description": "Creates traces from SentinelOne expectation processing results using trace builder URLs", + "description": "Creates traces from Template expectation processing results using trace builder URLs", } self.logger.debug(f"{LOG_PREFIX} Trace service info: {info}") return info diff --git a/template/src/services/utils/__init__.py b/template/src/services/utils/__init__.py index 2aa07028..def93d4a 100644 --- a/template/src/services/utils/__init__.py +++ b/template/src/services/utils/__init__.py @@ -1,9 +1,9 @@ -from src.services.utils.config_loader import SentinelOneConfig +from src.services.utils.config_loader import TemplateConfig from src.services.utils.signature_extractor import SignatureExtractor from src.services.utils.trace_builder import TraceBuilder __all__ = [ - "SentinelOneConfig", + "TemplateConfig", "SignatureExtractor", "TraceBuilder", ] diff --git a/template/src/services/utils/config_loader.py b/template/src/services/utils/config_loader.py index c29ac902..8bac7298 100644 --- a/template/src/services/utils/config_loader.py +++ b/template/src/services/utils/config_loader.py @@ -8,11 +8,11 @@ LOG_PREFIX = "[CollectorConfig]" -class SentinelOneConfig: - """Class for loading SentinelOne configuration.""" +class TemplateConfig: + """Class for loading Template configuration.""" def __init__(self) -> None: - """Initialize SentinelOne configuration loader. + """Initialize Template configuration loader. Loads configuration from YAML files, environment variables, and defaults. Sets up logging and validates the configuration structure. @@ -22,9 +22,9 @@ def __init__(self) -> None: """ self.logger = logging.getLogger(__name__) - self.logger.debug(f"{LOG_PREFIX} Initializing SentinelOne configuration loader") + self.logger.debug(f"{LOG_PREFIX} Initializing Template configuration loader") self.load = self._load_config() - self.logger.info(f"{LOG_PREFIX} SentinelOne configuration loaded successfully") + self.logger.info(f"{LOG_PREFIX} Template configuration loaded successfully") def _load_config(self) -> ConfigLoader: """Load configuration with proper error handling and logging. @@ -56,7 +56,7 @@ def _load_config(self) -> ConfigLoader: ) self.logger.debug(f"{LOG_PREFIX} OpenAEV URL: {load_settings.openaev.url}") self.logger.debug( - f"{LOG_PREFIX} SentinelOne base URL: {load_settings.sentinelone.base_url}" + f"{LOG_PREFIX} Template key: {load_settings.template.key}" ) return load_settings diff --git a/template/src/services/utils/signature_extractor.py b/template/src/services/utils/signature_extractor.py index be61cb4c..77bd203f 100644 --- a/template/src/services/utils/signature_extractor.py +++ b/template/src/services/utils/signature_extractor.py @@ -1,4 +1,4 @@ -"""Signature extraction utilities for SentinelOne expectation processing.""" +"""Signature extraction utilities for Template expectation processing.""" from datetime import datetime, timezone from typing import TYPE_CHECKING diff --git a/template/src/services/utils/trace_builder.py b/template/src/services/utils/trace_builder.py index d44b6529..e2f79551 100644 --- a/template/src/services/utils/trace_builder.py +++ b/template/src/services/utils/trace_builder.py @@ -1,31 +1,26 @@ -"""Trace building utilities for SentinelOne expectation processing.""" +"""Trace building utilities for Template expectation processing.""" import logging from datetime import datetime, timezone from typing import TYPE_CHECKING, Any -from urllib.parse import quote if TYPE_CHECKING: - from ..model_threat import SentinelOneThreat + from ..model_data import TemplateData -LOG_PREFIX = "[SentinelOneTraceBuilder]" +LOG_PREFIX = "[TemplateTraceBuilder]" class TraceBuilder: """Utility class for building trace information.""" @staticmethod - def create_threat_trace( - threat: "SentinelOneThreat", - base_url: str, - events: list[dict[str, Any]], + def create_data_trace( + data: "TemplateData", ) -> dict[str, Any]: - """Create trace information for a threat. + """Create trace information for a data. Args: - threat: SentinelOne threat object. - base_url: Base URL for SentinelOne web interface. - events: List of events associated with the threat. + data: Template data object. Returns: Dictionary containing trace information with alert name, link, date, @@ -33,40 +28,17 @@ def create_threat_trace( """ logger = logging.getLogger(__name__) - alert_link = "" - if base_url and threat.threat_id: - try: - web_base = base_url.rstrip("/") - encoded_threat_id = quote(threat.threat_id) - alert_link = ( - f"{web_base}/incidents/threats/{encoded_threat_id}/overview" - ) - logger.debug(f"{LOG_PREFIX} Generated threat URL: {alert_link}") - except Exception as e: - logger.error(f"{LOG_PREFIX} Error generating URL: {e}") - alert_link = "" - else: - logger.warning( - f"{LOG_PREFIX} Cannot generate URL - base_url='{base_url}', threat_id='{threat.threat_id}'" - ) - alert_name = "SentinelOne Alert" - if threat.hostname: - alert_name = f"{alert_name} - {threat.hostname}" - elif threat.threat_id: - alert_name = f"{alert_name} {threat.threat_id}" + alert_name = "Template Alert" + alert_link = "http://foo.bar" trace_data = { "alert_name": alert_name, "alert_link": alert_link, "alert_date": datetime.now(timezone.utc).isoformat(), "additional_data": { - "threat_id": threat.threat_id, - "hostname": threat.hostname, - "is_mitigated": threat.is_mitigated, - "is_static": threat.is_static, - "events_count": len(events), - "data_source": "sentinelone", + "data_key_value": data.key, + "data_source": "template", }, } diff --git a/template/tests/gwt_shared.py b/template/tests/gwt_shared.py index 11513e11..2f541ba4 100644 --- a/template/tests/gwt_shared.py +++ b/template/tests/gwt_shared.py @@ -7,15 +7,14 @@ from os import environ as os_environ from typing import Any, Dict, List -from unittest.mock import Mock, patch +from unittest.mock import Mock import pytest from src.collector import Collector -from src.services.client_api import SentinelOneClientAPI -from src.services.converter import SentinelOneConverter -from src.services.exception import SentinelOneValidationError -from src.services.expectation_service import SentinelOneExpectationService -from src.services.model_threat import SentinelOneThreat +from src.services.converter import TemplateConverter +from src.services.exception import TemplateValidationError +from src.services.expectation_service import TemplateExpectationService +from src.services.model_data import TemplateData from tests.conftest import mock_env_vars from tests.services.fixtures.factories import create_test_config @@ -96,142 +95,106 @@ def given_config_with_invalid_value( # ----------------------------- -def given_initialized_converter() -> SentinelOneConverter: +def given_initialized_converter() -> TemplateConverter: """Create an initialized converter. Returns: - Initialized SentinelOneConverter instance. + Initialized TemplateConverter instance. """ - return SentinelOneConverter() - - -def given_initialized_client_api(): - """Create an initialized client API. - - Returns: - Initialized SentinelOneClientAPI instance. - - """ - config = given_test_config() - return SentinelOneClientAPI(config=config) + return TemplateConverter() def given_initialized_expectation_service(): """Create an initialized expectation service. Returns: - Initialized SentinelOneExpectationService instance. + Initialized TemplateExpectationService instance. """ config = given_test_config() - return SentinelOneExpectationService(config=config) + return TemplateExpectationService(config=config) # Data Creation Given Methods # --------------------------- -def given_threat_with_complete_data( - threat_id: str = "test_threat_123", hostname: str = "test-host.example.com" -) -> SentinelOneThreat: - """Create a threat with complete data. - - Args: - threat_id: ID for the threat. - hostname: Hostname for the threat. - - Returns: - SentinelOneThreat with complete data. - - """ - return SentinelOneThreat(threat_id=threat_id, hostname=hostname) - - -def given_threat_without_hostname( - threat_id: str = "no_hostname_threat", -) -> SentinelOneThreat: - """Create a threat without hostname. +def given_data_with_complete_data(key: str = "value123") -> TemplateData: + """Create a data with complete data. Args: - threat_id: ID for the threat. + key: example value. Returns: - SentinelOneThreat without hostname. + TemplateData with complete data. """ - return SentinelOneThreat(threat_id=threat_id, hostname=None) - + return TemplateData(key=key) -def given_threat_with_empty_id( - hostname: str = "empty-id-host.example.com", -) -> SentinelOneThreat: - """Create a threat with empty threat ID. - Args: - hostname: Hostname for the threat. +def given_data_with_empty_key() -> TemplateData: + """Create a data with empty key. Returns: - SentinelOneThreat with empty threat_id. + TemplateData with empty key. """ - return SentinelOneThreat(threat_id="", hostname=hostname) + return TemplateData(key="") -def given_multiple_threats(count: int = 3) -> List[SentinelOneThreat]: - """Create multiple threats with different data combinations. +def given_multiple_data(count: int = 3) -> List[TemplateData]: + """Create multiple data with different data combinations. Args: - count: Number of threats to create. + count: Number of data to create. Returns: - List of SentinelOneThreat objects. + List of TemplateData objects. """ - threats = [] + data = [] for i in range(count): - threat_id = f"multi_threat_{i + 1}" - hostname = f"host{i + 1}.example.com" if i % 2 == 0 else None - threats.append(SentinelOneThreat(threat_id=threat_id, hostname=hostname)) - return threats + key = f"multi_data_{i + 1}" + data.append(TemplateData(key=key)) + return data -def given_large_batch_of_threats(count: int = 100) -> List[SentinelOneThreat]: - """Create a large batch of threats for performance testing. +def given_large_batch_of_data(count: int = 100) -> List[TemplateData]: + """Create a large batch of data for performance testing. Args: - count: Number of threats to create. + count: Number of data to create. Returns: - List of SentinelOneThreat objects. + List of TemplateData objects. """ return [ - SentinelOneThreat( - threat_id=f"bulk_threat_{i}", - hostname=f"host{i}.example.com" if i % 2 == 0 else None, + TemplateData( + key=f"bulk_data_{i}", ) for i in range(count) ] -def given_mixed_valid_invalid_objects(valid_threat_id: str = "valid_mixed_123") -> List: +def given_mixed_valid_invalid_objects(valid_data_key: str = "valid_mixed_123") -> List: """Create a list with mixed valid and invalid objects. Args: - valid_threat_id: ID for the valid threat in the list. + valid_data_key: key for the valid data in the list. Returns: - List containing valid threats and invalid objects. + List containing valid data and invalid objects. """ - valid_threat = SentinelOneThreat( - threat_id=valid_threat_id, hostname="valid-mixed.example.com" + valid_data = TemplateData( + key=valid_data_key, ) return [ - valid_threat, - {"threat_id": "dict_threat"}, - "string_threat", + valid_data, + {"key_data": "dict_data"}, + "string_data", 42, ] @@ -250,60 +213,31 @@ def given_invalid_input_data() -> str: # ------------------------ -def given_mock_session_that_fails(): - """Create a mock session that fails during creation. - - Returns: - Context manager for mocking session failure. - - """ - return patch("requests.Session", side_effect=Exception("Session creation failed")) - - -def given_mock_session_with_header_failure(): - """Create a mock session that fails during header setup. - - Returns: - Context manager for mocking header setup failure. - - """ - - def create_failing_session(): - mock_session = Mock() - mock_session.headers.update.side_effect = Exception("Header setup failed") - return mock_session - - return patch("requests.Session", side_effect=create_failing_session) - - -def given_conversion_error_setup(converter: SentinelOneConverter) -> List: - """Set up threats that will cause conversion errors. +def given_conversion_error_setup(converter: TemplateConverter) -> List: + """Set up data that will cause conversion errors. Args: converter: The converter instance to mock. Returns: - List with valid threats and error-causing mock threats. + List with valid data and error-causing mock data. """ - error_threat = Mock(spec=SentinelOneThreat) - error_threat.threat_id = "error_threat" - error_threat.hostname = None + error_data = Mock(spec=TemplateData) + error_data.key = "error_data" - valid_threat = SentinelOneThreat( - threat_id="valid_error_test_123", hostname="valid-error-test.example.com" - ) + valid_data = TemplateData(key="valid_error_test_123") - original_convert = converter._convert_threat_to_oaev + original_convert = converter._convert_data_to_oaev - def mock_convert(threat): - if hasattr(threat, "threat_id") and threat.threat_id == "error_threat": + def mock_convert(data): + if hasattr(data, "key") and data.key == "error_data": raise Exception("Conversion error") - return original_convert(threat) + return original_convert(data) - converter._convert_threat_to_oaev = mock_convert + converter._convert_data_to_oaev = mock_convert - return [error_threat, valid_threat] + return [error_data, valid_data] # ======================================================================== @@ -325,72 +259,59 @@ def when_create_collector() -> Collector: return Collector() -def when_initialize_converter() -> SentinelOneConverter: +def when_initialize_converter() -> TemplateConverter: """Initialize a converter. Returns: - Initialized SentinelOneConverter instance. - - """ - return SentinelOneConverter() - - -def when_initialize_client_api(): - """Initialize a client API. - - Returns: - Initialized SentinelOneClientAPI instance. + Initialized TemplateConverter instance. """ - config = given_test_config() - return SentinelOneClientAPI(config=config) + return TemplateConverter() def when_initialize_expectation_service(): """Initialize an expectation service. Returns: - Initialized SentinelOneExpectationService instance. + Initialized TemplateExpectationService instance. """ config = given_test_config() - return SentinelOneExpectationService(config=config) + return TemplateExpectationService(config=config) # Data Processing When Methods # ---------------------------- -def when_convert_threats_to_oaev( - converter: SentinelOneConverter, threats: List -) -> List: - """Convert threats to OAEV format. +def when_convert_data_to_oaev(converter: TemplateConverter, data: List) -> List: + """Convert data to OAEV format. Args: converter: The converter instance. - threats: List of threats to convert. + data: List of data to convert. Returns: List of converted OAEV format data. """ - return converter.convert_threats_to_oaev(threats) + return converter.convert_data_to_oaev(data) def when_call_private_conversion_method( - converter: SentinelOneConverter, threat: SentinelOneThreat + converter: TemplateConverter, data: TemplateData ) -> Dict: """Call the private conversion method directly. Args: converter: The converter instance. - threat: The threat to convert. + data: The data to convert. Returns: Converted OAEV format dictionary. """ - return converter._convert_threat_to_oaev(threat) + return converter._convert_data_to_oaev(data) # Error Handling When Methods @@ -412,7 +333,7 @@ def when_create_collector_expecting_error(mock_env: Any) -> None: def when_convert_invalid_data_expecting_validation_error( - converter: SentinelOneConverter, invalid_data: Any + converter: TemplateConverter, invalid_data: Any ) -> None: """Attempt to convert invalid data and expect validation error. @@ -421,26 +342,10 @@ def when_convert_invalid_data_expecting_validation_error( invalid_data: Invalid input data. """ - with pytest.raises(SentinelOneValidationError) as exc_info: - converter.convert_threats_to_oaev(invalid_data) - - assert "threats must be a list" in str(exc_info.value) # noqa: S101 - - -def when_call_private_method_expecting_validation_error( - converter: SentinelOneConverter, threat: SentinelOneThreat -) -> None: - """Call private method and expect validation error. - - Args: - converter: The converter instance. - threat: The threat with empty threat_id. - - """ - with pytest.raises(SentinelOneValidationError) as exc_info: - converter._convert_threat_to_oaev(threat) + with pytest.raises(TemplateValidationError) as exc_info: + converter.convert_data_to_oaev(invalid_data) - assert "Threat must have a threat_id" in str(exc_info.value) # noqa: S101 + assert "data must be a list" in str(exc_info.value) # noqa: S101 # ======================================================================== @@ -487,15 +392,12 @@ def then_collector_has_valid_configuration( assert daemon_config.get("collector_name") == expected_config.get( # noqa: S101 "COLLECTOR_NAME" ) - assert daemon_config.get( # noqa: S101 - "sentinelone_base_url" - ) == expected_config.get("SENTINELONE_BASE_URL") - assert daemon_config.get( # noqa: S101 - "sentinelone_api_key" - ) == expected_config.get("SENTINELONE_API_KEY") + assert daemon_config.get("template_key") == expected_config.get( # noqa: S101 + "TEMPLATE_KEY" + ) -def then_converter_initialized_successfully(converter: SentinelOneConverter) -> None: +def then_converter_initialized_successfully(converter: TemplateConverter) -> None: """Verify converter was initialized successfully. Args: @@ -506,22 +408,8 @@ def then_converter_initialized_successfully(converter: SentinelOneConverter) -> assert converter.logger is not None # noqa: S101 -def then_client_api_initialized_successfully(client: SentinelOneClientAPI) -> None: - """Verify client API was initialized successfully. - - Args: - client: The client API instance to verify. - - """ - assert client is not None # noqa: S101 - assert hasattr(client, "config") # noqa: S101 - assert hasattr(client, "session") # noqa: S101 - assert hasattr(client, "base_url") # noqa: S101 - assert hasattr(client, "api_key") # noqa: S101 - - def then_expectation_service_initialized_successfully( - service: SentinelOneExpectationService, + service: TemplateExpectationService, ) -> None: """Verify expectation service was initialized successfully. @@ -530,9 +418,8 @@ def then_expectation_service_initialized_successfully( """ assert service is not None # noqa: S101 - assert service.client_api is not None # noqa: S101 assert service.converter is not None # noqa: S101 - assert service.threat_fetcher is not None # noqa: S101 + assert service.data_fetcher is not None # noqa: S101 # Data Validation Then Methods @@ -549,80 +436,40 @@ def then_empty_list_returned(result: List) -> None: assert result == [] # noqa: S101 -def then_single_threat_converted_completely( - result: List, threat: SentinelOneThreat -) -> None: - """Verify single threat was converted with all fields. - - Args: - result: The conversion result to verify. - threat: The original threat object. - - """ - assert len(result) == 1 # noqa: S101 - - converted = result[0] - - assert "threat_id" in converted # noqa: S101 - assert converted["threat_id"]["type"] == "fuzzy" # noqa: S101 - assert converted["threat_id"]["data"] == [threat.threat_id] # noqa: S101 - assert converted["threat_id"]["score"] == 95 # noqa: S101 - - if threat.hostname: - assert "target_hostname_address" in converted # noqa: S101 - assert converted["target_hostname_address"]["type"] == "fuzzy" # noqa: S101 - assert converted["target_hostname_address"]["data"] == [ # noqa: S101 - threat.hostname - ] - assert converted["target_hostname_address"]["score"] == 95 # noqa: S101 - - -def then_single_threat_converted_without_hostname( - result: List, threat: SentinelOneThreat -) -> None: - """Verify single threat was converted without hostname field. +def then_single_data_converted_completely(result: List, data: TemplateData) -> None: + """Verify single data was converted with all fields. Args: result: The conversion result to verify. - threat: The original threat object. + data: The original data object. """ assert len(result) == 1 # noqa: S101 converted = result[0] - assert "threat_id" in converted # noqa: S101 - assert converted["threat_id"]["data"] == [threat.threat_id] # noqa: S101 + assert "key" in converted # noqa: S101 + assert converted["key"] == [data.key] # noqa: S101 - assert "target_hostname_address" not in converted # noqa: S101 - -def then_multiple_threats_converted( - result: List, threats: List[SentinelOneThreat] -) -> None: - """Verify multiple threats were converted correctly. +def then_multiple_data_converted(result: List, data: List[TemplateData]) -> None: + """Verify multiple data were converted correctly. Args: result: The conversion result to verify. - threats: The original threats list. + data: The original data list. """ - valid_threats = [t for t in threats if t.threat_id and t.threat_id.strip()] - assert len(result) == len(valid_threats) # noqa: S101 + valid_data = [d for d in data if d.key and d.key.strip()] + assert len(result) == len(valid_data) # noqa: S101 - threat_ids = [item["threat_id"]["data"][0] for item in result] - for threat in valid_threats: - assert threat.threat_id in threat_ids # noqa: S101 + keys = [item["key"]["data"][0] for item in result] + for d in valid_data: + assert d.key in keys # noqa: S101 - items_with_hostname = [item for item in result if "target_hostname_address" in item] - expected_hostname_count = sum(1 for threat in valid_threats if threat.hostname) - assert len(items_with_hostname) == expected_hostname_count # noqa: S101 - -def then_only_valid_threats_converted( - result: List, expected_valid_count: int = 1 -) -> None: - """Verify only valid threats were converted from mixed data. +def then_only_valid_data_converted(result: List, expected_valid_count: int = 1) -> None: + """Verify only valid data were converted from mixed data. Args: result: The conversion result to verify. @@ -642,65 +489,25 @@ def then_large_batch_converted_efficiently(result: List, expected_count: int) -> """ assert len(result) == expected_count # noqa: S101 - converted_ids = {item["threat_id"]["data"][0] for item in result} - assert len(converted_ids) == expected_count # noqa: S101 + converted_keys = {item["key"]["data"][0] for item in result} + assert len(converted_keys) == expected_count # noqa: S101 -def then_private_method_converts_properly( - result: Dict, threat: SentinelOneThreat -) -> None: - """Verify private method converts threat properly. +def then_private_method_converts_properly(result: Dict, data: TemplateData) -> None: + """Verify private method converts data properly. Args: result: The conversion result to verify. - threat: The original threat object. + data: The original data object. """ assert isinstance(result, dict) # noqa: S101 - assert "threat_id" in result # noqa: S101 - assert result["threat_id"]["type"] == "fuzzy" # noqa: S101 - assert result["threat_id"]["data"] == [threat.threat_id] # noqa: S101 - assert result["threat_id"]["score"] == 95 # noqa: S101 - - if threat.hostname: - assert "target_hostname_address" in result # noqa: S101 - assert result["target_hostname_address"]["data"] == [ # noqa: S101 - threat.hostname - ] # Session and Configuration Validation Then Methods # ------------------------------------------------- -def then_session_configured_properly( - client: SentinelOneClientAPI, expected_api_key: str -) -> None: - """Verify session is configured properly. - - Args: - client: The client API instance to verify. - expected_api_key: Expected API key value. - - """ - expected_auth = f"ApiToken {expected_api_key}" - assert client.session.headers["Authorization"] == expected_auth # noqa: S101 - assert client.session.headers["Content-Type"] == "application/json" # noqa: S101 - assert client.session.headers["Accept"] == "application/json" # noqa: S101 - - -def then_base_url_normalized(client: SentinelOneClientAPI, expected_base: str) -> None: - """Verify base URL is properly normalized. - - Args: - client: The client API instance to verify. - expected_base: Expected base URL without trailing slash. - - """ - assert not client.base_url.endswith("/") # noqa: S101 - assert client.base_url == expected_base # noqa: S101 - - def then_collector_logged_initialization_success( capfd: Any, daemon_config: Dict[str, str] ) -> None: @@ -713,7 +520,7 @@ def then_collector_logged_initialization_success( """ log_records = capfd.readouterr() if daemon_config.get("collector_log_level") in ["info", "debug"]: - registered_message = "SentinelOne Collector initialized successfully" + registered_message = "Template Collector initialized successfully" assert registered_message in log_records.err # noqa: S101 diff --git a/template/tests/services/conftest.py b/template/tests/services/conftest.py index 9b09251c..d4425467 100644 --- a/template/tests/services/conftest.py +++ b/template/tests/services/conftest.py @@ -8,7 +8,7 @@ ExpectationResultFactory, ExpectationTraceFactory, MockObjectsFactory, - SentinelOneThreatFactory, + TemplateDataFactory, TestDataFactory, create_test_config, ) @@ -25,17 +25,6 @@ def mock_config(): return create_test_config() -@pytest.fixture -def mock_client_api(): - """Provide a mock SentinelOne client API. - - Returns: - Mock SentinelOne client API instance for testing. - - """ - return MockObjectsFactory.create_mock_client_api() - - @pytest.fixture def mock_detection_helper(): """Provide a mock detection helper that matches by default. @@ -59,25 +48,25 @@ def mock_detection_helper_no_match(): @pytest.fixture -def sample_threat(): - """Provide a sample SentinelOne threat. +def sample_single_data(): + """Provide a sample Template data. Returns: - SentinelOneThreat instance for testing. + TemplateData instance for testing. """ - return SentinelOneThreatFactory.build() + return TemplateDataFactory.build() @pytest.fixture -def sample_threats(): - """Provide a list of sample SentinelOne threats. +def sample_data(): + """Provide a list of sample TemplateData data. Returns: - List of 2 SentinelOneThreat instances for testing. + List of 2 TemplateData instances for testing. """ - return [SentinelOneThreatFactory.build() for _ in range(2)] + return [TemplateDataFactory.build() for _ in range(2)] @pytest.fixture @@ -151,14 +140,14 @@ def oaev_prevention_data(): @pytest.fixture -def mixed_sentinelone_data(): - """Provide mixed SentinelOne data (DV events + threats). +def mixed_template_data(): + """Provide mixed Template data (DV data). Returns: - List containing both DeepVisibilityEvent and SentinelOneThreat instances. + List containing both DeepVisibilityEvent and TemplateData instances. """ - return TestDataFactory.create_mixed_sentinelone_data() + return TestDataFactory.create_mixed_template_data() @pytest.fixture @@ -277,14 +266,14 @@ def config_factory(): @pytest.fixture -def threat_factory(): - """Provide the SentinelOneThreatFactory for creating threats. +def data_factory(): + """Provide the TemplateDataFactory for creating data. Returns: - SentinelOneThreatFactory class for generating test threats. + TemplateDataFactory class for generating test data. """ - return SentinelOneThreatFactory + return TemplateDataFactory @pytest.fixture diff --git a/template/tests/services/fixtures/factories.py b/template/tests/services/fixtures/factories.py index 6062d882..67ac5887 100644 --- a/template/tests/services/fixtures/factories.py +++ b/template/tests/services/fixtures/factories.py @@ -1,4 +1,4 @@ -"""Essential polyfactory factories for SentinelOne models and test fixtures.""" +"""Essential polyfactory factories for Template models and test fixtures.""" import os import uuid @@ -10,8 +10,8 @@ from src.collector.models import ExpectationResult, ExpectationTrace from src.models.configs.collector_configs import _ConfigLoaderOAEV from src.models.configs.config_loader import ConfigLoader, ConfigLoaderCollector -from src.models.configs.sentinelone_configs import _ConfigLoaderSentinelOne -from src.services.model_threat import SentinelOneThreat +from src.models.configs.template_configs import _ConfigLoaderTemplate +from src.services.model_data import TemplateData class ConfigLoaderOAEVFactory(ModelFactory[_ConfigLoaderOAEV]): @@ -39,10 +39,10 @@ def build(cls, **kwargs): return super().build(**kwargs) -class ConfigLoaderSentinelOneFactory(ModelFactory[_ConfigLoaderSentinelOne]): - """Factory for SentinelOne configuration. +class ConfigLoaderTemplateFactory(ModelFactory[_ConfigLoaderTemplate]): + """Factory for Template configuration. - Creates test instances of SentinelOne configuration with required + Creates test instances of Template configuration with required environment variables automatically set. """ @@ -56,11 +56,10 @@ def build(cls, **kwargs): **kwargs: Additional keyword arguments for model creation. Returns: - _ConfigLoaderSentinelOne instance with test configuration. + _ConfigLoaderTemplate instance with test configuration. """ - os.environ["SENTINELONE_API_KEY"] = "test-sentinelone-api-key" - os.environ["SENTINELONE_BASE_URL"] = "https://test-sentinelone.example.com" + os.environ["TEMPLATE_KEY"] = "test-template-key" return super().build(**kwargs) @@ -73,29 +72,28 @@ class ConfigLoaderCollectorFactory(ModelFactory[ConfigLoaderCollector]): __check_model__ = False - id = Use(lambda: f"sentinelone--{uuid.uuid4()}") - name = "SentinelOne" + id = Use(lambda: f"template--{uuid.uuid4()}") + name = "Template" class ConfigLoaderFactory(ModelFactory[ConfigLoader]): """Factory for main configuration. Creates complete test configuration instances combining OpenAEV, - collector, and SentinelOne settings using subfactories. + collector, and Template settings using subfactories. """ __check_model__ = False openaev = Use(ConfigLoaderOAEVFactory.build) collector = Use(ConfigLoaderCollectorFactory.build) - sentinelone = Use(ConfigLoaderSentinelOneFactory.build) + template = Use(ConfigLoaderTemplateFactory.build) -class SentinelOneThreatFactory(ModelFactory[SentinelOneThreat]): - """Factory for SentinelOne threats. +class TemplateDataFactory(ModelFactory[TemplateData]): + """Factory for Template data. - Creates test instances of SentinelOne threat objects with - auto-generated threat IDs. + Creates test instances of Template data objects. """ __check_model__ = False @@ -133,20 +131,6 @@ class MockObjectsFactory: used throughout the test suite. """ - @staticmethod - def create_mock_client_api(): - """Create mock SentinelOne client API. - - Returns: - Mock SentinelOneClientAPI instance with basic attributes set. - - """ - mock_client = Mock() - mock_client.base_url = "https://test-api.example.com" - mock_client.session = Mock() - mock_client.session.headers = {} - return mock_client - @staticmethod def create_mock_detection_helper(match_result: bool = True): """Create mock detection helper. @@ -229,18 +213,7 @@ def create_oaev_detection_data() -> list[dict[str, Any]]: List of OAEV-formatted detection data dictionaries. """ - return [ - { - "parent_process_name": { - "type": "simple", - "data": [f"oaev-implant-test-{uuid.uuid4().hex[:8]}"], - }, - "threat_id": { - "type": "simple", - "data": [f"threat-{uuid.uuid4().hex[:8]}"], - }, - } - ] + return [] @staticmethod def create_oaev_prevention_data() -> list[dict[str, Any]]: @@ -250,25 +223,14 @@ def create_oaev_prevention_data() -> list[dict[str, Any]]: List of OAEV-formatted prevention data dictionaries. """ - return [ - { - "parent_process_name": { - "type": "simple", - "data": [f"oaev-implant-test-{uuid.uuid4().hex[:8]}"], - }, - "threat_id": { - "type": "simple", - "data": [f"threat-{uuid.uuid4().hex[:8]}"], - }, - } - ] + return [] @staticmethod - def create_mixed_sentinelone_data() -> list[Any]: - """Create mixed SentinelOne data (DV events + threats). + def create_mixed_template_data() -> list[Any]: + """Create mixed Template data. Returns: - List containing both DV event dicts and SentinelOneThreat instances. + List containing both DV event dicts and TemplateData instances. """ return [] @@ -288,37 +250,14 @@ def create_test_config(**overrides) -> ConfigLoader: return ConfigLoaderFactory.build(**overrides) -def create_test_dv_events(count: int = 1) -> list[dict]: - """Create test Deep Visibility events. - - Args: - count: Number of DV events to create (default 1). - - Returns: - List of dictionary representations of DV events for testing. - - """ - events = [] - for i in range(count): - events.append( - { - "src_proc_parent_name": f"oaev-implant-test-{uuid.uuid4().hex[:8]}", - "src_proc_name": f"process-{i}.exe", - "event_type": "process_creation", - "timestamp": "2024-01-01T10:00:00Z", - } - ) - return events - - -def create_test_threats(count: int = 1) -> list[SentinelOneThreat]: - """Create test SentinelOne threats. +def create_test_data(count: int = 1) -> list[TemplateData]: + """Create test TemplateData data. Args: - count: Number of threats to create (default 1). + count: Number of data to create (default 1). Returns: - List of SentinelOneThreat instances for testing. + List of TemplateData instances for testing. """ - return [SentinelOneThreatFactory.build() for _ in range(count)] + return [TemplateDataFactory.build() for _ in range(count)] diff --git a/template/tests/services/test_client_api.py b/template/tests/services/test_client_api.py deleted file mode 100644 index 9c2fcac0..00000000 --- a/template/tests/services/test_client_api.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Essential tests for SentinelOne Client API service - Gherkin GWT Format.""" - -from requests import Session -from src.services.client_api import SentinelOneClientAPI -from tests.gwt_shared import given_test_config # Given methods -from tests.gwt_shared import then_client_api_initialized_successfully - -# -------- -# Scenarios -# -------- - - -# Scenario: Initialize client API with valid configuration -def test_initialize_client_api_with_valid_config(): - """Scenario: Initialize client API with valid configuration.""" - # Given: A valid configuration is available - config = _given_valid_config_for_client_api() - - # When: I initialize the client API - client = _when_initialize_client_api_with_config(config) - - # Then: The client API should be initialized successfully - _then_client_api_initialized_with_valid_config(client, config) - - -# Scenario: Initialize with invalid configuration raises error -def test_initialize_with_invalid_config(): - """Scenario: Initialize with invalid configuration raises error.""" - # Given: An invalid configuration (None) - invalid_config = _given_invalid_config() - - # When: I attempt to initialize the client API - # Then: An AttributeError should be raised - _when_initialize_client_api_then_attribute_error_raised(invalid_config) - - -# -------- -# Given Methods -# -------- - - -# Given: A valid configuration is available -def _given_valid_config_for_client_api(): - """Create a valid configuration for client API testing. - - Returns: - Test configuration object. - - """ - return given_test_config() - - -# Given: An invalid configuration (None) -def _given_invalid_config(): - """Create an invalid configuration. - - Returns: - None (invalid configuration). - - """ - return None - - -# -------- -# When Methods -# -------- - - -# When: I initialize the client API with configuration -def _when_initialize_client_api_with_config(config): - """Initialize client API with given configuration. - - Args: - config: Configuration object to use. - - Returns: - Initialized SentinelOneClientAPI instance. - - """ - return SentinelOneClientAPI(config=config) - - -# When: I attempt to initialize with invalid config and expect AttributeError -def _when_initialize_client_api_then_attribute_error_raised(invalid_config): - """Attempt to initialize with invalid config and expect AttributeError. - - Args: - invalid_config: Invalid configuration to test. - - """ - import pytest - - with pytest.raises(AttributeError): - SentinelOneClientAPI(config=invalid_config) - - -# -------- -# Then Methods -# -------- - - -# Then: The client API should be initialized successfully with valid config -def _then_client_api_initialized_with_valid_config(client, config): - """Verify client API was initialized successfully with valid configuration. - - Args: - client: The client API instance to verify. - config: The configuration used for initialization. - - """ - then_client_api_initialized_successfully(client) - - assert client.config == config # noqa: S101 - assert client.base_url == str(config.sentinelone.base_url).rstrip("/") # noqa: S101 - assert client.api_key == config.sentinelone.api_key.get_secret_value() # noqa: S101 - assert isinstance(client.session, Session) # noqa: S101 - assert client.time_window == config.sentinelone.time_window # noqa: S101 diff --git a/template/tests/services/test_converter.py b/template/tests/services/test_converter.py index 98d2a35b..8ecdc264 100644 --- a/template/tests/services/test_converter.py +++ b/template/tests/services/test_converter.py @@ -1,9 +1,9 @@ -"""Essential tests for SentinelOne Converter services - Gherkin GWT Format.""" +"""Essential tests for Template Converter services - Gherkin GWT Format.""" import pytest -from src.services.converter import SentinelOneConverter -from src.services.exception import SentinelOneValidationError -from src.services.model_threat import SentinelOneThreat +from src.services.converter import TemplateConverter +from src.services.exception import TemplateValidationError +from src.services.model_data import TemplateData # -------- # Scenarios @@ -23,32 +23,32 @@ def test_initialize_converter(): _then_converter_initialized_successfully(converter) -# Scenario: Convert empty threats list -def test_convert_empty_threats(): - """Scenario: Convert empty threats list.""" +# Scenario: Convert empty data list +def test_convert_empty_data(): + """Scenario: Convert empty data list.""" # Given: A converter is available converter = _given_initialized_converter() - # When: I convert an empty threats list - result = _when_convert_threats_to_oaev(converter, []) + # When: I convert an empty data list + result = _when_convert_data_to_oaev(converter, []) # Then: An empty list should be returned _then_empty_list_returned(result) -# Scenario: Convert single threat with complete data -def test_convert_single_threat_complete_data(): - """Scenario: Convert single threat with complete data.""" +# Scenario: Convert single data with complete data +def test_convert_single_data_complete_data(): + """Scenario: Convert single data with complete data.""" # Given: A converter is available converter = _given_initialized_converter() - # Given: A threat with complete data - threat = _given_threat_with_complete_data() + # Given: A data with complete data + data = _given_data_with_complete_data() - # When: I convert the threat to OAEV format - result = _when_convert_threats_to_oaev(converter, [threat]) + # When: I convert the data to OAEV format + result = _when_convert_data_to_oaev(converter, [data]) - # Then: The threat should be converted with all fields - _then_single_threat_converted_completely(result, threat) + # Then: The data should be converted with all fields + _then_single_data_converted_completely(result, data) # Scenario: Convert invalid data type @@ -76,27 +76,25 @@ def _given_converter_dependencies_available(): # Given: A converter is available -def _given_initialized_converter() -> SentinelOneConverter: +def _given_initialized_converter() -> TemplateConverter: """Create and return an initialized converter. Returns: - Initialized SentinelOneConverter instance. + Initialized TemplateConverter instance. """ - return SentinelOneConverter() + return TemplateConverter() -# Given: A threat with complete data -def _given_threat_with_complete_data() -> SentinelOneThreat: - """Create a threat with complete data. +# Given: A data with complete data +def _given_data_with_complete_data() -> TemplateData: + """Create a data with complete data. Returns: - SentinelOneThreat with threat_id and hostname. + TemplateData with key. """ - return SentinelOneThreat( - threat_id="complete_threat_123", hostname="complete-host.example.com" - ) + return TemplateData(key="complete_data_123") # Given: Invalid input data (not a list) @@ -116,36 +114,34 @@ def _given_invalid_input_data() -> str: # When: I initialize the converter -def _when_initialize_converter() -> SentinelOneConverter: +def _when_initialize_converter() -> TemplateConverter: """Initialize the converter. Returns: - Initialized SentinelOneConverter instance. + Initialized TemplateConverter instance. """ - return SentinelOneConverter() + return TemplateConverter() -# When: I convert threats to OAEV format -def _when_convert_threats_to_oaev( - converter: SentinelOneConverter, threats: list -) -> list: - """Convert threats to OAEV format. +# When: I convert data to OAEV format +def _when_convert_data_to_oaev(converter: TemplateConverter, data: list) -> list: + """Convert data to OAEV format. Args: converter: The converter instance. - threats: List of threats to convert. + data: List of data to convert. Returns: List of converted OAEV format data. """ - return converter.convert_threats_to_oaev(threats) + return converter.convert_data_to_oaev(data) # When: I attempt to convert invalid data and expect validation error def _when_convert_invalid_data_then_validation_error_raised( - converter: SentinelOneConverter, invalid_data: str + converter: TemplateConverter, invalid_data: str ) -> None: """Attempt to convert invalid data and expect validation error. @@ -154,10 +150,10 @@ def _when_convert_invalid_data_then_validation_error_raised( invalid_data: Invalid input data. """ - with pytest.raises(SentinelOneValidationError) as exc_info: - converter.convert_threats_to_oaev(invalid_data) + with pytest.raises(TemplateValidationError) as exc_info: + converter.convert_data_to_oaev(invalid_data) - assert "threats must be a list" in str(exc_info.value) # noqa: S101 + assert "data must be a list" in str(exc_info.value) # noqa: S101 # -------- @@ -166,7 +162,7 @@ def _when_convert_invalid_data_then_validation_error_raised( # Then: The converter should be initialized successfully -def _then_converter_initialized_successfully(converter: SentinelOneConverter) -> None: +def _then_converter_initialized_successfully(converter: TemplateConverter) -> None: """Verify the converter was initialized successfully. Args: @@ -188,29 +184,13 @@ def _then_empty_list_returned(result: list) -> None: assert result == [] # noqa: S101 -# Then: The threat should be converted with all fields -def _then_single_threat_converted_completely( - result: list, threat: SentinelOneThreat -) -> None: - """Verify single threat was converted with all fields. +# Then: The data should be converted with all fields +def _then_single_data_converted_completely(result: list, data: TemplateData) -> None: + """Verify single data was converted with all fields. Args: result: The conversion result to verify. - threat: The original threat object. + data: The original data object. """ assert len(result) == 1 # noqa: S101 - - converted = result[0] - - assert "threat_id" in converted # noqa: S101 - assert converted["threat_id"]["type"] == "fuzzy" # noqa: S101 - assert converted["threat_id"]["data"] == [threat.threat_id] # noqa: S101 - assert converted["threat_id"]["score"] == 95 # noqa: S101 - - assert "target_hostname_address" in converted # noqa: S101 - assert converted["target_hostname_address"]["type"] == "fuzzy" # noqa: S101 - assert converted["target_hostname_address"]["data"] == [ # noqa: S101 - threat.hostname - ] - assert converted["target_hostname_address"]["score"] == 95 # noqa: S101 diff --git a/template/tests/services/test_expectation_service.py b/template/tests/services/test_expectation_service.py index b1b305be..a30b1657 100644 --- a/template/tests/services/test_expectation_service.py +++ b/template/tests/services/test_expectation_service.py @@ -1,4 +1,4 @@ -"""Essential tests for SentinelOne Expectation Service - Gherkin GWT Format.""" +"""Essential tests for Template Expectation Service - Gherkin GWT Format.""" from unittest.mock import Mock from uuid import uuid4 @@ -7,9 +7,9 @@ from pyoaev.signatures.types import SignatureTypes from src.services.expectation_service import ( ExpectationResult, - SentinelOneExpectationService, + TemplateExpectationService, ) -from src.services.model_threat import SentinelOneThreat +from src.services.model_data import TemplateData from tests.gwt_shared import ( given_initialized_expectation_service, given_test_config, @@ -52,8 +52,8 @@ def test_handle_single_detection_expectation(): service = _given_initialized_expectation_service() # Given: A detection helper detection_helper = _given_mock_detection_helper() - # Given: Mock threats are available - _given_mock_threats_for_service(service) + # Given: Mock data are available + _given_mock_data_for_service(service) # Given: A detection expectation expectation = _given_detection_expectation() @@ -71,8 +71,6 @@ def test_handle_prevention_expectation(): service = _given_initialized_expectation_service() # Given: A detection helper detection_helper = _given_mock_detection_helper() - # Given: Mock mitigated threats are available - _given_mock_mitigated_threats_for_service(service) # Given: A prevention expectation expectation = _given_prevention_expectation() @@ -83,106 +81,19 @@ def test_handle_prevention_expectation(): _then_prevention_result_returned(result, expectation) -# Scenario: Handle static threats with Deep Visibility enabled -def test_handle_static_threats_with_deep_visibility_enabled(): - """Scenario: Handle static threats with Deep Visibility enabled.""" - # Given: A detection helper - detection_helper = _given_mock_detection_helper() - # Given: A static expectation - expectation = _given_static_expectation() - - # When: I handle the static expectation with Deep Visibility enabled - with _given_expectation_service_with_deep_visibility_enabled() as service: - mock_static_threats = _given_mock_static_threats_for_service(service) - mock_dv_events = _given_mock_deep_visibility_events_for_service(service) - - with mock_static_threats, mock_dv_events: - result = _when_handle_batch_expectations( - service, [expectation], detection_helper - ) - - # Then: A static result with Deep Visibility events should be returned - _then_static_result_with_deep_visibility_returned(result, expectation) - - -# Scenario: Handle static threats with Deep Visibility disabled -def test_handle_static_threats_with_deep_visibility_disabled(): - """Scenario: Handle static threats with Deep Visibility disabled.""" - # Given: A detection helper - detection_helper = _given_mock_detection_helper() - # Given: A static expectation - expectation = _given_static_expectation() - - # When: I handle the static expectation with Deep Visibility disabled - with _given_expectation_service_with_deep_visibility_disabled() as service: - mock_static_threats = _given_mock_static_threats_for_service(service) - - with mock_static_threats: - result = _when_handle_batch_expectations( - service, [expectation], detection_helper - ) - - # Then: A static result without Deep Visibility events should be returned - _then_static_result_without_deep_visibility_returned(result, expectation) - - -# Scenario: Verify Deep Visibility fetcher is called when enabled -def test_deep_visibility_fetcher_called_when_enabled(): - """Scenario: Verify Deep Visibility fetcher is called when enabled.""" - # Given: A detection helper - detection_helper = _given_mock_detection_helper() - # Given: A static expectation - static_expectation = _given_static_expectation() - - # When: I handle the static expectation with Deep Visibility enabled - with _given_expectation_service_with_deep_visibility_enabled() as service: - mock_static_threats = _given_mock_static_threats_for_service(service) - mock_dv_events = _given_mock_deep_visibility_events_for_service(service) - - with mock_static_threats, mock_dv_events as dv_mock: - _when_handle_batch_expectations( - service, [static_expectation], detection_helper - ) - - # Then: Deep Visibility fetcher should have been called - dv_mock.assert_called_once() - - -# Scenario: Verify Deep Visibility fetcher is not called when disabled -def test_deep_visibility_fetcher_not_called_when_disabled(): - """Scenario: Verify Deep Visibility fetcher is not called when disabled.""" - # Given: A detection helper - detection_helper = _given_mock_detection_helper() - # Given: A static expectation - static_expectation = _given_static_expectation() - - # When: I handle the static expectation with Deep Visibility disabled - with _given_expectation_service_with_deep_visibility_disabled() as service: - mock_static_threats = _given_mock_static_threats_for_service(service) - mock_dv_events = _given_mock_deep_visibility_events_for_service(service) - - with mock_static_threats, mock_dv_events as dv_mock: - _when_handle_batch_expectations( - service, [static_expectation], detection_helper - ) - - # Then: Deep Visibility fetcher should not have been called - dv_mock.assert_not_called() - - -# Scenario: Match threats to expectations -def test_match_threats_to_expectations(): - """Scenario: Match threats to expectations.""" +# Scenario: Match data to expectations +def test_match_data_to_expectations(): + """Scenario: Match data to expectations.""" # Given: An initialized expectation service service = _given_initialized_expectation_service() - # Given: Threats and expectations - threats, expectations = _given_threats_and_expectations() + # Given: data and expectations + data, expectations = _given_data_and_expectations() - # When: I match threats to expectations - matches = _when_match_threats_to_expectations(service, threats, expectations) + # When: I match data to expectations + matches = _when_match_data_to_expectations(service, data, expectations) # Then: Proper matches should be found - _then_proper_matches_found(matches, threats, expectations) + _then_proper_matches_found(matches, data, expectations) # Then: The match should succeed without requiring mitigation _then_match_succeeds_without_mitigation_requirement(matches) @@ -219,7 +130,7 @@ def _given_initialized_expectation_service(): """Create an initialized expectation service. Returns: - Initialized SentinelOneExpectationService instance. + Initialized TemplateExpectationService instance. """ return given_initialized_expectation_service() @@ -236,44 +147,20 @@ def _given_mock_detection_helper(): return Mock() -# Given: Mock threats are available -def _given_mock_threats_for_service(service): - """Set up mock threats for the service. - - Args: - service: The expectation service instance. - - """ - mock_threats = [ - SentinelOneThreat( - threat_id="test_threat_1", - hostname="target-host.example.com", - is_mitigated=False, - ) - ] - service.threat_fetcher.fetch_threats_for_time_window = Mock( - return_value=mock_threats - ) - - -# Given: Mock mitigated threats are available -def _given_mock_mitigated_threats_for_service(service): - """Set up mock mitigated threats for the service. +# Given: Mock data are available +def _given_mock_data_for_service(service): + """Set up mock data for the service. Args: service: The expectation service instance. """ - mock_threats = [ - SentinelOneThreat( - threat_id="test_threat_1", - hostname="target-host.example.com", - is_mitigated=True, + mock_data = [ + TemplateData( + key="test_data_1", ) ] - service.threat_fetcher.fetch_threats_for_time_window = Mock( - return_value=mock_threats - ) + service.data_fetcher.fetch_data_for_time_window = Mock(return_value=mock_data) # Given: A detection expectation @@ -319,19 +206,17 @@ def _given_prevention_expectation(): return expectation -# Given: Threats and expectations -def _given_threats_and_expectations(): - """Create threats and expectations for matching tests. +# Given: data and expectations +def _given_data_and_expectations(): + """Create data and expectations for matching tests. Returns: - Tuple of (threats, expectations). + Tuple of (data, expectations). """ - threats = [ - SentinelOneThreat( - threat_id="match_threat_1", - hostname="match-host.example.com", - is_mitigated=False, + data = [ + TemplateData( + key="match_data_1", ) ] @@ -341,68 +226,7 @@ def _given_threats_and_expectations(): expectation = _create_mock_expectation(signatures=[hostname_sig]) expectations = [expectation] - return threats, expectations - - -# Given: An unmitigated threat -def _given_unmitigated_threat(): - """Create an unmitigated threat. - - Returns: - SentinelOneThreat instance that is not mitigated. - - """ - return SentinelOneThreat( - threat_id="unmitigated_threat", - hostname="unmitigated-host.example.com", - is_mitigated=False, - ) - - -# Given: An expectation service with Deep Visibility enabled -def _given_expectation_service_with_deep_visibility_enabled(): - """Create an expectation service with Deep Visibility enabled. - - Returns: - Context manager that yields SentinelOneExpectationService with Deep Visibility enabled. - - """ - import os - from contextlib import contextmanager - from unittest.mock import patch - - @contextmanager - def _service_context(): - with patch.dict( - os.environ, {"SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH": "true"} - ): - config = given_test_config() - yield SentinelOneExpectationService(config) - - return _service_context() - - -# Given: An expectation service with Deep Visibility disabled -def _given_expectation_service_with_deep_visibility_disabled(): - """Create an expectation service with Deep Visibility disabled. - - Returns: - Context manager that yields SentinelOneExpectationService with Deep Visibility disabled. - - """ - import os - from contextlib import contextmanager - from unittest.mock import patch - - @contextmanager - def _service_context(): - with patch.dict( - os.environ, {"SENTINELONE_ENABLE_DEEP_VISIBILITY_SEARCH": "false"} - ): - config = given_test_config() - yield SentinelOneExpectationService(config) - - return _service_context() + return data, expectations # Given: A static expectation @@ -426,140 +250,6 @@ def _given_static_expectation(): return expectation -# Given: Mock static threats for service -def _given_mock_static_threats_for_service(service): - """Set up mock static threats for the service. - - Args: - service: The expectation service to mock. - - """ - from unittest.mock import patch - - static_threats = [ - SentinelOneThreat( - threat_id="static_threat_1", - hostname="static-host.example.com", - is_mitigated=False, - is_static=True, - sha1="a1b2c3d4e5f6789012345678901234567890abcd", - ), - SentinelOneThreat( - threat_id="static_threat_2", - hostname="static-host.example.com", - is_mitigated=False, - is_static=True, - sha1="b2c3d4e5f6789012345678901234567890abcdef", - ), - ] - - return patch.object( - service.threat_fetcher, - "fetch_threats_for_time_window", - return_value=static_threats, - ) - - -# Given: Mock mixed threats for service -def _given_mock_mixed_threats_for_service(service): - """Set up mock mixed threats (static and non-static) for the service. - - Args: - service: The expectation service to mock. - - """ - from unittest.mock import patch - - mixed_threats = [ - SentinelOneThreat( - threat_id="static_threat_1", - hostname="mixed-host.example.com", - is_mitigated=False, - is_static=True, - sha1="a1b2c3d4e5f6789012345678901234567890abcd", - ), - SentinelOneThreat( - threat_id="behavior_threat_1", - hostname="mixed-host.example.com", - is_mitigated=False, - is_static=False, - sha1=None, - ), - ] - - return patch.object( - service.threat_fetcher, - "fetch_threats_for_time_window", - return_value=mixed_threats, - ) - - -# Given: Mock Deep Visibility events for service -def _given_mock_deep_visibility_events_for_service(service): - """Set up mock Deep Visibility events for the service. - - Args: - service: The expectation service to mock. - - """ - from unittest.mock import patch - - mock_dv_events = { - "a1b2c3d4e5f6789012345678901234567890abcd": [ - { - "fileSha1": "a1b2c3d4e5f6789012345678901234567890abcd", - "processName": "oaev-implant-test.exe", - "timestamp": "2024-01-01T12:00:00Z", - "eventType": "Process Creation", - "parentProcessName": "cmd.exe", - } - ], - "b2c3d4e5f6789012345678901234567890abcdef": [ - { - "fileSha1": "b2c3d4e5f6789012345678901234567890abcdef", - "processName": "oaev-implant-test2.exe", - "timestamp": "2024-01-01T12:01:00Z", - "eventType": "Process Creation", - "parentProcessName": "powershell.exe", - } - ], - } - - return patch.object( - service.deep_visibility_fetcher, - "fetch_events_for_batch_sha1", - return_value=mock_dv_events, - ) - - -# Given: Mock threat events for service -def _given_mock_threat_events_for_service(service): - """Set up mock threat events for the service. - - Args: - service: The expectation service to mock. - - """ - from unittest.mock import patch - - mock_threat_events = { - "behavior_threat_1": [ - { - "processName": "oaev-implant-behavior.exe", - "parentProcessName": "cmd.exe", - "timestamp": "2024-01-01T12:00:00Z", - "eventType": "Process Creation", - } - ] - } - - return patch.object( - service.threat_events_fetcher, - "fetch_events_for_threat", - return_value=mock_threat_events, - ) - - # -------- # When Methods # -------- @@ -573,10 +263,10 @@ def _when_initialize_expectation_service(config): config: Configuration object to use. Returns: - Initialized SentinelOneExpectationService instance. + Initialized TemplateExpectationService instance. """ - return SentinelOneExpectationService(config=config) + return TemplateExpectationService(config=config) # When: I attempt to initialize with invalid config and expect AttributeError @@ -588,7 +278,7 @@ def _when_initialize_expectation_service_then_attribute_error_raised(invalid_con """ with pytest.raises(AttributeError): - SentinelOneExpectationService(config=invalid_config) + TemplateExpectationService(config=invalid_config) # When: I handle batch expectations @@ -608,47 +298,41 @@ def _when_handle_batch_expectations(service, expectations, detection_helper): return results -# When: I match threats to expectations -def _when_match_threats_to_expectations(service, threats, expectations): - """Match threats to expectations. +# When: I match data to expectations +def _when_match_data_to_expectations(service, data, expectations): + """Match data to expectations. Args: service: The expectation service instance. - threats: List of threats. + data: List of data. expectations: List of expectations. Returns: List of matches. """ - threat_events = {threat.threat_id: [] for threat in threats} - return service._match_threats_to_expectations( - expectations, threats, threat_events, "detection" - ) + return service._match_data_to_expectations(expectations, data, "detection") -# When: I check if expectation matches threat data -def _when_check_expectation_matches_threat(service, expectation, threat): - """Check if expectation matches threat data. +# When: I check if expectation matches data +def _when_check_expectation_matches_data(service, expectation, data): + """Check if expectation matches data. Args: service: The expectation service instance. expectation: The expectation to check. - threat: The threat to match against. + data: The data to match against. Returns: Boolean indicating if there's a match. """ - events = [] expectation_type = ( "prevention" if hasattr(expectation, "is_prevention") and expectation.is_prevention else "detection" ) - return service._expectation_matches_threat_data( - expectation, threat, events, expectation_type - ) + return service._expectation_matches_data(expectation, data, expectation_type) # -------- @@ -666,7 +350,7 @@ def _then_expectation_service_initialized_with_valid_config(service, config): """ then_expectation_service_initialized_successfully(service) - assert service.batch_size == config.sentinelone.expectation_batch_size # noqa: S101 + assert service.batch_size == config.template.expectation_batch_size # noqa: S101 # Then: A detection result should be returned @@ -698,12 +382,12 @@ def _then_prevention_result_returned(result, expectation): # Then: Proper matches should be found -def _then_proper_matches_found(matches, threats, expectations): +def _then_proper_matches_found(matches, data, expectations): """Verify proper matches were found. Args: matches: The found matches. - threats: The original threats. + data: The original data. expectations: The original expectations. """ diff --git a/template/tests/services/test_fetcher_data.py b/template/tests/services/test_fetcher_data.py new file mode 100644 index 00000000..1e239369 --- /dev/null +++ b/template/tests/services/test_fetcher_data.py @@ -0,0 +1,168 @@ +"""Essential tests for Template Data Fetcher service - Gherkin GWT Format.""" + +import pytest +from src.services.exception import TemplateValidationError +from src.services.fetcher_data import FetcherData + +# -------- +# Scenarios +# -------- + + +# Scenario: Fetch data for time window successfully +def test_fetch_data_for_time_window_successfully(): + """Scenario: Fetch data for time window successfully.""" + # Given: A valid data fetcher + fetcher = _given_valid_data_fetcher() + # Given: A valid time window + time_window = _given_valid_time_window() + # When: I fetch data for the time window + data = _when_fetch_data_for_time_window(fetcher, time_window) + + # Then: Data should be returned successfully + _then_data_returned_successfully(data) + + +# Scenario: Handle invalid time window +def test_handle_invalid_time_window(): + """Scenario: Handle invalid time window.""" + # Given: A valid data fetcher + fetcher = _given_valid_data_fetcher() + # Given: An invalid time window + invalid_time_window = _given_invalid_time_window() + + # When: I attempt to fetch data with invalid time window + # Then: A validation error should be raised + _when_fetch_data_then_validation_error_raised(fetcher, invalid_time_window) + + +# -------- +# Given Methods +# -------- + + +# Given: A valid data fetcher +def _given_valid_data_fetcher(): + """Create a valid data fetcher for testing. + + Returns: + Initialized FetcherData instance. + + """ + return FetcherData() + + +# Given: A valid time window +def _given_valid_time_window(): + """Create a valid time window for testing. + + Returns: + Valid timedelta object. + + """ + from datetime import timedelta + + return timedelta(hours=24) + + +# Given: An invalid time window +def _given_invalid_time_window(): + """Create an invalid time window. + + Returns: + Invalid time window (None). + + """ + return None + + +# -------- +# When Methods +# -------- + + +# When: I initialize the data fetcher +def _when_initialize_data_fetcher(): + """Initialize data fetcher. + + Returns: + Initialized FetcherData instance. + + """ + return FetcherData() + + +# When: I fetch data for the time window +def _when_fetch_data_for_time_window(fetcher, time_window): + """Fetch data for given time window. + + Args: + fetcher: The data fetcher instance. + time_window: Time window to fetch data for. + + Returns: + List of fetched data. + + """ + from datetime import datetime, timezone + + end_time = datetime.now(timezone.utc) + start_time = end_time - time_window + return fetcher.fetch_data_for_time_window(start_time, end_time) + + +# When: I attempt to fetch data and expect validation error +def _when_fetch_data_then_validation_error_raised(fetcher, invalid_time_window): + """Attempt to fetch data and expect validation error. + + Args: + fetcher: The data fetcher instance. + invalid_time_window: Invalid time window to test. + + """ + from datetime import datetime, timezone + + if invalid_time_window is not None: + end_time = datetime.now(timezone.utc) + start_time = end_time - invalid_time_window + with pytest.raises(TemplateValidationError): + fetcher.fetch_data_for_time_window(start_time, end_time) + else: + with pytest.raises(TemplateValidationError): + fetcher.fetch_data_for_time_window(None, None) + + +# -------- +# Then Methods +# -------- + + +# Then: The data fetcher should be initialized successfully +def _then_data_fetcher_initialized_successfully(fetcher): + """Verify data fetcher was initialized successfully. + + Args: + fetcher: The data fetcher instance to verify. + + """ + assert fetcher is not None # noqa: S101 + assert fetcher.logger is not None # noqa: S101 + + +# Then: data should be returned successfully +def _then_data_returned_successfully(data): + """Verify data were returned successfully. + + Args: + data: The fetched data to verify. + + """ + assert isinstance(data, list) # noqa: S101 + assert len(data) > 0 # noqa: S101 + + # Basic verification that we got data back + from src.services.model_data import TemplateData + + assert all( # noqa: S101 + isinstance(single_data, TemplateData) for single_data in data + ) diff --git a/template/tests/services/test_fetcher_deep_visibility.py b/template/tests/services/test_fetcher_deep_visibility.py deleted file mode 100644 index 0af9fd45..00000000 --- a/template/tests/services/test_fetcher_deep_visibility.py +++ /dev/null @@ -1,431 +0,0 @@ -"""Essential tests for SentinelOne Deep Visibility Fetcher service - Gherkin GWT Format.""" - -from datetime import datetime, timedelta, timezone -from unittest.mock import Mock, patch - -import pytest -from src.services.exception import SentinelOneValidationError -from src.services.fetcher_deep_visibility import FetcherDeepVisibility -from tests.gwt_shared import given_initialized_client_api - -# -------- -# Scenarios -# -------- - - -# Scenario: Initialize deep visibility fetcher with valid client API -def test_initialize_deep_visibility_fetcher_with_valid_client_api(): - """Scenario: Initialize deep visibility fetcher with valid client API.""" - # Given: A valid client API is available - client_api = _given_valid_client_api() - - # When: I initialize the deep visibility fetcher - fetcher = _when_initialize_deep_visibility_fetcher(client_api) - - # Then: The deep visibility fetcher should be initialized successfully - _then_deep_visibility_fetcher_initialized_successfully(fetcher, client_api) - - -# Scenario: Fetch events for single SHA1 successfully -def test_fetch_events_for_single_sha1_successfully(): - """Scenario: Fetch events for single SHA1 successfully.""" - # Given: A valid deep visibility fetcher - fetcher = _given_valid_deep_visibility_fetcher() - # Given: A valid SHA1 hash - sha1 = _given_valid_sha1() - # Given: A valid time range - start_time, end_time = _given_valid_time_range() - - # When: I fetch events for the SHA1 (with mocked API) - with _mock_deep_visibility_success_response(fetcher, sha1): - events = _when_fetch_events_for_sha1(fetcher, sha1, start_time, end_time) - - # Then: Events should be returned successfully - _then_events_returned_successfully_for_single_sha1(events, sha1) - - -# Scenario: Fetch events for batch SHA1s successfully -def test_fetch_events_for_batch_sha1s_successfully(): - """Scenario: Fetch events for batch SHA1s successfully.""" - # Given: A valid deep visibility fetcher - fetcher = _given_valid_deep_visibility_fetcher() - # Given: A list of valid SHA1 hashes - sha1_list = _given_valid_sha1_list() - # Given: A valid time range - start_time, end_time = _given_valid_time_range() - - # When: I fetch events for the SHA1 batch (with mocked API) - with _mock_deep_visibility_batch_success_response(fetcher, sha1_list): - events_dict = _when_fetch_events_for_batch_sha1( - fetcher, sha1_list, start_time, end_time - ) - - # Then: Events should be returned successfully for all SHA1s - _then_events_returned_successfully_for_batch_sha1s(events_dict, sha1_list) - - -# Scenario: Handle invalid SHA1 input -def test_handle_invalid_sha1_input(): - """Scenario: Handle invalid SHA1 input.""" - # Given: A valid deep visibility fetcher - fetcher = _given_valid_deep_visibility_fetcher() - # Given: Invalid SHA1 inputs - invalid_sha1_cases = [None, "", 123, [], {}] - - for invalid_sha1 in invalid_sha1_cases: - # When: I attempt to fetch events with invalid SHA1 - # Then: A validation error should be raised - _when_fetch_events_for_sha1_then_validation_error_raised(fetcher, invalid_sha1) - - -# Scenario: Handle invalid SHA1 list input -def test_handle_invalid_sha1_list_input(): - """Scenario: Handle invalid SHA1 list input.""" - # Given: A valid deep visibility fetcher - fetcher = _given_valid_deep_visibility_fetcher() - - # When: I attempt to fetch events with None SHA1 list - # Then: A validation error should be raised - with pytest.raises(SentinelOneValidationError): - fetcher.fetch_events_for_batch_sha1(None) - - # When: I attempt to fetch events with empty SHA1 list - # Then: A validation error should be raised - with pytest.raises(SentinelOneValidationError): - fetcher.fetch_events_for_batch_sha1([]) - - -# Scenario: Handle API connection error during single SHA1 fetch -def test_handle_api_connection_error_single_sha1(): - """Scenario: Handle API connection error during single SHA1 fetch.""" - # Given: A valid deep visibility fetcher - fetcher = _given_valid_deep_visibility_fetcher() - # Given: A valid SHA1 hash - sha1 = _given_valid_sha1() - - # When: I attempt to fetch events with connection error - with _mock_connection_error(fetcher): - _when_fetch_events_for_sha1_then_api_error_raised(fetcher, sha1) - - -# Scenario: Filter events correctly by SHA1 in single fetch -def test_filter_events_correctly_by_sha1_single_fetch(): - """Scenario: Filter events correctly by SHA1 in single fetch.""" - # Given: A valid deep visibility fetcher - fetcher = _given_valid_deep_visibility_fetcher() - # Given: A target SHA1 hash - target_sha1 = _given_valid_sha1() - - # When: I fetch events and API returns mixed SHA1 events - with _mock_mixed_sha1_events_response(fetcher, target_sha1): - events = _when_fetch_events_for_sha1(fetcher, target_sha1) - - # Then: Only events matching the target SHA1 should be returned - _then_only_target_sha1_events_returned(events, target_sha1) - - -# -------- -# Given Methods -# -------- - - -# Given: A valid client API is available -def _given_valid_client_api(): - """Create a valid client API for testing. - - Returns: - Valid client API instance. - - """ - return given_initialized_client_api() - - -# Given: A valid deep visibility fetcher -def _given_valid_deep_visibility_fetcher(): - """Create a valid deep visibility fetcher for testing. - - Returns: - Initialized FetcherDeepVisibility instance. - - """ - client_api = given_initialized_client_api() - return FetcherDeepVisibility(client_api) - - -# Given: A valid SHA1 hash -def _given_valid_sha1(): - """Create a valid SHA1 hash for testing. - - Returns: - Valid SHA1 string. - - """ - return "a1b2c3d4e5f6789012345678901234567890abcd" - - -# Given: A list of valid SHA1 hashes -def _given_valid_sha1_list(): - """Create a list of valid SHA1 hashes for testing. - - Returns: - List of valid SHA1 strings. - - """ - return [ - "a1b2c3d4e5f6789012345678901234567890abcd", - "b2c3d4e5f6789012345678901234567890abcdef", - "c3d4e5f6789012345678901234567890abcdef01", - ] - - -# Given: A valid time range -def _given_valid_time_range(): - """Create a valid time range for testing. - - Returns: - Tuple of (start_time, end_time) datetime objects. - - """ - end_time = datetime.now(timezone.utc) - start_time = end_time - timedelta(hours=1) - return start_time, end_time - - -# -------- -# Mock Context Managers -# -------- - - -def _mock_deep_visibility_success_response(fetcher, sha1): - """Mock Deep Visibility API to return successful response for single SHA1.""" - mock_query_response = Mock() - mock_query_response.data = Mock() - mock_query_response.data.query_id = "test_query_id" - - mock_events = [ - { - "fileSha1": sha1, - "processName": "test_process.exe", - "timestamp": "2023-01-01T12:00:00Z", - "eventType": "Process Creation", - }, - { - "fileSha1": "different_sha1", - "processName": "other_process.exe", - "timestamp": "2023-01-01T12:01:00Z", - "eventType": "Process Creation", - }, - ] - - return patch.multiple( - fetcher, - _init_dv_query=Mock(return_value=mock_query_response), - _execute_query=Mock(return_value=mock_events), - ) - - -def _mock_deep_visibility_batch_success_response(fetcher, sha1_list): - """Mock Deep Visibility API to return successful response for batch SHA1s.""" - mock_query_response = Mock() - mock_query_response.data = Mock() - mock_query_response.data.query_id = "test_batch_query_id" - - mock_events = [] - for i, sha1 in enumerate(sha1_list): - mock_events.append( - { - "fileSha1": sha1, - "processName": f"test_process_{i}.exe", - "timestamp": f"2023-01-01T12:0{i}:00Z", - "eventType": "Process Creation", - } - ) - - return patch.multiple( - fetcher, - _init_dv_query=Mock(return_value=mock_query_response), - _execute_query=Mock(return_value=mock_events), - ) - - -def _mock_connection_error(fetcher): - """Mock connection error during API call.""" - from requests.exceptions import ConnectionError - - return patch.object( - fetcher, - "_init_dv_query", - side_effect=ConnectionError("Connection failed"), - ) - - -def _mock_mixed_sha1_events_response(fetcher, target_sha1): - """Mock API to return events with mixed SHA1s.""" - mock_query_response = Mock() - mock_query_response.data = Mock() - mock_query_response.data.query_id = "mixed_query_id" - - mock_events = [ - {"fileSha1": target_sha1, "processName": "target_process.exe"}, - {"fileSha1": "other_sha1_1", "processName": "other_process1.exe"}, - {"fileSha1": target_sha1, "processName": "target_process2.exe"}, - {"fileSha1": "other_sha1_2", "processName": "other_process2.exe"}, - ] - - return patch.multiple( - fetcher, - _init_dv_query=Mock(return_value=mock_query_response), - _execute_query=Mock(return_value=mock_events), - ) - - -# -------- -# When Methods -# -------- - - -# When: I initialize the deep visibility fetcher -def _when_initialize_deep_visibility_fetcher(client_api): - """Initialize deep visibility fetcher with given client API. - - Args: - client_api: Client API instance to use. - - Returns: - Initialized FetcherDeepVisibility instance. - - """ - return FetcherDeepVisibility(client_api) - - -# When: I fetch events for SHA1 -def _when_fetch_events_for_sha1(fetcher, sha1, start_time=None, end_time=None): - """Fetch events for given SHA1. - - Args: - fetcher: The deep visibility fetcher instance. - sha1: SHA1 hash to fetch events for. - start_time: Start time for search (optional). - end_time: End time for search (optional). - - Returns: - List of fetched events. - - """ - return fetcher.fetch_events_for_sha1(sha1, start_time, end_time) - - -# When: I fetch events for batch SHA1s -def _when_fetch_events_for_batch_sha1( - fetcher, sha1_list, start_time=None, end_time=None -): - """Fetch events for given SHA1 list. - - Args: - fetcher: The deep visibility fetcher instance. - sha1_list: List of SHA1 hashes to fetch events for. - start_time: Start time for search (optional). - end_time: End time for search (optional). - - Returns: - Dictionary mapping SHA1 to events. - - """ - return fetcher.fetch_events_for_batch_sha1(sha1_list, start_time, end_time) - - -# When: I attempt to fetch events for SHA1 and expect validation error -def _when_fetch_events_for_sha1_then_validation_error_raised(fetcher, invalid_sha1): - """Attempt to fetch events for invalid SHA1 and expect validation error. - - Args: - fetcher: The deep visibility fetcher instance. - invalid_sha1: Invalid SHA1 to test. - - """ - with pytest.raises(SentinelOneValidationError): - fetcher.fetch_events_for_sha1(invalid_sha1) - - -# When: I attempt to fetch events for SHA1 and expect API error -def _when_fetch_events_for_sha1_then_api_error_raised(fetcher, sha1): - """Attempt to fetch events and expect API error. - - Args: - fetcher: The deep visibility fetcher instance. - sha1: SHA1 hash to fetch events for. - - """ - from src.services.exception import SentinelOneAPIError - - with pytest.raises(SentinelOneAPIError): - fetcher.fetch_events_for_sha1(sha1) - - -# -------- -# Then Methods -# -------- - - -# Then: The deep visibility fetcher should be initialized successfully -def _then_deep_visibility_fetcher_initialized_successfully(fetcher, client_api): - """Verify deep visibility fetcher was initialized successfully. - - Args: - fetcher: The fetcher instance to verify. - client_api: The client API used for initialization. - - """ - assert fetcher is not None # noqa: S101 - assert fetcher.client_api == client_api # noqa: S101 - assert fetcher.logger is not None # noqa: S101 - - -# Then: Events should be returned successfully for single SHA1 -def _then_events_returned_successfully_for_single_sha1(events, sha1): - """Verify events were returned successfully for single SHA1. - - Args: - events: The fetched events to verify. - sha1: The SHA1 that was searched for. - - """ - assert isinstance(events, list) # noqa: S101 - assert len(events) > 0 # noqa: S101 - - for event in events: - assert event.get("fileSha1") == sha1 # noqa: S101 - assert isinstance(event, dict) # noqa: S101 - - -# Then: Events should be returned successfully for batch SHA1s -def _then_events_returned_successfully_for_batch_sha1s(events_dict, sha1_list): - """Verify events were returned successfully for batch SHA1s. - - Args: - events_dict: Dictionary of SHA1 to events. - sha1_list: The SHA1 list that was searched for. - - """ - assert isinstance(events_dict, dict) # noqa: S101 - assert len(events_dict) == len(sha1_list) # noqa: S101 - - for sha1 in sha1_list: - assert sha1 in events_dict # noqa: S101 - assert isinstance(events_dict[sha1], list) # noqa: S101 - - -# Then: Only events matching the target SHA1 should be returned -def _then_only_target_sha1_events_returned(events, target_sha1): - """Verify only events matching target SHA1 are returned. - - Args: - events: The fetched events to verify. - target_sha1: The target SHA1 that should match. - - """ - assert isinstance(events, list) # noqa: S101 - assert len(events) > 0 # noqa: S101 - - for event in events: - assert event.get("fileSha1") == target_sha1 # noqa: S101 diff --git a/template/tests/services/test_fetcher_threat.py b/template/tests/services/test_fetcher_threat.py deleted file mode 100644 index 63512f4f..00000000 --- a/template/tests/services/test_fetcher_threat.py +++ /dev/null @@ -1,336 +0,0 @@ -"""Essential tests for SentinelOne Threat Fetcher service - Gherkin GWT Format.""" - -import pytest -from src.services.exception import SentinelOneNetworkError, SentinelOneValidationError -from src.services.fetcher_threat import FetcherThreat -from tests.gwt_shared import given_initialized_client_api - -# -------- -# Scenarios -# -------- - - -# Scenario: Initialize threat fetcher with valid client API -def test_initialize_threat_fetcher_with_valid_client_api(): - """Scenario: Initialize threat fetcher with valid client API.""" - # Given: A valid client API is available - client_api = _given_valid_client_api() - - # When: I initialize the threat fetcher - fetcher = _when_initialize_threat_fetcher(client_api) - - # Then: The threat fetcher should be initialized successfully - _then_threat_fetcher_initialized_successfully(fetcher, client_api) - - -# Scenario: Initialize with invalid client API raises error -def test_initialize_with_invalid_client_api(): - """Scenario: Initialize with invalid client API raises error.""" - # Given: An invalid client API (None) - invalid_client_api = _given_invalid_client_api() - - # When: I attempt to initialize the threat fetcher - # Then: A validation error should be raised - _when_initialize_threat_fetcher_then_validation_error_raised(invalid_client_api) - - -# Scenario: Fetch threats for time window successfully -def test_fetch_threats_for_time_window_successfully(): - """Scenario: Fetch threats for time window successfully.""" - # Given: A valid threat fetcher - fetcher = _given_valid_threat_fetcher() - # Given: A valid time window - time_window = _given_valid_time_window() - # When: I fetch threats for the time window (with mocked API) - from unittest.mock import Mock, patch - - mock_response = Mock() - mock_response.json.return_value = { - "data": [ - { - "threatInfo": { - "threatId": "test_threat_1", - "computerName": "test-host.example.com", - "mitigationStatus": "not_mitigated", - } - } - ] - } - mock_response.raise_for_status.return_value = None - - with patch.object(fetcher.client_api.session, "get", return_value=mock_response): - threats = _when_fetch_threats_for_time_window(fetcher, time_window) - - # Then: Threats should be returned successfully - _then_threats_returned_successfully(threats) - - -# Scenario: Handle API connection error -def test_handle_api_connection_error(): - """Scenario: Handle API connection error.""" - # Given: A valid threat fetcher - fetcher = _given_valid_threat_fetcher() - # Given: A valid time window - time_window = _given_valid_time_window() - - # When: I attempt to fetch threats with connection error - from unittest.mock import patch - - from requests.exceptions import ConnectionError - - with patch.object( - fetcher.client_api.session, - "get", - side_effect=ConnectionError("Connection failed"), - ): - _when_fetch_threats_then_network_error_raised(fetcher, time_window) - - -# Scenario: Handle invalid time window -def test_handle_invalid_time_window(): - """Scenario: Handle invalid time window.""" - # Given: A valid threat fetcher - fetcher = _given_valid_threat_fetcher() - # Given: An invalid time window - invalid_time_window = _given_invalid_time_window() - - # When: I attempt to fetch threats with invalid time window - # Then: A validation error should be raised - _when_fetch_threats_then_validation_error_raised(fetcher, invalid_time_window) - - -# -------- -# Given Methods -# -------- - - -# Given: A valid client API is available -def _given_valid_client_api(): - """Create a valid client API for testing. - - Returns: - Valid client API instance. - - """ - return given_initialized_client_api() - - -# Given: An invalid client API (None) -def _given_invalid_client_api(): - """Create an invalid client API. - - Returns: - None (invalid client API). - - """ - return None - - -# Given: A valid threat fetcher -def _given_valid_threat_fetcher(): - """Create a valid threat fetcher for testing. - - Returns: - Initialized SentinelOneThreatFetcher instance. - - """ - client_api = given_initialized_client_api() - return FetcherThreat(client_api) - - -# Given: A valid time window -def _given_valid_time_window(): - """Create a valid time window for testing. - - Returns: - Valid timedelta object. - - """ - from datetime import timedelta - - return timedelta(hours=24) - - -# Given: An invalid time window -def _given_invalid_time_window(): - """Create an invalid time window. - - Returns: - Invalid time window (None). - - """ - return None - - -# Given: Mock API returns threat data -def _given_mock_api_returns_threat_data(fetcher): - """Set up mock API to return threat data. - - Args: - fetcher: The threat fetcher instance to mock. - - """ - from unittest.mock import Mock, patch - - mock_response = Mock() - mock_response.json.return_value = { - "data": [ - { - "threatInfo": { - "threatId": "test_threat_1", - "computerName": "test-host.example.com", - "mitigationStatus": "not_mitigated", - } - } - ] - } - mock_response.raise_for_status.return_value = None - - with patch.object(fetcher.client_api.session, "get", return_value=mock_response): - pass - - -# Given: API connection will fail -def _given_api_connection_will_fail(fetcher): - """Set up API connection to fail. - - Args: - fetcher: The threat fetcher instance to mock. - - """ - from unittest.mock import patch - - from requests.exceptions import ConnectionError - - with patch.object( - fetcher.client_api.session, - "get", - side_effect=ConnectionError("Connection failed"), - ): - pass - - -# -------- -# When Methods -# -------- - - -# When: I initialize the threat fetcher -def _when_initialize_threat_fetcher(client_api): - """Initialize threat fetcher with given client API. - - Args: - client_api: Client API instance to use. - - Returns: - Initialized SentinelOneThreatFetcher instance. - - """ - return FetcherThreat(client_api) - - -# When: I attempt to initialize with invalid client API and expect validation error -def _when_initialize_threat_fetcher_then_validation_error_raised(invalid_client_api): - """Attempt to initialize with invalid client API and expect validation error. - - Args: - invalid_client_api: Invalid client API to test. - - """ - with pytest.raises(SentinelOneValidationError): - FetcherThreat(invalid_client_api) - - -# When: I fetch threats for the time window -def _when_fetch_threats_for_time_window(fetcher, time_window): - """Fetch threats for given time window. - - Args: - fetcher: The threat fetcher instance. - time_window: Time window to fetch threats for. - - Returns: - List of fetched threats. - - """ - from datetime import datetime, timezone - - end_time = datetime.now(timezone.utc) - start_time = end_time - time_window - return fetcher.fetch_threats_for_time_window(start_time, end_time) - - -# When: I attempt to fetch threats and expect network error -def _when_fetch_threats_then_network_error_raised(fetcher, time_window): - """Attempt to fetch threats and expect network error. - - Args: - fetcher: The threat fetcher instance. - time_window: Time window to fetch threats for. - - """ - from datetime import datetime, timezone - - end_time = datetime.now(timezone.utc) - start_time = end_time - time_window - with pytest.raises(SentinelOneNetworkError): - fetcher.fetch_threats_for_time_window(start_time, end_time) - - -# When: I attempt to fetch threats and expect validation error -def _when_fetch_threats_then_validation_error_raised(fetcher, invalid_time_window): - """Attempt to fetch threats and expect validation error. - - Args: - fetcher: The threat fetcher instance. - invalid_time_window: Invalid time window to test. - - """ - from datetime import datetime, timezone - - if invalid_time_window is not None: - end_time = datetime.now(timezone.utc) - start_time = end_time - invalid_time_window - with pytest.raises(SentinelOneValidationError): - fetcher.fetch_threats_for_time_window(start_time, end_time) - else: - with pytest.raises(SentinelOneValidationError): - fetcher.fetch_threats_for_time_window(None, None) - - -# -------- -# Then Methods -# -------- - - -# Then: The threat fetcher should be initialized successfully -def _then_threat_fetcher_initialized_successfully(fetcher, client_api): - """Verify threat fetcher was initialized successfully. - - Args: - fetcher: The threat fetcher instance to verify. - client_api: The client API used for initialization. - - """ - assert fetcher is not None # noqa: S101 - assert fetcher.client_api == client_api # noqa: S101 - assert fetcher.logger is not None # noqa: S101 - - -# Then: Threats should be returned successfully -def _then_threats_returned_successfully(threats): - """Verify threats were returned successfully. - - Args: - threats: The fetched threats to verify. - - """ - assert isinstance(threats, list) # noqa: S101 - assert len(threats) > 0 # noqa: S101 - - # Basic verification that we got threats back - from src.services.model_threat import SentinelOneThreat - - assert all( # noqa: S101 - isinstance(threat, SentinelOneThreat) for threat in threats - ) diff --git a/template/tests/services/test_fetcher_threat_events.py b/template/tests/services/test_fetcher_threat_events.py deleted file mode 100644 index 37f8ea4a..00000000 --- a/template/tests/services/test_fetcher_threat_events.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Essential tests for SentinelOne Threat Events Fetcher service - Gherkin GWT Format.""" - -import pytest -from src.services.exception import SentinelOneValidationError -from src.services.fetcher_threat_events import FetcherThreatEvents -from tests.gwt_shared import ( - given_initialized_client_api, - given_threat_with_complete_data, -) - -# -------- -# Scenarios -# -------- - - -# Scenario: Initialize threat events fetcher with valid client API -def test_initialize_threat_events_fetcher_with_valid_client_api(): - """Scenario: Initialize threat events fetcher with valid client API.""" - # Given: A valid client API is available - client_api = _given_valid_client_api() - - # When: I initialize the threat events fetcher - fetcher = _when_initialize_threat_events_fetcher(client_api) - - # Then: The threat events fetcher should be initialized successfully - _then_threat_events_fetcher_initialized_successfully(fetcher, client_api) - - -# Scenario: Initialize with invalid client API raises error -def test_initialize_with_invalid_client_api(): - """Scenario: Initialize with invalid client API raises error.""" - # Given: An invalid client API (None) - invalid_client_api = _given_invalid_client_api() - - # When: I attempt to initialize the threat events fetcher - # Then: A validation error should be raised - _when_initialize_threat_events_fetcher_then_validation_error_raised( - invalid_client_api - ) - - -# Scenario: Fetch events for threat successfully -def test_fetch_events_for_threat_successfully(): - """Scenario: Fetch events for threat successfully.""" - # Given: A valid threat events fetcher - fetcher = _given_valid_threat_events_fetcher() - # Given: A threat to fetch events for - threat = _given_threat_for_event_fetching() - - # When: I fetch events for the threat (with mocked API) - from unittest.mock import Mock, patch - - mock_response = Mock() - mock_response.json.return_value = { - "data": [ - { - "id": "event_1", - "processName": "test.exe", - "createdAt": "2024-01-01T12:00:00Z", - } - ] - } - mock_response.raise_for_status.return_value = None - - with patch.object(fetcher.client_api.session, "get", return_value=mock_response): - events = _when_fetch_events_for_threat(fetcher, threat) - - # Then: Events should be returned successfully - _then_events_returned_successfully(events) - - -# -------- -# Given Methods -# -------- - - -# Given: A valid client API is available -def _given_valid_client_api(): - """Create a valid client API for testing. - - Returns: - Valid client API instance. - - """ - return given_initialized_client_api() - - -# Given: An invalid client API (None) -def _given_invalid_client_api(): - """Create an invalid client API. - - Returns: - None (invalid client API). - - """ - return None - - -# Given: A valid threat events fetcher -def _given_valid_threat_events_fetcher(): - """Create a valid threat events fetcher for testing. - - Returns: - Initialized FetcherThreatEvents instance. - - """ - client_api = given_initialized_client_api() - return FetcherThreatEvents(client_api) - - -# Given: A threat to fetch events for -def _given_threat_for_event_fetching(): - """Create a threat for event fetching testing. - - Returns: - Threat object for testing. - - """ - return given_threat_with_complete_data() - - -# Given: Mock API returns event data -def _given_mock_api_returns_event_data(fetcher): - """Set up mock API to return event data. - - Args: - fetcher: The threat events fetcher instance to mock. - - """ - pass - - -# -------- -# When Methods -# -------- - - -# When: I initialize the threat events fetcher -def _when_initialize_threat_events_fetcher(client_api): - """Initialize threat events fetcher with given client API. - - Args: - client_api: Client API instance to use. - - Returns: - Initialized FetcherThreatEvents instance. - - """ - return FetcherThreatEvents(client_api) - - -# When: I attempt to initialize with invalid client API and expect validation error -def _when_initialize_threat_events_fetcher_then_validation_error_raised( - invalid_client_api, -): - """Attempt to initialize with invalid client API and expect validation error. - - Args: - invalid_client_api: Invalid client API to test. - - """ - with pytest.raises(SentinelOneValidationError): - FetcherThreatEvents(invalid_client_api) - - -# When: I fetch events for the threat -def _when_fetch_events_for_threat(fetcher, threat): - """Fetch events for given threat. - - Args: - fetcher: The threat events fetcher instance. - threat: Threat to fetch events for. - - Returns: - List of fetched events. - - """ - return fetcher.fetch_events_for_threat(threat) - - -# -------- -# Then Methods -# -------- - - -# Then: The threat events fetcher should be initialized successfully -def _then_threat_events_fetcher_initialized_successfully(fetcher, client_api): - """Verify threat events fetcher was initialized successfully. - - Args: - fetcher: The fetcher instance to verify. - client_api: The client API used for initialization. - - """ - assert fetcher is not None # noqa: S101 - assert fetcher.client_api == client_api # noqa: S101 - assert fetcher.logger is not None # noqa: S101 - - -# Then: Events should be returned successfully -def _then_events_returned_successfully(events): - """Verify events were returned successfully. - - Args: - events: The fetched events to verify. - - """ - assert isinstance(events, list) # noqa: S101 - assert len(events) > 0 # noqa: S101 - - # Basic verification that we got events back - assert all(isinstance(event, dict) for event in events) # noqa: S101 diff --git a/template/tests/services/test_trace_service.py b/template/tests/services/test_trace_service.py index c5296918..6ea00452 100644 --- a/template/tests/services/test_trace_service.py +++ b/template/tests/services/test_trace_service.py @@ -1,8 +1,8 @@ -"""Essential tests for SentinelOne Trace Service - Gherkin GWT Format.""" +"""Essential tests for Template Trace Service - Gherkin GWT Format.""" import pytest -from src.services.exception import SentinelOneValidationError -from src.services.trace_service import SentinelOneTraceService +from src.services.exception import TemplateValidationError +from src.services.trace_service import TemplateTraceService from tests.gwt_shared import given_test_config # -------- @@ -96,11 +96,11 @@ def _given_valid_trace_service(): """Create a valid trace service for testing. Returns: - Initialized SentinelOneTraceService instance. + Initialized TemplateTraceService instance. """ config = given_test_config() - return SentinelOneTraceService(config=config) + return TemplateTraceService(config=config) # Given: Valid expectation results with matching alerts @@ -121,8 +121,8 @@ def _given_valid_expectation_results_with_alerts(): "alert_id": "alert_1", "severity": "high", "message": "Test alert 1", - "alert_name": "SentinelOne Test Alert 1", - "alert_link": "https://console.sentinelone.com/alerts/alert_1", + "alert_name": "Template Test Alert 1", + "alert_link": "https://foo.bar", } ] result1.expectation = Mock() @@ -144,10 +144,10 @@ def _when_initialize_trace_service(config): config: Configuration object to use. Returns: - Initialized SentinelOneTraceService instance. + Initialized TemplateTraceService instance. """ - return SentinelOneTraceService(config=config) + return TemplateTraceService(config=config) # When: I attempt to initialize with invalid config and expect validation error @@ -158,8 +158,8 @@ def _when_initialize_trace_service_then_validation_error_raised(invalid_config): invalid_config: Invalid configuration to test. """ - with pytest.raises(SentinelOneValidationError): - SentinelOneTraceService(config=invalid_config) + with pytest.raises(TemplateValidationError): + TemplateTraceService(config=invalid_config) # When: I create traces from the results diff --git a/template/tests/test_create_collector.py b/template/tests/test_create_collector.py index e40b9bfa..6f5b067f 100644 --- a/template/tests/test_create_collector.py +++ b/template/tests/test_create_collector.py @@ -1,4 +1,4 @@ -"""Test module for the SentinelOne Collector initialization - Gherkin GWT Format.""" +"""Test module for the Template Collector initialization - Gherkin GWT Format.""" from os import environ as os_environ from typing import Any @@ -26,10 +26,9 @@ def collector_config() -> dict[str, str]: # type: ignore "OPENAEV_URL": "http://fake-url/", "OPENAEV_TOKEN": "fake-oaev-token", "COLLECTOR_ID": "fake-collector-id", - "COLLECTOR_NAME": "SentinelOne", - "SENTINELONE_BASE_URL": "https://fake-sentinelone.net/", - "SENTINELONE_API_KEY": "fake-api-key", - "COLLECTOR_ICON_FILEPATH": "src/img/sentinelone-logo.png", + "COLLECTOR_NAME": "Template", + "TEMPLATE_KEY": "fake-key", + "COLLECTOR_ICON_FILEPATH": "src/img/template-logo.png", "COLLECTOR_LOG_LEVEL": "debug", } @@ -58,23 +57,6 @@ def test_create_collector_with_valid_config(capfd, collector_config): # type: i _then_collector_created_successfully(capfd, mock_env, collector, collector_config) -# Scenario: Create a collector with missing required config -def test_create_collector_with_missing_api_key(collector_config) -> None: - """Scenario: Create a collector with missing required config. - - Args: - collector_config: Fixture providing base collector configuration. - - """ - # Given: Configuration with missing required SentinelOne API key - incomplete_config = _given_config_missing_api_key(collector_config) - mock_env = _given_valid_collector_config(incomplete_config) - - # When: I attempt to create the collector - # Then: The collector creation should fail with configuration error - _when_create_collector_then_raises_config_error(mock_env) - - # -------- # Given Methods # -------- @@ -95,26 +77,6 @@ def _given_valid_collector_config(config_data: dict[str, str]) -> Any: # type: return mock_env -# Given: Configuration with missing required SentinelOne API key -def _given_config_missing_api_key(base_config: dict[str, str]) -> dict[str, str]: - """Create configuration with missing SentinelOne API key. - - Args: - base_config: Base configuration dictionary. - - Returns: - Configuration dictionary without SentinelOne API key. - - """ - config = base_config.copy() - config.pop("SENTINELONE_API_KEY", None) - - if "SENTINELONE_API_KEY" in os_environ: - del os_environ["SENTINELONE_API_KEY"] - - return config - - # -------- # When Methods # -------- @@ -195,15 +157,8 @@ def _then_collector_created_successfully( assert daemon_config.get("collector_name") == expected_config.get( "COLLECTOR_NAME" ) # noqa: S101 - assert daemon_config.get( - "sentinelone_base_url" - ) == expected_config.get( # noqa: S101 - "SENTINELONE_BASE_URL" - ) - assert daemon_config.get( - "sentinelone_api_key" - ) == expected_config.get( # noqa: S101 - "SENTINELONE_API_KEY" + assert daemon_config.get("template_key") == expected_config.get( # noqa: S101 + "TEMPLATE_KEY" ) assert daemon_config.get( "collector_log_level" @@ -229,5 +184,5 @@ def _then_collector_logged_initialization_success( """ log_records = capfd.readouterr() if daemon_config.get("collector_log_level") in ["info", "debug"]: - registered_message = "SentinelOne Collector initialized successfully" + registered_message = "Template Collector initialized successfully" assert registered_message in log_records.err # noqa: S101 From fc02033125007e7d126775b7cd91433699880ac3 Mon Sep 17 00:00:00 2001 From: guzmud Date: Tue, 21 Apr 2026 09:32:30 +0200 Subject: [PATCH 03/25] [template] refactor(collector): expectation models moved to base models --- template/src/collector/{models.py => base_models/expectations.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename template/src/collector/{models.py => base_models/expectations.py} (100%) diff --git a/template/src/collector/models.py b/template/src/collector/base_models/expectations.py similarity index 100% rename from template/src/collector/models.py rename to template/src/collector/base_models/expectations.py From d7e173954e20b78d403854d2e64b56ed0b38c431 Mon Sep 17 00:00:00 2001 From: guzmud Date: Tue, 21 Apr 2026 09:35:55 +0200 Subject: [PATCH 04/25] [template] feat(protocols): adding source-related protocols --- template/src/collector/protocols/__init__.py | 5 +++++ .../src/collector/protocols/data_fetcher.py | 6 +++++ .../src/collector/protocols/source_data.py | 9 ++++++++ .../src/collector/protocols/source_handler.py | 22 +++++++++++++++++++ 4 files changed, 42 insertions(+) create mode 100644 template/src/collector/protocols/__init__.py create mode 100644 template/src/collector/protocols/data_fetcher.py create mode 100644 template/src/collector/protocols/source_data.py create mode 100644 template/src/collector/protocols/source_handler.py diff --git a/template/src/collector/protocols/__init__.py b/template/src/collector/protocols/__init__.py new file mode 100644 index 00000000..ea8c73ad --- /dev/null +++ b/template/src/collector/protocols/__init__.py @@ -0,0 +1,5 @@ +from src.collector.protocols.data_fetcher import DataFetcherProtocol +from src.collector.protocols.source_data import SourceDataProtocol +from src.collector.protocols.source_handler import SourceHandlerProtocol + +__all__ = ["DataFetcherProtocol", "SourceDataProtocol", "SourceHandlerProtocol"] diff --git a/template/src/collector/protocols/data_fetcher.py b/template/src/collector/protocols/data_fetcher.py new file mode 100644 index 00000000..f23acf05 --- /dev/null +++ b/template/src/collector/protocols/data_fetcher.py @@ -0,0 +1,6 @@ +from typing import Protocol + + +class DataFetcherProtocol(Protocol): + def fetch_data(self): + ... diff --git a/template/src/collector/protocols/source_data.py b/template/src/collector/protocols/source_data.py new file mode 100644 index 00000000..b2155897 --- /dev/null +++ b/template/src/collector/protocols/source_data.py @@ -0,0 +1,9 @@ +from typing import Protocol + + +class SourceDataProtocol(Protocol): + def to_oaev_data(self): + ... + + def to_traces_data(self): + ... diff --git a/template/src/collector/protocols/source_handler.py b/template/src/collector/protocols/source_handler.py new file mode 100644 index 00000000..f049cff3 --- /dev/null +++ b/template/src/collector/protocols/source_handler.py @@ -0,0 +1,22 @@ +from typing import Protocol + + +class SourceHandlerProtocol(Protocol): + + def get_source_data(self): + ... + + def get_oaev_data(self): + ... + + def get_signatures(self): + ... + + def match_signatures(self): + ... + + def get_traces_data(self): + ... + + def match_expectations(self): + ... From 4eab80e4bdc3b991d8cea34c908fc8079d85d561 Mon Sep 17 00:00:00 2001 From: guzmud Date: Tue, 21 Apr 2026 09:37:00 +0200 Subject: [PATCH 05/25] [template] refactor(settings): renaming configs to settings --- template/src/models/__init__.py | 2 +- template/src/models/{configs => settings}/__init__.py | 0 template/src/models/{configs => settings}/base_settings.py | 0 template/src/models/{configs => settings}/collector_configs.py | 0 template/src/models/{configs => settings}/config_loader.py | 0 template/src/models/{configs => settings}/template_configs.py | 0 template/src/services/trace_service.py | 2 +- 7 files changed, 2 insertions(+), 2 deletions(-) rename template/src/models/{configs => settings}/__init__.py (100%) rename template/src/models/{configs => settings}/base_settings.py (100%) rename template/src/models/{configs => settings}/collector_configs.py (100%) rename template/src/models/{configs => settings}/config_loader.py (100%) rename template/src/models/{configs => settings}/template_configs.py (100%) diff --git a/template/src/models/__init__.py b/template/src/models/__init__.py index 22144e34..30f22504 100644 --- a/template/src/models/__init__.py +++ b/template/src/models/__init__.py @@ -1,3 +1,3 @@ -from src.models.configs.config_loader import ConfigLoader +from src.models.settings.config_loader import ConfigLoader __all__ = ["ConfigLoader"] diff --git a/template/src/models/configs/__init__.py b/template/src/models/settings/__init__.py similarity index 100% rename from template/src/models/configs/__init__.py rename to template/src/models/settings/__init__.py diff --git a/template/src/models/configs/base_settings.py b/template/src/models/settings/base_settings.py similarity index 100% rename from template/src/models/configs/base_settings.py rename to template/src/models/settings/base_settings.py diff --git a/template/src/models/configs/collector_configs.py b/template/src/models/settings/collector_configs.py similarity index 100% rename from template/src/models/configs/collector_configs.py rename to template/src/models/settings/collector_configs.py diff --git a/template/src/models/configs/config_loader.py b/template/src/models/settings/config_loader.py similarity index 100% rename from template/src/models/configs/config_loader.py rename to template/src/models/settings/config_loader.py diff --git a/template/src/models/configs/template_configs.py b/template/src/models/settings/template_configs.py similarity index 100% rename from template/src/models/configs/template_configs.py rename to template/src/models/settings/template_configs.py diff --git a/template/src/services/trace_service.py b/template/src/services/trace_service.py index d7d912fe..e8389e27 100644 --- a/template/src/services/trace_service.py +++ b/template/src/services/trace_service.py @@ -5,7 +5,7 @@ from typing import Any from ..collector.models import ExpectationResult, ExpectationTrace -from ..models.configs.config_loader import ConfigLoader +from ..models.settings.config_loader import ConfigLoader from .exception import TemplateDataConversionError, TemplateValidationError LOG_PREFIX = "[TemplateTraceService]" From 390988d6b9b8234ec17a21bd65178d7a8b906d9c Mon Sep 17 00:00:00 2001 From: guzmud Date: Wed, 22 Apr 2026 11:17:53 +0200 Subject: [PATCH 06/25] [template] refactor(collector): exception models move to base models --- template/src/collector/{ => base_models}/exception.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename template/src/collector/{ => base_models}/exception.py (100%) diff --git a/template/src/collector/exception.py b/template/src/collector/base_models/exception.py similarity index 100% rename from template/src/collector/exception.py rename to template/src/collector/base_models/exception.py From 22a31d48625371aade8958fd819fd5bd346874dc Mon Sep 17 00:00:00 2001 From: guzmud Date: Thu, 7 May 2026 11:51:19 +0200 Subject: [PATCH 07/25] [template] feat(protocols): adding collector protocols to the template --- template/src/collector/protocols/__init__.py | 5 - .../src/collector/protocols/data_fetcher.py | 8 +- template/src/collector/protocols/engine.py | 22 ++ .../src/collector/protocols/source_data.py | 17 +- .../src/collector/protocols/source_handler.py | 47 ++-- .../collector/protocols/test_data_fetcher.py | 24 +++ .../tests/collector/protocols/test_engine.py | 35 +++ .../collector/protocols/test_source_data.py | 107 ++++++++++ .../protocols/test_source_handler.py | 201 ++++++++++++++++++ 9 files changed, 440 insertions(+), 26 deletions(-) create mode 100644 template/src/collector/protocols/engine.py create mode 100644 template/tests/collector/protocols/test_data_fetcher.py create mode 100644 template/tests/collector/protocols/test_engine.py create mode 100644 template/tests/collector/protocols/test_source_data.py create mode 100644 template/tests/collector/protocols/test_source_handler.py diff --git a/template/src/collector/protocols/__init__.py b/template/src/collector/protocols/__init__.py index ea8c73ad..e69de29b 100644 --- a/template/src/collector/protocols/__init__.py +++ b/template/src/collector/protocols/__init__.py @@ -1,5 +0,0 @@ -from src.collector.protocols.data_fetcher import DataFetcherProtocol -from src.collector.protocols.source_data import SourceDataProtocol -from src.collector.protocols.source_handler import SourceHandlerProtocol - -__all__ = ["DataFetcherProtocol", "SourceDataProtocol", "SourceHandlerProtocol"] diff --git a/template/src/collector/protocols/data_fetcher.py b/template/src/collector/protocols/data_fetcher.py index f23acf05..2603487c 100644 --- a/template/src/collector/protocols/data_fetcher.py +++ b/template/src/collector/protocols/data_fetcher.py @@ -1,6 +1,8 @@ -from typing import Protocol +from typing import Protocol, runtime_checkable +from src.collector.protocols.source_data import SourceDataProtocol + +@runtime_checkable class DataFetcherProtocol(Protocol): - def fetch_data(self): - ... + def fetch_data(self) -> list[SourceDataProtocol]: ... diff --git a/template/src/collector/protocols/engine.py b/template/src/collector/protocols/engine.py new file mode 100644 index 00000000..9e3d5704 --- /dev/null +++ b/template/src/collector/protocols/engine.py @@ -0,0 +1,22 @@ +from typing import Protocol, runtime_checkable + +from pyoaev.client import OpenAEV +from src.collector.models.source import Source +from src.collector.protocols.source_handler import SourceHandlerProtocol + + +@runtime_checkable +class CollectorEngineProtocol(Protocol): + def __init__( + self, + name: str, + collector_id: str, + source: Source, + source_handler: SourceHandlerProtocol, + oaev_api: OpenAEV, + batching: bool = False, + ): ... + + def configure_engine(self, config, batching=False): ... + + def run_engine(self) -> None: ... diff --git a/template/src/collector/protocols/source_data.py b/template/src/collector/protocols/source_data.py index b2155897..9690e0b9 100644 --- a/template/src/collector/protocols/source_data.py +++ b/template/src/collector/protocols/source_data.py @@ -1,9 +1,16 @@ -from typing import Protocol +from typing import Protocol, runtime_checkable +from src.collector.models.data import OAEVData, TraceData + +@runtime_checkable class SourceDataProtocol(Protocol): - def to_oaev_data(self): - ... + def to_oaev_data(self) -> OAEVData: ... + + def to_traces_data(self) -> TraceData: ... + + def is_prevented(self) -> bool: ... + + def is_detected(self) -> bool: ... - def to_traces_data(self): - ... + def __str__(self) -> str: ... diff --git a/template/src/collector/protocols/source_handler.py b/template/src/collector/protocols/source_handler.py index f049cff3..3cc7a4df 100644 --- a/template/src/collector/protocols/source_handler.py +++ b/template/src/collector/protocols/source_handler.py @@ -1,22 +1,43 @@ -from typing import Protocol +from typing import Protocol, runtime_checkable +from pyoaev.apis.inject_expectation.model.expectation import ( + DetectionExpectation, + PreventionExpectation, +) +from pyoaev.helpers import OpenAEVDetectionHelper +from pyoaev.signatures.types import SignatureTypes +from src.collector.models.data import OAEVData, TraceData +from src.collector.protocols.data_fetcher import DataFetcherProtocol +from src.collector.protocols.source_data import SourceDataProtocol +from src.collector.types.collector import SignatureGroups + +@runtime_checkable class SourceHandlerProtocol(Protocol): - def get_source_data(self): - ... + def get_source_data( + self, data_fetcher: DataFetcherProtocol + ) -> list[SourceDataProtocol]: ... - def get_oaev_data(self): - ... + def serialize_as_oaevdata(self, data: SourceDataProtocol) -> OAEVData: ... - def get_signatures(self): - ... + def get_expectation_signature_groups( + self, + signatures: list[SignatureTypes], + expectation: DetectionExpectation | PreventionExpectation, + ) -> SignatureGroups: ... - def match_signatures(self): - ... + def match_signature_groups_and_oaevdata( + self, + signature_groups: SignatureGroups, + oaev_data: OAEVData, + oaev_detection_helper: OpenAEVDetectionHelper, + ) -> bool: ... - def get_traces_data(self): - ... + def serialize_as_tracedata(self, data: SourceDataProtocol) -> TraceData: ... - def match_expectations(self): - ... + def match_expectation_and_sourcedata( + self, + expectation: DetectionExpectation | PreventionExpectation, + data: SourceDataProtocol, + ) -> tuple[bool, bool]: ... diff --git a/template/tests/collector/protocols/test_data_fetcher.py b/template/tests/collector/protocols/test_data_fetcher.py new file mode 100644 index 00000000..954066b4 --- /dev/null +++ b/template/tests/collector/protocols/test_data_fetcher.py @@ -0,0 +1,24 @@ +import unittest +from unittest.mock import MagicMock + +import src.collector.protocols.data_fetcher as module + + +class DataFetcherProtocolTest(unittest.TestCase): + def test_following_data_fetcher_protocol(self): + """verify that a class following the protocol is seen as such""" + + class GoodDataFetcher: + def fetch_data(self): + return [MagicMock()] + + self.assertTrue(issubclass(GoodDataFetcher, module.DataFetcherProtocol)) + + def test_not_following_data_fetcher_protocol(self): + """verify that a class not following the protocol is detected as such""" + + class BadDataFetcher: + def retrieve_data(self): + return [MagicMock()] + + self.assertFalse(issubclass(BadDataFetcher, module.DataFetcherProtocol)) diff --git a/template/tests/collector/protocols/test_engine.py b/template/tests/collector/protocols/test_engine.py new file mode 100644 index 00000000..a76d3f07 --- /dev/null +++ b/template/tests/collector/protocols/test_engine.py @@ -0,0 +1,35 @@ +import unittest + +import src.collector.protocols.engine as module + + +class CollectorEngineProtocolTest(unittest.TestCase): + def test_following_collector_engine_protocol(self): + """verify that a class following the protocol is seen as such""" + + class GoodCollectorEngine: + def __init__(self, **kwargs): + pass + + def configure_engine(self, config, batching=False): + pass + + def run_engine(self): + pass + + self.assertTrue(issubclass(GoodCollectorEngine, module.CollectorEngineProtocol)) + + def test_not_following_collector_engine_protocol(self): + """verify that a class not following the protocol is detected as such""" + + class BadCollectorEngine: + def __init__(self, **kwargs): + pass + + def configure_engine(self, config, batching=False): + pass + + def start_engine(self): + pass + + self.assertFalse(issubclass(BadCollectorEngine, module.CollectorEngineProtocol)) diff --git a/template/tests/collector/protocols/test_source_data.py b/template/tests/collector/protocols/test_source_data.py new file mode 100644 index 00000000..58fe80da --- /dev/null +++ b/template/tests/collector/protocols/test_source_data.py @@ -0,0 +1,107 @@ +import unittest +from unittest.mock import MagicMock + +import src.collector.protocols.source_data as module + + +class SourceDataProtocolTest(unittest.TestCase): + def test_following_source_data_protocol(self): + """verify that a class following the protocol is seen as such""" + + class GoodSourceData: + def to_oaev_data(self): + return MagicMock() + + def to_traces_data(self): + return MagicMock() + + def is_prevented(self): + return False + + def is_detected(self): + return True + + def __str__(self): + return "" + + self.assertTrue(issubclass(GoodSourceData, module.SourceDataProtocol)) + + def test_not_following_source_data_protocol_no_oaev_data(self): + """verify that a class not following the protocol is detected as such""" + + class BadSourceData_no_oaev_data: + def to_traces_data(self): + return MagicMock() + + def is_prevented(self): + return False + + def is_detected(self): + return True + + def __str__(self): + return "" + + self.assertFalse( + issubclass(BadSourceData_no_oaev_data, module.SourceDataProtocol) + ) + + def test_not_following_source_data_protocol_no_traces_data(self): + """verify that a class not following the protocol is detected as such""" + + class BadSourceData_no_traces_data: + def to_oaev_data(self): + return MagicMock() + + def is_prevented(self): + return False + + def is_detected(self): + return True + + def __str__(self): + return "" + + self.assertFalse( + issubclass(BadSourceData_no_traces_data, module.SourceDataProtocol) + ) + + def test_not_following_source_data_protocol_no_prevented(self): + """verify that a class not following the protocol is detected as such""" + + class BadSourceData_no_prevented: + def to_oaev_data(self): + return MagicMock() + + def to_traces_data(self): + return MagicMock() + + def is_detected(self): + return True + + def __str__(self): + return "" + + self.assertFalse( + issubclass(BadSourceData_no_prevented, module.SourceDataProtocol) + ) + + def test_not_following_source_data_protocol_no_detected(self): + """verify that a class not following the protocol is detected as such""" + + class BadSourceData_no_detected: + def to_oaev_data(self): + return MagicMock() + + def to_traces_data(self): + return MagicMock() + + def is_prevented(self): + return False + + def __str__(self): + return "" + + self.assertFalse( + issubclass(BadSourceData_no_detected, module.SourceDataProtocol) + ) diff --git a/template/tests/collector/protocols/test_source_handler.py b/template/tests/collector/protocols/test_source_handler.py new file mode 100644 index 00000000..0cf402d2 --- /dev/null +++ b/template/tests/collector/protocols/test_source_handler.py @@ -0,0 +1,201 @@ +import unittest +from unittest.mock import MagicMock + +import src.collector.protocols.source_handler as module + + +class SourceHandlerProtocolTest(unittest.TestCase): + def test_following_source_handler_protocol(self): + """verify that a class following the protocol is seen as such""" + + class GoodSourceHandler: + def get_source_data(self, data_fetcher): + return [MagicMock()] + + def serialize_as_oaevdata(self, data): + return MagicMock() + + def get_expectation_signature_groups(self, signatures, expectations): + return [{"foo": "bar"}] + + def match_signature_groups_and_oaevdata( + self, signature_groups, oaev_data, oaev_detection_helper + ): + return True + + def serialize_as_tracedata(self, data): + return MagicMock() + + def match_expectation_and_sourcedata(self, expectation, data): + return [False, False] + + self.assertTrue(issubclass(GoodSourceHandler, module.SourceHandlerProtocol)) + + def test_not_following_source_handler_protocol_no_get_source_data(self): + """verify that a class not following the protocol is detected as such""" + + class BadSourceHandler_no_get_source_data: + def serialize_as_oaevdata(self, data): + return MagicMock() + + def get_expectation_signature_groups(self, signatures, expectations): + return [{"foo": "bar"}] + + def match_signature_groups_and_oaevdata( + self, signature_groups, oaev_data, oaev_detection_helper + ): + return True + + def serialize_as_tracedata(self, data): + return MagicMock() + + def match_expectation_and_sourcedata(self, expectation, data): + return [False, False] + + self.assertFalse( + issubclass( + BadSourceHandler_no_get_source_data, module.SourceHandlerProtocol + ) + ) + + def test_not_following_source_handler_protocol_no_serialize_as_oaevdata(self): + """verify that a class not following the protocol is detected as such""" + + class BadSourceHandler_no_serialize_as_oaevdata: + def get_source_data(self, data_fetcher): + return [MagicMock()] + + def get_expectation_signature_groups(self, signatures, expectations): + return [{"foo": "bar"}] + + def match_signature_groups_and_oaevdata( + self, signature_groups, oaev_data, oaev_detection_helper + ): + return True + + def serialize_as_tracedata(self, data): + return MagicMock() + + def match_expectation_and_sourcedata(self, expectation, data): + return [False, False] + + self.assertFalse( + issubclass( + BadSourceHandler_no_serialize_as_oaevdata, module.SourceHandlerProtocol + ) + ) + + def test_not_following_source_handler_protocol_no_get_expectation_signature_groups( + self, + ): + """verify that a class not following the protocol is detected as such""" + + class BadSourceHandler_no_get_expectation_signature_groups: + def get_source_data(self, data_fetcher): + return [MagicMock()] + + def serialize_as_oaevdata(self, data): + return MagicMock() + + def match_signature_groups_and_oaevdata( + self, signature_groups, oaev_data, oaev_detection_helper + ): + return True + + def serialize_as_tracedata(self, data): + return MagicMock() + + def match_expectation_and_sourcedata(self, expectation, data): + return [False, False] + + self.assertFalse( + issubclass( + BadSourceHandler_no_get_expectation_signature_groups, + module.SourceHandlerProtocol, + ) + ) + + def test_not_following_source_handler_protocol_no_match_signature_groups_and_oaevdata( + self, + ): + """verify that a class not following the protocol is detected as such""" + + class BadSourceHandler_no_match_signature_groups_and_oaevdata: + def get_source_data(self, data_fetcher): + return [MagicMock()] + + def serialize_as_oaevdata(self, data): + return MagicMock() + + def get_expectation_signature_groups(self, signatures, expectations): + return [{"foo": "bar"}] + + def serialize_as_tracedata(self, data): + return MagicMock() + + def match_expectation_and_sourcedata(self, expectation, data): + return [False, False] + + self.assertFalse( + issubclass( + BadSourceHandler_no_match_signature_groups_and_oaevdata, + module.SourceHandlerProtocol, + ) + ) + + def test_not_following_source_handler_protocol_no_serialize_as_tracedata(self): + """verify that a class not following the protocol is detected as such""" + + class BadSourceHandler_no_serialize_as_tracedata: + def get_source_data(self, data_fetcher): + return [MagicMock()] + + def serialize_as_oaevdata(self, data): + return MagicMock() + + def get_expectation_signature_groups(self, signatures, expectations): + return [{"foo": "bar"}] + + def match_signature_groups_and_oaevdata( + self, signature_groups, oaev_data, oaev_detection_helper + ): + return True + + def match_expectation_and_sourcedata(self, expectation, data): + return [False, False] + + self.assertFalse( + issubclass( + BadSourceHandler_no_serialize_as_tracedata, module.SourceHandlerProtocol + ) + ) + + def test_not_following_source_handler_protocol_no_match_expectation_and_sourcedata( + self, + ): + """verify that a class not following the protocol is detected as such""" + + class BadSourceHandler_no_match_expectation_and_sourcedata: + def get_source_data(self, data_fetcher): + return [MagicMock()] + + def serialize_as_oaevdata(self, data): + return MagicMock() + + def get_expectation_signature_groups(self, signatures, expectations): + return [{"foo": "bar"}] + + def match_signature_groups_and_oaevdata( + self, signature_groups, oaev_data, oaev_detection_helper + ): + return True + + def serialize_as_tracedata(self, data): + return MagicMock() + + self.assertFalse( + issubclass( + BadSourceHandler_no_match_expectation_and_sourcedata, + module.SourceHandlerProtocol, + ) + ) From a069524eebf05eebbb79fbb409b0e40c48237cb4 Mon Sep 17 00:00:00 2001 From: guzmud Date: Thu, 7 May 2026 11:52:25 +0200 Subject: [PATCH 08/25] [template] feat(types): adding custom collector types --- template/src/collector/types/__init__.py | 0 template/src/collector/types/collector.py | 3 +++ template/src/collector/types/internals.py | 6 ++++++ 3 files changed, 9 insertions(+) create mode 100644 template/src/collector/types/__init__.py create mode 100644 template/src/collector/types/collector.py create mode 100644 template/src/collector/types/internals.py diff --git a/template/src/collector/types/__init__.py b/template/src/collector/types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/src/collector/types/collector.py b/template/src/collector/types/collector.py new file mode 100644 index 00000000..384756f9 --- /dev/null +++ b/template/src/collector/types/collector.py @@ -0,0 +1,3 @@ +from typing import TypeAlias + +SignatureGroups: TypeAlias = list[dict[str, str]] diff --git a/template/src/collector/types/internals.py b/template/src/collector/types/internals.py new file mode 100644 index 00000000..ba2045f7 --- /dev/null +++ b/template/src/collector/types/internals.py @@ -0,0 +1,6 @@ +from typing import Any, Callable, Iterable, TypeAlias + +PrepareBulkFunction: TypeAlias = Callable[[list[Any]], tuple[Iterable[Any], int]] +BulkUploadFunction: TypeAlias = Callable[[Iterable[Any]], None] +UnpackBulkFunction: TypeAlias = Callable[[Iterable[Any]], Iterable[tuple[Any, Any]]] +IndividualUploadFunction: TypeAlias = Callable[[Any, Any], None] From 4ec8a1d5fd8fe9b26f24d6eed853260cc800082e Mon Sep 17 00:00:00 2001 From: guzmud Date: Thu, 7 May 2026 11:53:33 +0200 Subject: [PATCH 09/25] [template] feat(models): adding collector models to the template --- template/src/collector/models/__init__.py | 0 template/src/collector/models/data.py | 54 ++++ template/src/collector/models/exception.py | 85 +++++ template/src/collector/models/expectations.py | 263 +++++++++++++++ template/src/collector/models/source.py | 136 ++++++++ template/tests/collector/models/__init__.py | 0 template/tests/collector/models/test_data.py | 115 +++++++ .../collector/models/test_expectations.py | 257 +++++++++++++++ .../tests/collector/models/test_source.py | 304 ++++++++++++++++++ 9 files changed, 1214 insertions(+) create mode 100644 template/src/collector/models/__init__.py create mode 100644 template/src/collector/models/data.py create mode 100644 template/src/collector/models/exception.py create mode 100644 template/src/collector/models/expectations.py create mode 100644 template/src/collector/models/source.py create mode 100644 template/tests/collector/models/__init__.py create mode 100644 template/tests/collector/models/test_data.py create mode 100644 template/tests/collector/models/test_expectations.py create mode 100644 template/tests/collector/models/test_source.py diff --git a/template/src/collector/models/__init__.py b/template/src/collector/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/src/collector/models/data.py b/template/src/collector/models/data.py new file mode 100644 index 00000000..d270df5b --- /dev/null +++ b/template/src/collector/models/data.py @@ -0,0 +1,54 @@ +from datetime import UTC, datetime +from typing import Any, ClassVar + +from pydantic import AnyUrl, BaseModel, Field, model_validator +from pyoaev.signatures.types import SignatureTypes + + +class OAEVData(BaseModel, extra="allow"): + """ + Source-side version of OAEV formatted data. + Apart from context, the allowed fields are signature types (e.g. parent_process_name) + """ + + __pydantic_extra__: dict[str, str] = {} # or should it be dict[str, Any] + _allowed_values: ClassVar[frozenset[str]] = frozenset( + [sig.value for sig in SignatureTypes] + ) + + @model_validator(mode="before") + @classmethod + def check_field_names(cls, data: Any) -> Any: + """Check whether the fields provided through extra are actually signature types""" + if isinstance(data, dict): + for key in data: + if key not in cls._allowed_values: + raise ValueError("Only signature types are allowed") + return data + + def __str__(self) -> str: + """str formatted version of the object""" + text = ", ".join( + f"{key}='{value}'" + for key, value in self.model_dump().items() + if key in self._allowed_values + ) + text = f"OAEVData({text})" + return text + + +class TraceData(BaseModel): + """Source-side version of OAEV formatted data""" + + alert_name: str = Field(..., description="Alert name") + alert_link: AnyUrl = Field(..., description="Alert link") + alert_date: datetime = Field( + ..., description="Alert date", default_factory=lambda: datetime.now(UTC) + ) + + def __str__(self) -> str: + """str formatted version of the object""" + return ( + f"TraceData(alert_name='{self.alert_name}', alert_link='{self.alert_link}', " + f"alert_date='{self.alert_date}')" + ) diff --git a/template/src/collector/models/exception.py b/template/src/collector/models/exception.py new file mode 100644 index 00000000..bb8e5072 --- /dev/null +++ b/template/src/collector/models/exception.py @@ -0,0 +1,85 @@ +"""Custom exceptions for the collector.""" + + +class CollectorError(Exception): + """Base exception for the collector.""" + + pass + + +class CollectorConfigError(CollectorError): + """Exception raised when there is an error in the collector configuration.""" + + pass + + +class CollectorEngineConfigError(CollectorError): + """Exception raised when there is an error in the collector engine configuration.""" + + pass + + +class CollectorSetupError(CollectorError): + """Exception raised when there is an error setting up the collector.""" + + pass + + +class CollectorProcessingError(CollectorError): + """Exception raised when there is an error processing data in the collector.""" + + pass + + +class ExpectationHandlerError(CollectorError): + """Exception raised when there is an error in expectation handling.""" + + pass + + +class ExpectationProcessingError(CollectorError): + """Exception raised when there is an error processing expectations.""" + + pass + + +class ExpectationUpdateError(CollectorError): + """Exception raised when there is an error updating expectations.""" + + pass + + +class BulkUploadError(ExpectationUpdateError): + """Exception raised when there is an error during bulk upload operations.""" + + pass + + +class BulkPreparationError(ExpectationUpdateError): + """Exception raised when there is an error during bulk preparation operations.""" + + pass + + +class APIError(CollectorError): + """Exception raised when there is an error with API operations.""" + + pass + + +class TracingError(CollectorError): + """Exception raised when there is an error with tracing operations.""" + + pass + + +class TraceSubmissionError(TracingError): + """Exception raised when there is an error submitting traces.""" + + pass + + +class TraceCreationError(TracingError): + """Exception raised when there is an error creating traces.""" + + pass diff --git a/template/src/collector/models/expectations.py b/template/src/collector/models/expectations.py new file mode 100644 index 00000000..800d4360 --- /dev/null +++ b/template/src/collector/models/expectations.py @@ -0,0 +1,263 @@ +"""Pydantic models for collector data structures.""" + +from datetime import UTC, datetime +from typing import Any + +from pydantic import BaseModel, Field, field_validator +from pyoaev.apis.inject_expectation.model import ( + DetectionExpectation, + PreventionExpectation, +) + + +class ExpectationResult(BaseModel): + """Model for expectation processing results.""" + + expectation_id: str = Field(..., description="ID of the processed expectation") + is_valid: bool = Field(..., description="Whether the expectation was validated") + expectation: Any | None = Field(None, description="The original expectation object") + matched_alerts: list[dict[str, Any]] | None = Field( + None, description="List of alerts that matched this expectation" + ) + error_message: str | None = Field( + None, description="Error message if processing failed" + ) + processing_time: float | None = Field( + None, description="Time taken to process this expectation in seconds" + ) + + @classmethod + def from_error( + cls, error: Exception, expectation: DetectionExpectation | PreventionExpectation + ): + """ + Produce an ExpectationResult based on an error message + and the related expectation + """ + return cls( + expectation_id=str(expectation.inject_expectation_id), + is_valid=False, + expectation=expectation, + matched_alerts=None, + error_message=str(error), + processing_time=None, + ) + + def to_result_text(self) -> str: + """ + Produce the text-based result required for bulk upload + """ + if not self.expectation: + return "Unknown" + + text = "" + if not self.is_valid: + text += "Not " + if isinstance(self.expectation, DetectionExpectation): + text += "Detected" + else: + text += "Prevented" + + return text + + +class ExpectationTrace(BaseModel): + """Pydantic model for expectation trace data. + + This model represents the structure of trace data that gets sent to the + OpenAEV API for expectation tracking and validation. + """ + + inject_expectation_trace_expectation: str = Field( + description="The expectation ID this trace is associated with" + ) + inject_expectation_trace_source_id: str = Field( + description="The collector/source ID that generated this trace" + ) + inject_expectation_trace_alert_name: str = Field( + description="Name of the alert that was matched" + ) + inject_expectation_trace_alert_link: str = Field( + description="Link to the alert in the source system" + ) + inject_expectation_trace_date: str = Field( + description="Date when the trace was created (ISO format string)" + ) + + @field_validator("inject_expectation_trace_expectation") + @classmethod + def expectation_must_not_be_empty(cls, v: str) -> str: + """Validate that expectation ID is not empty. + + Args: + v: The expectation ID value to validate. + + Returns: + The trimmed expectation ID. + + Raises: + ValueError: If the expectation ID is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Expectation ID cannot be empty") + return v.strip() + + @field_validator("inject_expectation_trace_source_id") + @classmethod + def source_id_must_not_be_empty(cls, v: str) -> str: + """Validate that source ID is not empty. + + Args: + v: The source ID value to validate. + + Returns: + The trimmed source ID. + + Raises: + ValueError: If the source ID is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Source ID cannot be empty") + return v.strip() + + @field_validator("inject_expectation_trace_alert_name") + @classmethod + def alert_name_must_not_be_empty(cls, v: str) -> str: + """Validate that alert name is not empty. + + Args: + v: The alert name value to validate. + + Returns: + The trimmed alert name. + + Raises: + ValueError: If the alert name is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Alert name cannot be empty") + return v.strip() + + @field_validator("inject_expectation_trace_alert_link") + @classmethod + def alert_link_must_not_be_empty(cls, v: str) -> str: + """Validate that alert link is not empty. + + Args: + v: The alert link value to validate. + + Returns: + The trimmed alert link. + + Raises: + ValueError: If the alert link is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Alert link cannot be empty") + return v.strip() + + @field_validator("inject_expectation_trace_date") + @classmethod + def date_must_not_be_empty(cls, v: str) -> str: + """Validate that date is not empty. + + Args: + v: The date value to validate. + + Returns: + The trimmed date string. + + Raises: + ValueError: If the date is empty or whitespace only. + + """ + if not v or not v.strip(): + raise ValueError("Trace date cannot be empty") + return v.strip() + + def to_api_dict(self) -> dict[str, str]: + """Convert the model to a dictionary suitable for API submission. + + This method ensures all values are strings as expected by the API, + replacing the manual sanitization logic in the expectation manager. + + Returns: + Dict with all values converted to strings. + + """ + return { + key: str(value) if value is not None else "" + for key, value in self.model_dump().items() + } + + @classmethod + def from_result( + cls, result: ExpectationResult, collector_id: str, collector_name: str + ): + """ + Produce an ExpectationTrace based on the provided ExpectationResult + and collector's ID and name + """ + matching_data = result.matched_alerts[0] or {} + alert_name = matching_data.get("alert_name", f"{collector_name} Alert") + trace_link = matching_data.get("alert_link", "") + trace_date = datetime.now(UTC).replace(microsecond=0) + date_str = trace_date.isoformat().replace("+00:00", "Z") # TODO WTF? + return cls( + inject_expectation_trace_expectation=str(result.expectation_id), + inject_expectation_trace_source_id=str(collector_id), + inject_expectation_trace_alert_name=alert_name, + inject_expectation_trace_alert_link=trace_link, + inject_expectation_trace_date=date_str, + ) + + +class ExpectationSummary(BaseModel): + """Model for expectation processing summary.""" + + received: int = Field( + default=0, description="Total number of expectations received" + ) + supported: int = Field( + default=0, description="Total number of expectations supported" + ) + processed: int = Field( + default=0, description="Total number of expectations processed" + ) + valid: int = Field(default=0, description="Number of valid expectations") + total_processing_time: float | None = Field( + None, description="Total processing time in seconds" + ) + + @property + def unsupported(self): + """Number of unsupported expectations received""" + return self.received - self.supported + + @property + def unprocessed(self): + """Number of expectations skipped during processing""" + return self.supported - self.processed + + @property + def invalid(self): + """Number of invalid expectations""" + return self.processed - self.valid + + @property + def total_skipped(self): + """Number of expectations skipped since receiving (unsupported+unprocessed)""" + return self.received - self.processed + + def __str__(self): + """Return an overview of the summary as a string""" + return ( + f"{self.received} expectations received, " + f"{self.supported} expectations supported ({self.unsupported} unsupported), " + f"{self.processed} expectations processed ({self.unprocessed} unprocessed), " + f"{self.valid} valid expectations ({self.invalid} invalid)" + ) diff --git a/template/src/collector/models/source.py b/template/src/collector/models/source.py new file mode 100644 index 00000000..83e05a0a --- /dev/null +++ b/template/src/collector/models/source.py @@ -0,0 +1,136 @@ +from pydantic import BaseModel +from pyoaev.apis.inject_expectation.model.expectation import ( + DetectionExpectation, + PreventionExpectation, +) +from pyoaev.helpers import OpenAEVDetectionHelper +from pyoaev.signatures.types import SignatureTypes +from src.collector.models.data import OAEVData, TraceData +from src.collector.protocols.data_fetcher import DataFetcherProtocol +from src.collector.protocols.source_data import SourceDataProtocol +from src.collector.protocols.source_handler import SourceHandlerProtocol +from src.collector.types.collector import SignatureGroups + + +class Source(BaseModel): + """ + A source is defined by three elements: + - the data fetcher model used to fetch the relevant data from the implemented tool/service + - the source data model used to serialize and deserialize the fetched data + - the list of signature types expected to eventually match the data + """ + + data_fetcher_model: type[DataFetcherProtocol] # or is it type[DataFetcherProtocol]? + source_data_model: type[SourceDataProtocol] # or is it type[SourceDataProtocol]? + signatures: list[SignatureTypes] + + +class SourceHandler(SourceHandlerProtocol): + """ + the source handler is an interface between the streamlined collector engine + and the custom source elements, providing the details for each of the + following functions: + - how to fetch the source data using the data fetcher (get_source_data) + - how to serialize the source data into OAEVData (serialize_as_oaevdata) + - how to group the signatures from the expectations (get_expectation_signature_groups) + - how to match the grouped expectation signatures and the OAEVData (match_signature_groups_and_oaevdata) + - how to serialize the source data into TraceData (serialize_as_tracedata) + - how to match an expectation and the source data to check for detection/prevention + """ + + def get_source_data( + self, data_fetcher: DataFetcherProtocol + ) -> list[SourceDataProtocol]: + """ + get source data using the data fetcher + """ + data = data_fetcher.fetch_data() + # TODO: pass end_date? pass signature_extracted from batch? pass batch? pass context? + return data + + def serialize_as_oaevdata(self, data: SourceDataProtocol) -> OAEVData: + """ + serialize provided data as oaevdata + """ + oaev_data = data.to_oaev_data() + return oaev_data + + def get_expectation_signature_groups( + self, + signatures: list[SignatureTypes], + expectation: DetectionExpectation | PreventionExpectation, + ) -> dict[str, list[dict]]: + """ + group the expectation's signatures according to the source provided signatures + """ + supported_types = {sig_type.value for sig_type in signatures} + signature_groups = {} + for sig in expectation.inject_expectation_signatures: + # ignore unsupported signatures according to source + if sig.type.value not in supported_types: + continue + # ignore end_date signature type + if sig.type.value == "end_date": + continue + # create or append to a list of dict-serialized signature data + signature_groups.setdefault(sig.type.value, []).append( + {"type": sig.type.value, "value": sig.value} + ) + return signature_groups + + def match_signature_groups_and_oaevdata( + self, + signature_groups: SignatureGroups, + oaev_data: OAEVData, + oaev_detection_helper: OpenAEVDetectionHelper, + ) -> bool: + """ + matching signatures extracted from an expectation and already filtered against source's signatures + against the fetched data serialized in an OAEVData format (signature types oriented formating) + """ + if not oaev_data: + return False + + for sig_type, signature_data in signature_groups.items(): + try: + filtered_data = {sig_type: oaev_data.sig_type} + except AttributeError: + return False + match_result = oaev_detection_helper.match_alert_elements( + signature_data, filtered_data + ) + if not match_result: + return False + return True + + def serialize_as_tracedata(self, data: SourceDataProtocol) -> TraceData: + """ + use pydantic-based TraceData model to serialize then return in dictionary format + """ + trace = data.to_traces_data() + trace = trace.model_dump() + return trace + + def match_expectation_and_sourcedata( + self, + expectation: DetectionExpectation | PreventionExpectation, + data: SourceDataProtocol, + ) -> tuple[bool, bool]: + """ + matching expectation with fetched data to determine + whether an expectation has been satisfied + """ + # in any case an expectation is satisfied + matchflag = False + + # in case a prevention expectation is satisfied to skip useless processing + breakflag = False + + if isinstance(expectation, PreventionExpectation): + if data.is_prevented(): + matchflag = True + breakflag = True + else: + if data.is_detected(): + matchflag = True + return matchflag, breakflag diff --git a/template/tests/collector/models/__init__.py b/template/tests/collector/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/tests/collector/models/test_data.py b/template/tests/collector/models/test_data.py new file mode 100644 index 00000000..cd33f91e --- /dev/null +++ b/template/tests/collector/models/test_data.py @@ -0,0 +1,115 @@ +import unittest +from datetime import UTC, datetime + +import src.collector.models.data as module +from pydantic import AnyUrl, ValidationError + + +class OAEVDataTest(unittest.TestCase): + def test_oaev_data_good_init(self): + """ + testing the proper init of OAEVData + """ + parent_process_name = "mother" + end_date = str(datetime.now(UTC)) + + oaevdata = module.OAEVData( + parent_process_name=parent_process_name, end_date=end_date + ) + + self.assertEqual(oaevdata.parent_process_name, parent_process_name) + self.assertEqual(oaevdata.end_date, end_date) + + def test_oaev_data_wrong_init(self): + """ + testing the failure of OAEVData init + due to parameters outside the allowed range (signature types) + """ + parent_process_name = "mother" + foo = "bar" + + with self.assertRaises(ValidationError): + module.OAEVData(parent_process_name=parent_process_name, foo=foo) + + def test_oaev_data_str_format(self): + """ + testing the proper init of OAEVData + with all parameters + """ + parent_process_name = "mother" + end_date = str(datetime.now(UTC)) + + oaevdata = module.OAEVData( + parent_process_name=parent_process_name, end_date=end_date + ) + + self.assertEqual( + str(oaevdata), + f"OAEVData(parent_process_name='mother', end_date='{end_date}')", + ) + + +class TraceDataTest(unittest.TestCase): + def test_trace_data_minimum_init(self): + """ + testing the proper init of TraceData + with only required parameters + """ + alert_name = "my name is" + alert_link = "https://foo.bar/" + + tracedata = module.TraceData(alert_name=alert_name, alert_link=alert_link) + + self.assertEqual(tracedata.alert_name, alert_name) + self.assertIsInstance(tracedata.alert_link, AnyUrl) + self.assertEqual(str(tracedata.alert_link), alert_link) + self.assertIsNotNone(tracedata.alert_date) + self.assertIsInstance(tracedata.alert_date, datetime) + + def test_trace_data_maximal_init(self): + """ + testing the proper init of TraceData + with all parameters + """ + alert_name = "my name is" + alert_link = "https://foo.bar/" + alert_date = datetime.now(UTC) + + tracedata = module.TraceData( + alert_name=alert_name, alert_link=alert_link, alert_date=alert_date + ) + + self.assertEqual(tracedata.alert_name, alert_name) + self.assertIsInstance(tracedata.alert_link, AnyUrl) + self.assertEqual(str(tracedata.alert_link), alert_link) + self.assertIsInstance(tracedata.alert_date, datetime) + self.assertEqual(tracedata.alert_date, alert_date) + + def test_trace_data_wrong_init(self): + """ + testing a failed init of TraceData + due to a non-URL for alert_link + """ + alert_name = "my name is" + alert_link = "this is not a URL" + + with self.assertRaises(ValidationError): + module.TraceData(alert_name=alert_name, alert_link=alert_link) + + def test_trace_data_str_format(self): + """ + testing the proper init of TraceData + with all parameters + """ + alert_name = "my name is" + alert_link = "https://foo.bar/" + alert_date = datetime.now(UTC) + + tracedata = module.TraceData( + alert_name=alert_name, alert_link=alert_link, alert_date=alert_date + ) + + self.assertEqual( + str(tracedata), + f"TraceData(alert_name='my name is', alert_link='https://foo.bar/', alert_date='{alert_date}')", + ) diff --git a/template/tests/collector/models/test_expectations.py b/template/tests/collector/models/test_expectations.py new file mode 100644 index 00000000..f0b6d830 --- /dev/null +++ b/template/tests/collector/models/test_expectations.py @@ -0,0 +1,257 @@ +import unittest +from unittest.mock import ANY, MagicMock + +import src.collector.models.expectations as module + + +class ExpectationResultTest(unittest.TestCase): + def test_expectation_result_minimal_init(self): + """ + testing the proper init of ExpectationResult with only required parameters + """ + expectation_id = "id" + is_valid = False + + expectation_result = module.ExpectationResult( + expectation_id=expectation_id, is_valid=is_valid + ) + + self.assertEqual(expectation_id, expectation_result.expectation_id) + self.assertEqual(is_valid, expectation_result.is_valid) + self.assertIsNone(expectation_result.expectation) + self.assertIsNone(expectation_result.matched_alerts) + self.assertIsNone(expectation_result.error_message) + self.assertIsNone(expectation_result.processing_time) + + def test_expectation_result_full_init(self): + """ + testing the proper init of ExpectationResult with all parameters + """ + expectation_id = "id" + is_valid = False + expectation = MagicMock() + matched_alerts = [{"key": "valjue"}] + error_message = "this is an error" + processing_time = 12.34 + + expectation_result = module.ExpectationResult( + expectation_id=expectation_id, + is_valid=is_valid, + expectation=expectation, + matched_alerts=matched_alerts, + error_message=error_message, + processing_time=processing_time, + ) + + self.assertEqual(expectation_id, expectation_result.expectation_id) + self.assertEqual(is_valid, expectation_result.is_valid) + self.assertEqual(expectation, expectation_result.expectation) + self.assertEqual(matched_alerts, expectation_result.matched_alerts) + self.assertEqual(error_message, expectation_result.error_message) + self.assertEqual(processing_time, expectation_result.processing_time) + + def test_expectation_result_from_error(self): + """ + testing the proper init of ExpectationResult with from_error + """ + error = Exception("chat are we cooked") + expectation = MagicMock() + expectation.inject_expectation_id = "id" + + expectation_result = module.ExpectationResult.from_error(error, expectation) + + self.assertEqual("id", expectation_result.expectation_id) + self.assertFalse(expectation_result.is_valid) + self.assertEqual(expectation, expectation_result.expectation) + self.assertIsNone(expectation_result.matched_alerts) + self.assertEqual(str(error), expectation_result.error_message) + self.assertIsNone(expectation_result.processing_time) + + def test_expectation_result_to_result_text(self): + """ + testing the various output of the to_result_text + """ + detection_expectation = MagicMock(spec=module.DetectionExpectation) + prevention_expectation = MagicMock(spec=module.PreventionExpectation) + id = "id" + + self.assertEqual( + module.ExpectationResult( + expectation_id=id, is_valid=True, expectation=detection_expectation + ).to_result_text(), + "Detected", + ) + + self.assertEqual( + module.ExpectationResult( + expectation_id=id, is_valid=False, expectation=detection_expectation + ).to_result_text(), + "Not Detected", + ) + + self.assertEqual( + module.ExpectationResult( + expectation_id=id, is_valid=True, expectation=prevention_expectation + ).to_result_text(), + "Prevented", + ) + + self.assertEqual( + module.ExpectationResult( + expectation_id=id, is_valid=False, expectation=prevention_expectation + ).to_result_text(), + "Not Prevented", + ) + + +class ExpectationTraceTest(unittest.TestCase): + def test_expectation_trace_init(self): + """ + testing the proper init of ExpectationTrace + """ + expectation_id = "exp_id" + source_id = "source_id" + alert_name = "alert name" + alert_link = "http://alert.link" + date = "this is a date" + + expectation_trace = module.ExpectationTrace( + inject_expectation_trace_expectation=expectation_id, + inject_expectation_trace_source_id=source_id, + inject_expectation_trace_alert_name=alert_name, + inject_expectation_trace_alert_link=alert_link, + inject_expectation_trace_date=date, + ) + + self.assertEqual( + expectation_id, expectation_trace.inject_expectation_trace_expectation + ) + self.assertEqual( + source_id, expectation_trace.inject_expectation_trace_source_id + ) + self.assertEqual( + alert_name, expectation_trace.inject_expectation_trace_alert_name + ) + self.assertEqual( + alert_link, expectation_trace.inject_expectation_trace_alert_link + ) + self.assertEqual(date, expectation_trace.inject_expectation_trace_date) + + def test_expectation_trace_to_api_dict(self): + """ """ + expectation_id = "exp_id" + source_id = "source_id" + alert_name = "alert name" + alert_link = "http://alert.link" + date = "this is a date" + expectation_trace = module.ExpectationTrace( + inject_expectation_trace_expectation=expectation_id, + inject_expectation_trace_source_id=source_id, + inject_expectation_trace_alert_name=alert_name, + inject_expectation_trace_alert_link=alert_link, + inject_expectation_trace_date=date, + ) + + api_dict = expectation_trace.to_api_dict() + + self.assertEqual( + api_dict["inject_expectation_trace_expectation"], expectation_id + ) + self.assertEqual(api_dict["inject_expectation_trace_source_id"], source_id) + self.assertEqual(api_dict["inject_expectation_trace_alert_name"], alert_name) + self.assertEqual(api_dict["inject_expectation_trace_alert_link"], alert_link) + self.assertEqual(api_dict["inject_expectation_trace_date"], date) + + def test_expectation_trace_from_result(self): + """ """ + expectation_id = "id" + is_valid = False + expectation = MagicMock() + _name = "my name is" + _url = "http://alert.link" + matched_alerts = [{"alert_name": _name, "alert_link": _url}] + error_message = "this is an error" + processing_time = 12.34 + expectation_result = module.ExpectationResult( + expectation_id=expectation_id, + is_valid=is_valid, + expectation=expectation, + matched_alerts=matched_alerts, + error_message=error_message, + processing_time=processing_time, + ) + collector_id = "collector_id" + collector_name = "collector name" + + expectation_trace = module.ExpectationTrace.from_result( + expectation_result, collector_id, collector_name + ) + + self.assertEqual( + expectation_trace.inject_expectation_trace_expectation, + expectation_result.expectation_id, + ) + self.assertEqual( + expectation_trace.inject_expectation_trace_source_id, collector_id + ) + self.assertEqual(expectation_trace.inject_expectation_trace_alert_name, _name) + self.assertEqual(expectation_trace.inject_expectation_trace_alert_link, _url) + self.assertEqual(expectation_trace.inject_expectation_trace_date, ANY) + + +class ExpectationSummaryTest(unittest.TestCase): + def test_expectation_summary_minimal_init(self): + """ + testing the proper init of ExpectationSummary with only required parameters + """ + + summary = module.ExpectationSummary() + + self.assertEqual(0, summary.received) + self.assertEqual(0, summary.supported) + self.assertEqual(0, summary.unsupported) + self.assertEqual(0, summary.processed) + self.assertEqual(0, summary.unprocessed) + self.assertEqual(0, summary.valid) + self.assertEqual(0, summary.invalid) + self.assertEqual(0, summary.total_skipped) + + def test_expectation_summary_full_init(self): + """ + testing the proper init of ExpectationSummary with all parameters + """ + received = 100 + supported = 75 + processed = 40 + valid = 35 + + summary = module.ExpectationSummary( + received=received, supported=supported, processed=processed, valid=valid + ) + + self.assertEqual(100, summary.received) + self.assertEqual(75, summary.supported) + self.assertEqual(25, summary.unsupported) + self.assertEqual(40, summary.processed) + self.assertEqual(35, summary.unprocessed) + self.assertEqual(35, summary.valid) + self.assertEqual(5, summary.invalid) + self.assertEqual(60, summary.total_skipped) + + def test_expectation_summary_str(self): + """ + testing the formatting of ExpectationSummary into str + """ + received = 100 + supported = 75 + processed = 40 + valid = 35 + + summary = module.ExpectationSummary( + received=received, supported=supported, processed=processed, valid=valid + ) + + self.assertEqual( + str(summary), + "100 expectations received, 75 expectations supported (25 unsupported), 40 expectations processed (35 unprocessed), 35 valid expectations (5 invalid)", + ) diff --git a/template/tests/collector/models/test_source.py b/template/tests/collector/models/test_source.py new file mode 100644 index 00000000..4bf2ca79 --- /dev/null +++ b/template/tests/collector/models/test_source.py @@ -0,0 +1,304 @@ +import unittest +from unittest.mock import MagicMock + +import src.collector.models.source as module +from pydantic import ValidationError + + +class SourceTest(unittest.TestCase): + def setUp(self): + class FakeSourceData: + def to_oaev_data(self): + return module.OAEVData(parent_process_name="mother") + + def to_traces_data(self): + return module.TraceData( + alert_name="my name is", alert_link="http://foo.bar/" + ) + + def is_prevented(self): + return False + + def is_detected(self): + return True + + def __str__(self): + return "" + + self.source_data_model = FakeSourceData + + class FakeDataFetcher: + def fetch_data(self): + return [FakeSourceData(), FakeSourceData()] + + self.data_fetcher_model = FakeDataFetcher + + self.signatures = [module.SignatureTypes.SIG_TYPE_PARENT_PROCESS_NAME] + + def test_source_good_init(self): + """ + testing the proper init of Source + """ + source = module.Source( + data_fetcher_model=self.data_fetcher_model, + source_data_model=self.source_data_model, + signatures=self.signatures, + ) + + self.assertEqual(source.data_fetcher_model, self.data_fetcher_model) + self.assertEqual(source.source_data_model, self.source_data_model) + self.assertEqual(source.signatures, self.signatures) + + def test_source_wrong_datafetcher_init(self): + """ + testing the failure of Source init + due to a data fetcher not following the data fetcher protocol + """ + + class WrongDataFetcher: + def dont_fetch_data(self): + pass + + with self.assertRaises(ValidationError): + module.Source( + data_fetcher_model=WrongDataFetcher(), + source_data_model=self.source_data_model, + signatures=self.signatures, + ) + + def test_source_wrong_sourcedata_init(self): + """ + testing the failure of Source init + due to a source data not following the source data protocol + """ + + class WrongSourceData: + def is_prevented(self): + return False + + def is_detected(self): + return True + + def __str__(self): + return "" + + with self.assertRaises(ValidationError): + module.Source( + data_fetcher_model=self.data_fetcher_model, + source_data_model=WrongSourceData(), + signatures=self.signatures, + ) + + def test_source_wrong_signatures_init(self): + """ + testing the failure of Source init + due to signatures having the wrong type + """ + + wrong_signatures = ["one", "two"] + + with self.assertRaises(ValidationError): + module.Source( + data_fetcher_model=self.data_fetcher_model, + source_data_model=self.source_data_model, + signatures=wrong_signatures, + ) + + +class SourceHandlerTest(unittest.TestCase): + def test_get_source_data(self): + """ + assert the calls made to data fetcher by source handler + for the get_source_data function + """ + data_fetcher = MagicMock() + + module.SourceHandler().get_source_data(data_fetcher) + + data_fetcher.fetch_data.assert_called_once() + + def test_serialize_as_oaevdata(self): + """ + assert the calls made to source data by source handler + for the serialize_as_oaevdata function + """ + data = MagicMock() + + module.SourceHandler().serialize_as_oaevdata(data) + + data.to_oaev_data.assert_called_once() + + def test_get_expectation_signature_groups(self): + """ + testing the data manipulation made by source handler + in order to produce signature groups + through the get_expectation_signature_groups function + """ + value = "my_type" + signature = MagicMock(value=value) + signatures = [signature] + _type = MagicMock(value=value) + inject_expectation_signature_1 = MagicMock(type=_type, value="my_value") + inject_expectation_signature_2 = MagicMock(type=_type, value="my_other_value") + end_date_type = MagicMock(value="end_date") + end_date_ies = MagicMock(type=end_date_type, value="now") + expectation = MagicMock( + inject_expectation_signatures=[ + inject_expectation_signature_1, + inject_expectation_signature_2, + end_date_ies, + ] + ) + + signature_groups = module.SourceHandler().get_expectation_signature_groups( + signatures, expectation + ) + + self.assertEqual(1, len(signature_groups)) + self.assertEqual(2, len(signature_groups.get("my_type"))) + self.assertIn( + {"type": "my_type", "value": "my_value"}, signature_groups.get("my_type") + ) + self.assertIn( + {"type": "my_type", "value": "my_other_value"}, + signature_groups.get("my_type"), + ) + + def test_match_signature_groups_and_oaevdata_success(self): + """ + testing the calls made to oaev detection helper by the source handler + for a successful matching + """ + signature_groups = {"my_type": [{"type": "my_type", "value": "my_value"}]} + oaev_data = MagicMock() + oaev_detection_helper = MagicMock() + oaev_detection_helper.match_alert_elements.return_value = True + + flag = module.SourceHandler().match_signature_groups_and_oaevdata( + signature_groups, oaev_data, oaev_detection_helper + ) + + oaev_detection_helper.match_alert_elements.assert_called_with( + signature_groups["my_type"], {"my_type": oaev_data.sig_type} + ) + self.assertTrue(flag) + + def test_match_signature_groups_and_oaevdata_failure(self): + """ + testing the calls made to oaev detection helper by the source handler + for a failed matching + """ + signature_groups = {"my_type": [{"type": "my_type", "value": "my_value"}]} + oaev_data = MagicMock() + oaev_detection_helper = MagicMock() + oaev_detection_helper.match_alert_elements.return_value = False + + flag = module.SourceHandler().match_signature_groups_and_oaevdata( + signature_groups, oaev_data, oaev_detection_helper + ) + + oaev_detection_helper.match_alert_elements.assert_called_with( + signature_groups["my_type"], {"my_type": oaev_data.sig_type} + ) + self.assertFalse(flag) + + def test_match_signature_groups_and_oaevdata_empty(self): + """ + testing the calls made to oaev detection helper by the source handler + for an empty input + """ + signature_groups = {"my_type": [{"type": "my_type", "value": "my_value"}]} + oaev_data = None + oaev_detection_helper = MagicMock() + oaev_detection_helper.match_alert_elements.return_value = True + + flag = module.SourceHandler().match_signature_groups_and_oaevdata( + signature_groups, oaev_data, oaev_detection_helper + ) + + oaev_detection_helper.match_alert_elements.assert_not_called() + self.assertFalse(flag) + + def test_serialize_as_tracedata(self): + """ + assert the calls made to source data by source handler + for the serialize_as_tracedata function + """ + data = MagicMock() + + module.SourceHandler().serialize_as_tracedata(data) + + data.to_traces_data.assert_called_once() + data.to_traces_data.return_value.model_dump.assert_called_once() + + def test_match_expectation_and_sourcedata_prevention_prevented(self): + """ + testing a prevented PreventionExpectation + """ + expectation = MagicMock(spec=module.PreventionExpectation) + data = MagicMock() + data.is_prevented.return_value = True + data.is_detected.return_value = True + + matchflag, breakflag = module.SourceHandler().match_expectation_and_sourcedata( + expectation, data + ) + + data.is_prevented.assert_called_once() + data.is_detected.assert_not_called() + self.assertTrue(matchflag) + self.assertTrue(breakflag) + + def test_match_expectation_and_sourcedata_prevention_not_prevented(self): + """ + testing a non-prevented PreventionExpectation + """ + expectation = MagicMock(spec=module.PreventionExpectation) + data = MagicMock() + data.is_prevented.return_value = False + data.is_detected.return_value = True + + matchflag, breakflag = module.SourceHandler().match_expectation_and_sourcedata( + expectation, data + ) + + data.is_prevented.assert_called_once() + data.is_detected.assert_not_called() + self.assertFalse(matchflag) + self.assertFalse(breakflag) + + def test_match_expectation_and_sourcedata_detection_detected(self): + """ + testing a detected DetectionExpectation + """ + expectation = MagicMock(spec=module.DetectionExpectation) + data = MagicMock() + data.is_prevented.return_value = True + data.is_detected.return_value = True + + matchflag, breakflag = module.SourceHandler().match_expectation_and_sourcedata( + expectation, data + ) + + data.is_prevented.assert_not_called() + data.is_detected.assert_called_once() + self.assertTrue(matchflag) + self.assertFalse(breakflag) + + def test_match_expectation_and_sourcedata_detection_not_detected(self): + """ + testing a non-detected DetectionExpectation + """ + expectation = MagicMock(spec=module.DetectionExpectation) + data = MagicMock() + data.is_prevented.return_value = True + data.is_detected.return_value = False + + matchflag, breakflag = module.SourceHandler().match_expectation_and_sourcedata( + expectation, data + ) + + data.is_prevented.assert_not_called() + data.is_detected.assert_called_once() + self.assertFalse(matchflag) + self.assertFalse(breakflag) From cf38d6c59c427a7dbc6a56c72d9d8c08616bd4b4 Mon Sep 17 00:00:00 2001 From: guzmud Date: Thu, 7 May 2026 11:54:35 +0200 Subject: [PATCH 10/25] [template] feat(uploaders): adding resilient uploaders to internals --- template/src/collector/internals/__init__.py | 0 .../src/collector/internals/oaev_uploaders.py | 149 +++++++++++++ .../collector/internals/resilient_uploader.py | 156 +++++++++++++ .../tests/collector/internals/__init__.py | 0 .../internals/test_oaev_uploaders.py | 209 ++++++++++++++++++ .../internals/test_resilient_uploader.py | 182 +++++++++++++++ 6 files changed, 696 insertions(+) create mode 100644 template/src/collector/internals/__init__.py create mode 100644 template/src/collector/internals/oaev_uploaders.py create mode 100644 template/src/collector/internals/resilient_uploader.py create mode 100644 template/tests/collector/internals/__init__.py create mode 100644 template/tests/collector/internals/test_oaev_uploaders.py create mode 100644 template/tests/collector/internals/test_resilient_uploader.py diff --git a/template/src/collector/internals/__init__.py b/template/src/collector/internals/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/src/collector/internals/oaev_uploaders.py b/template/src/collector/internals/oaev_uploaders.py new file mode 100644 index 00000000..b798c244 --- /dev/null +++ b/template/src/collector/internals/oaev_uploaders.py @@ -0,0 +1,149 @@ +"""Expectation uploader and expectation trace uploader based on the ResilientUploader object""" + +from typing import Any, Iterable + +from pyoaev.client import OpenAEV +from src.collector.internals.resilient_uploader import ResilientUploader +from src.collector.models.expectations import ExpectationTrace + +LOG_PREFIX = "[Uploader]" + + +class ExpectationUploader(ResilientUploader): + """ResilientUploader-based expectation uploader using the OpenAEV API""" + + def __init__(self, oaev_api: OpenAEV, collector_id: str): + self.oaev_api = oaev_api + self.collector_id = collector_id + super().__init__( + data_name="expectation", + _prepare_bulk_data=self.expectation_prepare_bulk_data, + _bulk_upload=self.expectation_bulk_upload, + _unpack_bulk_data=self.expectation_unpack_bulk_data, + _individual_upload=self.expectation_individual_upload, + ) + + def expectation_prepare_bulk_data( + self, results: list[Any] + ) -> tuple[Iterable[Any], int]: + """ + convert a list of results into the required format + for API's bulk_update later down the road + """ + bulk_data = {} + skipped_count = 0 + for result in results: + try: + # skipping result without expectation_id + if not result.expectation_id: + skipped_count += 1 + self.logger.debug( + f"{LOG_PREFIX} Skipping result without expectation_id" + ) + continue + + # skipping result without expectation + if not result.expectation: + skipped_count += 1 + self.logger.debug( + f"{LOG_PREFIX} Skipping result {result.expectation_id} " + f"without expectation object" + ) + continue + + bulk_data[result.expectation_id] = { + "collector_id": self.collector_id, + "result": result.to_result_text(), + "is_success": result.is_valid, + } + except Exception as err: + self.logger.debug(f"{LOG_PREFIX} Skipping result due to error: {err}") + skipped_count += 1 + + return bulk_data, skipped_count + + def expectation_bulk_upload(self, bulk_data: Iterable[Any]) -> None: + """expectation bulk update using the OpenAEV API""" + self.oaev_api.inject_expectation.bulk_update( + inject_expectation_input_by=bulk_data + ) + + def expectation_unpack_bulk_data( + self, bulk_data: Iterable[Any] + ) -> Iterable[tuple[str, Any]]: + """unpack the default expectation bulk data format into a (index,data) iterable""" + return bulk_data.items() + + def expectation_individual_upload( + self, expectation_id: str, expectation_data: Any + ) -> None: + """expectation single update using the OpenAEV API""" + self.oaev_api.inject_expectation.update( + inject_expectation_id=expectation_id, + inject_expectation=expectation_data, + ) + + +class TraceUploader(ResilientUploader): + """ResilientUploader-based expectation trace uploader using the OpenAEV API""" + + def __init__(self, oaev_api: OpenAEV, collector_id: str, collector_name: str): + self.oaev_api = oaev_api + self.collector_id = collector_id + self.collector_name = collector_name + super().__init__( + data_name="trace", + _prepare_bulk_data=self.trace_prepare_bulk_data, + _bulk_upload=self.trace_bulk_upload, + _unpack_bulk_data=self.trace_unpack_bulk_data, + _individual_upload=self.trace_individual_upload, + ) + + def trace_prepare_bulk_data(self, results: list[Any]) -> tuple[Iterable[Any], int]: + """ + convert a list of results into the required format + for API's bulk_create later down the road + """ + valid_results = [ + result for result in results if result.is_valid and result.matched_alerts + ] + if not valid_results: + return [], len(results) + + traces = [] + skipped_count = 0 + for result in results: + try: + # skipping result without expectation_id + if not result.expectation_id: + self.logger.debug( + f"{LOG_PREFIX} Skipping result without expectation_id" + ) + skipped_count += 1 + continue + + trace = ExpectationTrace.from_result( + result, self.collector_id, self.collector_name + ) + traces.append(trace) + except Exception as err: + self.logger.debug(f"{LOG_PREFIX} Skipping result due to error: {err}") + skipped_count += 1 + + return traces, skipped_count + + def trace_bulk_upload(self, traces: Iterable[Any]) -> None: + """expectation trace bulk upload using the OpenAEV API""" + self.oaev_api.inject_expectation_trace.bulk_create( + payload={"expectation_traces": [trace.to_api_dict() for trace in traces]} + ) + + def trace_unpack_bulk_data( + self, traces: Iterable[Any] + ) -> Iterable[tuple[int, Any]]: + """unpack the default expectation trace bulk data format into a (index,data) iterable""" + return enumerate(traces, 1) + + def trace_individual_upload(self, _, trace: Any) -> None: + """expectation trace single upload using the OpenAEV API""" + self.oaev_api.inject_expectation_trace.create(trace.to_api_dict()) diff --git a/template/src/collector/internals/resilient_uploader.py b/template/src/collector/internals/resilient_uploader.py new file mode 100644 index 00000000..5b8d27df --- /dev/null +++ b/template/src/collector/internals/resilient_uploader.py @@ -0,0 +1,156 @@ +""" +A generic class for resilient upload meant to be inherited +and injected with appropriate functions for packing the data, +uploading the packed data, unpacking it and uploading the +unpacked data +""" + +import logging +from typing import Any + +from src.collector.models.exception import ( + APIError, + BulkPreparationError, + BulkUploadError, +) +from src.collector.types.internals import ( + BulkUploadFunction, + IndividualUploadFunction, + PrepareBulkFunction, + UnpackBulkFunction, +) + +LOG_PREFIX = "[ResilientUploader]" + + +class ResilientUploader: + """ + A generic bulk uploader with a fallback method for single upload + """ + + def __init__( + self, + data_name: str, + _prepare_bulk_data: PrepareBulkFunction, + _bulk_upload: BulkUploadFunction, + _unpack_bulk_data: UnpackBulkFunction, + _individual_upload: IndividualUploadFunction, + ): + self.logger = logging.getLogger(__name__) + self.data_name = data_name + self._prepare_bulk_data = _prepare_bulk_data + self._bulk_upload = _bulk_upload + self._unpack_bulk_data = _unpack_bulk_data + self._individual_upload = _individual_upload + + def prepare_bulk_data(self, data: list[Any]) -> list[Any]: + """ + Using the provided function, prepare the data for bulk upload + """ + try: + bulk_data, skipped_count = self._prepare_bulk_data(data) + except Exception as err: + self.logger.error( + f"{LOG_PREFIX} Failure during bulk data preparation: {err}" + ) + raise BulkPreparationError( + f"Error in bulk {self.data_name} preparation: {err}" + ) from err + + if skipped_count > 0: + self.logger.debug( + f"{LOG_PREFIX} Skipped {skipped_count} input data " + f"during bulk {self.data_name} preparation" + ) + + return bulk_data + + def bulk_upload_data(self, bulk_data: list[Any]) -> None: + """ + Using the provided functions, attempt to bulk upload + and on failure, unpack the bulk data and attempt to + upload one data at a time + """ + try: + # try the bulk upload endpoint + self.logger.debug(f"{LOG_PREFIX} Attempting bulk upload...") + self._bulk_upload(bulk_data) + self.logger.info( + f"{LOG_PREFIX} Successfully bulk upload {len(bulk_data)} {self.data_name}" + ) + except Exception as bulk_error: + # bulk update endpoint failed + self.logger.warning( + f"{LOG_PREFIX} Bulk upload failed, falling back to individual updates: {bulk_error}" + ) + try: + # try the single upload endpoint + success_count = 0 + error_count = 0 + for index, data in self._unpack_bulk_data(bulk_data): + try: + self.logger.debug( + f"{LOG_PREFIX} Uploading individual " + f"{self.data_name} index {index}" + ) + self._individual_upload(index, data) + self.logger.debug( + f"{LOG_PREFIX} Successfully uploaded " + f"{self.data_name} index {index}" + ) + success_count += 1 + except APIError as api_err: + error_count += 1 + self.logger.error( + f"{LOG_PREFIX} Failed to update " + f"{self.data_name} index {index}: {api_err}" + ) + except Exception as err: + error_count += 1 + self.logger.error( + f"{LOG_PREFIX} Unexpected error updating " + f"{self.data_name} index {index}: {err}" + ) + self.logger.info( + f"{LOG_PREFIX} Individual uploads completed: " + f"{success_count} successful, {error_count} failed" + ) + except Exception as fallback_error: + # neither endpoint worked + raise BulkUploadError( + f"Both bulk and individual uploads failed: {fallback_error}" + ) from fallback_error + + def upload_data(self, data: list[Any]) -> None: + """ + Package the data for bulk upload then attempt to bulk upload it + """ + if not data: + self.logger.debug( + f"{LOG_PREFIX} No {self.data_name} to upload, skipping prepare and submit" + ) + return + + try: + self.logger.debug( + f"{LOG_PREFIX} Preparing bulk {self.data_name}: {len(data)} elements to process..." + ) + bulk_data = self.prepare_bulk_data(data) + + if not bulk_data: + self.logger.debug( + f"{LOG_PREFIX} No bulk {self.data_name} produces from input data" + ) + return + + self.logger.debug( + f"{LOG_PREFIX} Attempting bulk upload of {len(bulk_data)} {self.data_name}..." + ) + self.bulk_upload_data(bulk_data) + except Exception as err: + self.logger.error( + f"{LOG_PREFIX} Bulk {self.data_name} upload failed: {err}" + ) + raise BulkUploadError( + f"Error in bulk {self.data_name} upload: {err}" + ) from err diff --git a/template/tests/collector/internals/__init__.py b/template/tests/collector/internals/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/tests/collector/internals/test_oaev_uploaders.py b/template/tests/collector/internals/test_oaev_uploaders.py new file mode 100644 index 00000000..d41ff8b0 --- /dev/null +++ b/template/tests/collector/internals/test_oaev_uploaders.py @@ -0,0 +1,209 @@ +import unittest +from unittest.mock import MagicMock, patch + +import src.collector.internals.oaev_uploaders as module + + +class TestExpectationUploader(unittest.TestCase): + def setUp(self): + self.oaev_api = MagicMock() + self.collector_id = "1234abcd" + + self.expectation_uploader = module.ExpectationUploader( + oaev_api=self.oaev_api, collector_id=self.collector_id + ) + + def test_expectation_uploader_init(self): + self.assertEqual(self.expectation_uploader.oaev_api, self.oaev_api) + self.assertEqual(self.expectation_uploader.collector_id, self.collector_id) + self.assertEqual(self.expectation_uploader.data_name, "expectation") + + def test_expectation_uploader_expectation_prepare_bulk_data(self): + result1 = MagicMock() + result1.expectation_id = "1" + result1.expectation = MagicMock() + result2 = MagicMock() + result2.expectation_id = "2" + result2.expectation = MagicMock() + results = [result1, result2] + + bulked_data, skipped_count = ( + self.expectation_uploader.expectation_prepare_bulk_data(results) + ) + + self.assertEqual( + bulked_data, + { + "1": { + "collector_id": self.expectation_uploader.collector_id, + "result": result1.to_result_text.return_value, + "is_success": result1.is_valid, + }, + "2": { + "collector_id": self.expectation_uploader.collector_id, + "result": result2.to_result_text.return_value, + "is_success": result2.is_valid, + }, + }, + ) + self.assertEqual(skipped_count, 0) + + def test_expectation_uploader_expectation_prepare_bulk_data_one_skipped(self): + result1 = MagicMock() + result1.expectation_id = "1" + result1.expectation = None + result2 = MagicMock() + result2.expectation_id = "2" + result2.expectation = MagicMock() + results = [result1, result2] + + bulked_data, skipped_count = ( + self.expectation_uploader.expectation_prepare_bulk_data(results) + ) + + self.assertEqual( + bulked_data, + { + "2": { + "collector_id": self.expectation_uploader.collector_id, + "result": result2.to_result_text.return_value, + "is_success": result2.is_valid, + }, + }, + ) + self.assertEqual(skipped_count, 1) + + def test_expectation_uploader_expectation_bulk_upload(self): + bulk_data = [MagicMock(), MagicMock(), MagicMock()] + + self.expectation_uploader.expectation_bulk_upload(bulk_data) + + self.oaev_api.inject_expectation.bulk_update.assert_called_once_with( + inject_expectation_input_by=bulk_data + ) + + def test_expectation_uploader_expectation_unpack_bulk_data(self): + bulk_data = {"one": 1, "two": 2, "three": 3} + + unpacked_data = self.expectation_uploader.expectation_unpack_bulk_data( + bulk_data + ) + + self.assertEqual(list(unpacked_data), [("one", 1), ("two", 2), ("three", 3)]) + + def test_expectation_uploader_expectation_individual_upload(self): + expectation_id = 42 + expectation_data = MagicMock() + + self.expectation_uploader.expectation_individual_upload( + expectation_id, expectation_data + ) + + self.oaev_api.inject_expectation.update.assert_called_once_with( + inject_expectation_id=expectation_id, + inject_expectation=expectation_data, + ) + + +class TestTraceUploader(unittest.TestCase): + def setUp(self): + self.oaev_api = MagicMock() + self.collector_id = "1234abcd" + self.collector_name = "my name is" + + self.trace_uploader = module.TraceUploader( + oaev_api=self.oaev_api, + collector_id=self.collector_id, + collector_name=self.collector_name, + ) + + def test_trace_uploader_init(self): + self.assertEqual(self.trace_uploader.oaev_api, self.oaev_api) + self.assertEqual(self.trace_uploader.collector_id, self.collector_id) + self.assertEqual(self.trace_uploader.collector_name, self.collector_name) + self.assertEqual(self.trace_uploader.data_name, "trace") + + @patch.object(module, "ExpectationTrace") + def test_trace_uploader_trace_prepare_bulk_data(self, m_expectationtrace): + result1 = MagicMock() + result1.is_valid = True + result1.matched_alerts = [MagicMock()] + result1.expectation_id = "1" + result2 = MagicMock() + result2.is_valid = True + result2.matched_alerts = [MagicMock()] + result2.expectation_id = "2" + results = [result1, result2] + exptrace1 = MagicMock() + exptrace2 = MagicMock() + m_expectationtrace.from_result.side_effect = [exptrace1, exptrace2] + + traces, skipped_count = self.trace_uploader.trace_prepare_bulk_data(results) + + m_expectationtrace.from_result.assert_any_call( + result1, + self.trace_uploader.collector_id, + self.trace_uploader.collector_name, + ) + m_expectationtrace.from_result.assert_any_call( + result2, + self.trace_uploader.collector_id, + self.trace_uploader.collector_name, + ) + self.assertEqual(traces, [exptrace1, exptrace2]) + self.assertEqual(skipped_count, 0) + + @patch.object(module, "ExpectationTrace") + def test_trace_uploader_trace_prepare_bulk_data_one_skipped( + self, m_expectationtrace + ): + result1 = MagicMock() + result1.is_valid = True + result1.matched_alerts = [MagicMock()] + result1.expectation_id = "1" + result2 = MagicMock() + result2.is_valid = True + result2.matched_alerts = [MagicMock()] + result2.expectation_id = None + results = [result1, result2] + exptrace1 = MagicMock() + m_expectationtrace.from_result.return_value = exptrace1 + + traces, skipped_count = self.trace_uploader.trace_prepare_bulk_data(results) + + m_expectationtrace.from_result.assert_called_once_with( + result1, + self.trace_uploader.collector_id, + self.trace_uploader.collector_name, + ) + self.assertEqual(traces, [exptrace1]) + self.assertEqual(skipped_count, 1) + + def test_trace_uploader_trace_trace_bulk_upload(self): + trace1 = MagicMock() + trace2 = MagicMock() + traces = [trace1, trace2] + self.trace_uploader.trace_bulk_upload(traces) + + self.oaev_api.inject_expectation_trace.bulk_create.assert_called_once_with( + payload={ + "expectation_traces": [ + trace1.to_api_dict.return_value, + trace2.to_api_dict.return_value, + ] + } + ) + + def test_trace_uploader_trace_unpack_bulk_data(self): + traces = ["one", "two", "three"] + unpacked_data = self.trace_uploader.trace_unpack_bulk_data(traces) + + self.assertEqual(list(unpacked_data), [(1, "one"), (2, "two"), (3, "three")]) + + def test_trace_uploader_trace_trace_individual_upload(self): + trace = MagicMock() + self.trace_uploader.trace_individual_upload(None, trace) + + self.oaev_api.inject_expectation_trace.create.assert_called_with( + trace.to_api_dict.return_value + ) diff --git a/template/tests/collector/internals/test_resilient_uploader.py b/template/tests/collector/internals/test_resilient_uploader.py new file mode 100644 index 00000000..18027d9a --- /dev/null +++ b/template/tests/collector/internals/test_resilient_uploader.py @@ -0,0 +1,182 @@ +import unittest +from unittest.mock import MagicMock, patch + +import src.collector.internals.resilient_uploader as module + + +class TestResilientUploader(unittest.TestCase): + def test_resilient_uploader_init(self): + data_name = "testitest" + _prepare_bulk_data = MagicMock() + _bulk_upload = MagicMock() + _unpack_bulk_data = MagicMock() + _individual_upload = MagicMock() + + resilient_uploader = module.ResilientUploader( + data_name=data_name, + _prepare_bulk_data=_prepare_bulk_data, + _bulk_upload=_bulk_upload, + _unpack_bulk_data=_unpack_bulk_data, + _individual_upload=_individual_upload, + ) + + self.assertEqual(resilient_uploader.data_name, data_name) + self.assertEqual(resilient_uploader._prepare_bulk_data, _prepare_bulk_data) + self.assertEqual(resilient_uploader._bulk_upload, _bulk_upload) + self.assertEqual(resilient_uploader._unpack_bulk_data, _unpack_bulk_data) + self.assertEqual(resilient_uploader._individual_upload, _individual_upload) + + def test_resilient_uploader_prepare_bulk_data(self): + data = [MagicMock(), MagicMock()] + bulked_data = [MagicMock()] + data_name = "testitest" + _prepare_bulk_data = MagicMock() + _prepare_bulk_data.return_value = (bulked_data, 1) + _bulk_upload = MagicMock() + _unpack_bulk_data = MagicMock() + _individual_upload = MagicMock() + + resilient_uploader = module.ResilientUploader( + data_name=data_name, + _prepare_bulk_data=_prepare_bulk_data, + _bulk_upload=_bulk_upload, + _unpack_bulk_data=_unpack_bulk_data, + _individual_upload=_individual_upload, + ) + + bulk_data = resilient_uploader.prepare_bulk_data(data) + + _prepare_bulk_data.assert_called_once_with(data) + self.assertEqual(bulk_data, bulked_data) + + def test_resilient_uploader_bulk_upload_data_bulk_OK(self): + bulk_data = [MagicMock(), MagicMock()] + data_name = "testitest" + _prepare_bulk_data = MagicMock() + _bulk_upload = MagicMock() + _unpack_bulk_data = MagicMock() + _individual_upload = MagicMock() + + resilient_uploader = module.ResilientUploader( + data_name=data_name, + _prepare_bulk_data=_prepare_bulk_data, + _bulk_upload=_bulk_upload, + _unpack_bulk_data=_unpack_bulk_data, + _individual_upload=_individual_upload, + ) + + resilient_uploader.bulk_upload_data(bulk_data) + + _bulk_upload.assert_called_once_with(bulk_data) + _unpack_bulk_data.assert_not_called() + _individual_upload.assert_not_called() + + def test_resilient_uploader_bulk_upload_data_individual_OK(self): + bdata1 = MagicMock() + bdata2 = MagicMock() + bulk_data = [bdata1, bdata2] + data_name = "testitest" + _prepare_bulk_data = MagicMock() + _bulk_upload = MagicMock() + _bulk_upload.side_effect = Exception() + _unpack_bulk_data = MagicMock() + _unpack_bulk_data.return_value = [(1, bdata1), (2, bdata2)] + _individual_upload = MagicMock() + + resilient_uploader = module.ResilientUploader( + data_name=data_name, + _prepare_bulk_data=_prepare_bulk_data, + _bulk_upload=_bulk_upload, + _unpack_bulk_data=_unpack_bulk_data, + _individual_upload=_individual_upload, + ) + + resilient_uploader.bulk_upload_data(bulk_data) + + _bulk_upload.assert_called_once_with(bulk_data) + _unpack_bulk_data.assert_called_once_with(bulk_data) + self.assertEqual(_individual_upload._mock_call_count, 2) + _individual_upload.assert_any_call(1, bdata1) + _individual_upload.assert_called_with(2, bdata2) + + def test_resilient_uploader_bulk_upload_data_unpack_failure(self): + bdata1 = MagicMock() + bdata2 = MagicMock() + bulk_data = [bdata1, bdata2] + data_name = "testitest" + _prepare_bulk_data = MagicMock() + _bulk_upload = MagicMock() + _bulk_upload.side_effect = Exception() + _unpack_bulk_data = MagicMock() + _unpack_bulk_data.side_effect = Exception() + _individual_upload = MagicMock() + + resilient_uploader = module.ResilientUploader( + data_name=data_name, + _prepare_bulk_data=_prepare_bulk_data, + _bulk_upload=_bulk_upload, + _unpack_bulk_data=_unpack_bulk_data, + _individual_upload=_individual_upload, + ) + + with self.assertRaises(Exception): + resilient_uploader.bulk_upload_data(bulk_data) + + _bulk_upload.assert_called_once_with(bulk_data) + _unpack_bulk_data.assert_called_once_with(bulk_data) + + @patch.object(module.ResilientUploader, "bulk_upload_data") + @patch.object(module.ResilientUploader, "prepare_bulk_data") + def test_resilient_uploader_upload_data( + self, m_prepare_bulk_data, m_bulk_upload_data + ): + data = [MagicMock(), MagicMock(), MagicMock(), MagicMock()] + bdata1 = MagicMock() + bdata2 = MagicMock() + bulk_data = [bdata1, bdata2] + m_prepare_bulk_data.return_value = bulk_data + data_name = "testitest" + _prepare_bulk_data = MagicMock() + _bulk_upload = MagicMock() + _unpack_bulk_data = MagicMock() + _individual_upload = MagicMock() + + resilient_uploader = module.ResilientUploader( + data_name=data_name, + _prepare_bulk_data=_prepare_bulk_data, + _bulk_upload=_bulk_upload, + _unpack_bulk_data=_unpack_bulk_data, + _individual_upload=_individual_upload, + ) + + resilient_uploader.upload_data(data) + + m_prepare_bulk_data.assert_called_once_with(data) + m_bulk_upload_data.assert_called_once_with(bulk_data) + + @patch.object(module.ResilientUploader, "bulk_upload_data") + @patch.object(module.ResilientUploader, "prepare_bulk_data") + def test_resilient_uploader_upload_data_no_bulk_data( + self, m_prepare_bulk_data, m_bulk_upload_data + ): + data = [MagicMock(), MagicMock(), MagicMock(), MagicMock()] + bulk_data = [] + m_prepare_bulk_data.return_value = bulk_data + data_name = "testitest" + _prepare_bulk_data = MagicMock() + _bulk_upload = MagicMock() + _unpack_bulk_data = MagicMock() + _individual_upload = MagicMock() + + resilient_uploader = module.ResilientUploader( + data_name=data_name, + _prepare_bulk_data=_prepare_bulk_data, + _bulk_upload=_bulk_upload, + _unpack_bulk_data=_unpack_bulk_data, + _individual_upload=_individual_upload, + ) + + resilient_uploader.upload_data(data) + + m_prepare_bulk_data.assert_called_once_with(data) + m_bulk_upload_data.assert_not_called() From 2839eef5585900121d6aeba491647d276106a803 Mon Sep 17 00:00:00 2001 From: guzmud Date: Thu, 7 May 2026 12:00:01 +0200 Subject: [PATCH 11/25] [template] feat(utils): adding a retroport of batched for py3.11 support --- template/src/collector/utils/__init__.py | 0 .../collector/utils/retroport_itertools.py | 22 +++++++ template/tests/collector/utils/__init__.py | 0 .../utils/test_retroport_itertools.py | 58 +++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 template/src/collector/utils/__init__.py create mode 100644 template/src/collector/utils/retroport_itertools.py create mode 100644 template/tests/collector/utils/__init__.py create mode 100644 template/tests/collector/utils/test_retroport_itertools.py diff --git a/template/src/collector/utils/__init__.py b/template/src/collector/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/src/collector/utils/retroport_itertools.py b/template/src/collector/utils/retroport_itertools.py new file mode 100644 index 00000000..2e3c152a --- /dev/null +++ b/template/src/collector/utils/retroport_itertools.py @@ -0,0 +1,22 @@ +import itertools +import sys + + +def _batched(iterable, size): + """ + pseudo-itertools.batched for python 3.11 + based on https://docs.python.org/3/library/itertools.html#itertools.batched + """ + if size < 1: + raise ValueError("size must be at least one") + iterator = iter(iterable) + while batch := tuple(itertools.islice(iterator, size)): + yield batch + + +def batched(iterable, size): + """providing support for itertools.batched in 3.11""" + if not (sys.version_info.major >= 3 and sys.version_info.minor >= 12): + return _batched(iterable, size) + + return itertools.batched(iterable, size) diff --git a/template/tests/collector/utils/__init__.py b/template/tests/collector/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/tests/collector/utils/test_retroport_itertools.py b/template/tests/collector/utils/test_retroport_itertools.py new file mode 100644 index 00000000..44321fd2 --- /dev/null +++ b/template/tests/collector/utils/test_retroport_itertools.py @@ -0,0 +1,58 @@ +import unittest +from unittest.mock import MagicMock, patch + +import src.collector.utils.retroport_itertools as module + + +class TestRetroportItertools(unittest.TestCase): + @patch.object(module, "_batched") + @patch.object(module, "itertools") + @patch.object(module, "sys") + def test_batched_with_itertools(self, m_sys, m_itertools, m_batched): + version_info = MagicMock(major=3, minor=13) + m_sys.version_info = version_info + + iterable = MagicMock() + size = MagicMock() + + module.batched(iterable, size) + + m_itertools.batched.assert_called_with(iterable, size) + m_batched.assert_not_called() + + @patch.object(module, "_batched") + @patch.object(module, "itertools") + @patch.object(module, "sys") + def test_batched_without_itertools(self, m_sys, m_itertools, m_batched): + version_info = MagicMock(major=3, minor=11) + m_sys.version_info = version_info + + iterable = MagicMock() + size = MagicMock() + + module.batched(iterable, size) + + m_batched.assert_called_with(iterable, size) + m_itertools.batched.assert_not_called() + + def test_batched(self): + one = MagicMock() + two = MagicMock() + three = MagicMock() + iterable = [one, two, three] + size = 2 + + batches = module._batched(iterable, size) + + batch = next(batches) + self.assertEqual(batch, (one, two)) + + batch = next(batches) + self.assertEqual(batch, (three,)) + + def test_batched_wrong_size(self): + iterable = [MagicMock(), MagicMock(), MagicMock()] + size = -2 + + with self.assertRaises(ValueError): + next(module._batched(iterable, size)) From 4c39c9f40faf2bcfb7af705397c5c30aff4b6211 Mon Sep 17 00:00:00 2001 From: guzmud Date: Thu, 7 May 2026 12:02:22 +0200 Subject: [PATCH 12/25] [template] feat(engine): adding the basic collector engine --- template/src/collector/engines/basic.py | 328 +++++++++++++ template/tests/collector/engines/__init__.py | 0 .../tests/collector/engines/test_basic.py | 457 ++++++++++++++++++ 3 files changed, 785 insertions(+) create mode 100644 template/src/collector/engines/basic.py create mode 100644 template/tests/collector/engines/__init__.py create mode 100644 template/tests/collector/engines/test_basic.py diff --git a/template/src/collector/engines/basic.py b/template/src/collector/engines/basic.py new file mode 100644 index 00000000..72bed29d --- /dev/null +++ b/template/src/collector/engines/basic.py @@ -0,0 +1,328 @@ +import logging +import os + +from pyoaev.apis.inject_expectation.model import ( # type: ignore[import-untyped] + DetectionExpectation, + PreventionExpectation, +) +from pyoaev.client import OpenAEV # type: ignore[import_untyped] +from pyoaev.helpers import OpenAEVDetectionHelper +from src.collector.internals.oaev_uploaders import ExpectationUploader, TraceUploader +from src.collector.models.exception import ( + CollectorEngineConfigError, + CollectorProcessingError, + ExpectationProcessingError, +) +from src.collector.models.expectations import ExpectationResult, ExpectationSummary +from src.collector.models.source import Source +from src.collector.protocols.source_handler import SourceHandlerProtocol +from src.collector.utils.retroport_itertools import batched + +LOG_PREFIX = "[BasicCollectorEngine]" + + +class BasicCollectorEngine: + """ + Collector engine to be attached to a CollectorDaemon-based base collector. + This collector is use-case agnostic and works with any source provided. + """ + + def __init__( + self, + name: str, + collector_id: str, + source: Source, + source_handler: SourceHandlerProtocol, + oaev_api: OpenAEV, + batching: bool = False, + ): + self.name = name + self.collector_id = collector_id + + if source and not isinstance(source, Source): + # TODO custom exception + logging + raise TypeError("Source provided is not of type Source") # TODO + self.source = source + + if source_handler and not isinstance(source_handler, SourceHandlerProtocol): + # TODO custom exception + logging + raise TypeError( + "Source handler provided does not follow source handler protocol" + ) # TODO + self.source_handler = source_handler + + if oaev_api and not isinstance(oaev_api, OpenAEV): + # TODO custom exception + logging + raise TypeError( + "Source handler provided does not follow source handler protocol" + ) # TODO + self.oaev_api = oaev_api + + self.batching = batching + + self.logger = logging.getLogger(__name__) + self.current_summary = ExpectationSummary() + self.oaev_detection_helper = None + self.expectation_uploader = None + self.trace_uploader = None + + self.configured = False + + @property + def data_fetcher_model(self): + return self.source.data_fetcher_model + + @property + def signatures(self): + return self.source.signatures + + def configure_engine(self, config, batching=False): + self.logger.info( + f"{LOG_PREFIX} Supported signatures: {[sig.value for sig in self.signatures]}" + ) + self.config = config + self.batching = batching + + self.oaev_detection_helper = OpenAEVDetectionHelper( + logger=self.logger, + relevant_signatures_types=self.signatures, + ) + self.expectation_uploader = ExpectationUploader( + oaev_api=self.oaev_api, + collector_id=self.collector_id, + ) + self.trace_uploader = TraceUploader( + oaev_api=self.oaev_api, + collector_id=self.collector_id, + collector_name=self.name, + ) + + self._reset_summary() + self.configured = True + + def _reset_summary(self) -> None: + self.current_summary = ExpectationSummary( + received=0, + supported=0, + processed=0, + valid=0, + total_processing_time=None, + ) + + def _filter_supported(self, expectations): + return [ + exp + for exp in expectations + if isinstance(exp, (DetectionExpectation, PreventionExpectation)) + ] + + def _fetch_expectations(self) -> list[DetectionExpectation | PreventionExpectation]: + """Fetch expectations from OpenAEV. + + Returns: + List of expectations. + + """ + self.logger.debug( + f"{LOG_PREFIX} Fetching expectations for collector: {self.collector_id}" + ) + try: + expectations = ( + self.oaev_api.inject_expectation.expectations_models_for_source( + source_id=self.collector_id + ) + ) + self.logger.debug( + f"{LOG_PREFIX} Fetched {len(expectations)} expectations, reversing order..." + ) + expectations = list(reversed(expectations)) + return expectations + except Exception as e: + self.logger.error(f"{LOG_PREFIX} Error fetching expectations: {e}") + return [] + + def fetch_and_filter_expectations(self): + """fetch expectations and filter out unsupported ones (wrong expectation types)""" + self.logger.debug(f"{LOG_PREFIX} Fetching expectations from OpenAEV...") + expectations = self._fetch_expectations() + self.current_summary.received = len(expectations) + expectations = self._filter_supported(expectations) + self.current_summary.supported = len(expectations) + + self.logger.info( + f"{LOG_PREFIX} Received {self.current_summary.received} expectations: " + f"{self.current_summary.supported} supported, " + f"{self.current_summary.unsupported} skipped" + ) + if self.current_summary.unsupported: + self.logger.debug( + f"{LOG_PREFIX} Skipped {self.current_summary.unsupported} " + f"unsupported expectation types" + ) + return expectations + + def _process_batch( + self, batch: list[DetectionExpectation | PreventionExpectation] + ) -> list[ExpectationResult]: + """ + Processing a single batch of expectations through the following steps: + 0. per expectation + 1. fetch the data using the data fetcher provided + 2. serialize each data as OAEVData + 3. group the expectation signatures per expectation + 4. check for a match between the grouped signatures (3.) with the OAEVData (2.) + 5. serialize each data as TraceData + 6. check for a match between the expectation (0.) and the data (1.) + 7. create the ExpectationResult using the previous match (6.) and the TraceData (5.) + then return a list of all the results produced from the batch + """ + batch_results = [] + try: + # (1) fetch data + self.logger.info( + f"{LOG_PREFIX} Fetching data providing " + f"data fetcher {self.data_fetcher_model} to source handler" + ) + data = self.source_handler.get_source_data(self.data_fetcher_model()) + + for expectation in batch: + matched = False + traces = [] + for element in data: + # (2) serialize data as oaevdata + oaev_data = self.source_handler.serialize_as_oaevdata(element) + + # (3) get the expectation signature groups + signature_groups = ( + self.source_handler.get_expectation_signature_groups( + self.signatures, expectation + ) + ) + + # (4) match signature (3) with oaevdata (2) + flag = self.source_handler.match_signature_groups_and_oaevdata( + signature_groups, + oaev_data, + self.oaev_detection_helper, + ) + if flag: + # (5) serialize data as tracedata + trace = self.source_handler.serialize_as_tracedata(element) + traces.append(trace) + + # (6) match expectation (0) with sourcedata (1) + matchflag, breakflag = ( + self.source_handler.match_expectation_and_sourcedata( + expectation, element + ) + ) + if matchflag: + matched = True + if breakflag: + break + + # (7) create results from step 6 + tracedata (5) + result = ExpectationResult( + expectation_id=str(expectation.inject_expectation_id), + is_valid=matched, + expectation=expectation, + matched_alerts=traces, + ) + batch_results.append(result) + self.logger.info( + f"{LOG_PREFIX} Batch processed: {len(batch_results)} results" + ) + except Exception as err: # per batch + batch_results = [ + ExpectationResult.from_error( + ExpectationProcessingError(f"Batch processing error: {err}"), + expectation, + ) + for expectation in batch + ] + self.logger.error(f"{LOG_PREFIX} Error processing batch: {err}") + + return batch_results + + def run_engine(self) -> None: + """Process the callback for expectation processing. + + Executes a single processing cycle, handling expectations through the + expectation manager and logging results. Handles keyboard interrupts + and system exits gracefully. + + Raises: + CollectorProcessingError: If processing cycle fails. + """ + if not self.configured: + raise CollectorEngineConfigError( + "The collector engine is being ran before being configured." + "Please, be sure to call configure_engine before calling run_engine." + ) + + self.logger.info( + f"{LOG_PREFIX} Current summary reset before the new processing cycle" + ) + self._reset_summary() + + try: + self.logger.info(f"{LOG_PREFIX} Starting processing cycle...") + + # (0) fetch and filter expectations + expectations = self.fetch_and_filter_expectations() + + if not expectations: + self.logger.warning(f"{LOG_PREFIX} No expectations found to process") + return + + results = [] + + batches = [ + expectations, + ] # default: single giant batch of expectations + if self.batching: + # using a retro-compatible batched + # instead of itertools.batched due to python 3.11 support + batches = batched(expectations, self.config.expectation_batch_size) + + for batch in batches: + self.logger.info( + f"{LOG_PREFIX} Processing a batch of expectations of size {len(batch)}" + ) + batch_results = self._process_batch(batch) + results.extend(batch_results) + + self.current_summary.processed = len(results) + self.current_summary.valid = sum(1 for result in results if result.is_valid) + + self.logger.info( + f"{LOG_PREFIX} New batch processing completed: " + f"{self.current_summary.valid} valid, " + f"{self.current_summary.invalid} invalid, " + f"{self.current_summary.unprocessed} skipped" + ) + + # upload expectations using results + self.logger.debug(f"{LOG_PREFIX} Updating expectations in OpenAEV...") + self.expectation_uploader.upload_data(results) + + # upload expectation traces using results + self.logger.debug(f"{LOG_PREFIX} Creating and submitting traces...") + self.trace_uploader.upload_data(results) + + except (KeyboardInterrupt, SystemExit): # per batch processing cycle + self.logger.info(f"{LOG_PREFIX} Collector stopping...") + self.logger.info( + f"{LOG_PREFIX} Current summary info: {self.current_summary}" + ) + os._exit(0) + except Exception as e: # per batch processing cycle + self.logger.error(f"{LOG_PREFIX} Error during processing cycle: {str(e)}") + self.logger.info( + f"{LOG_PREFIX} Current summary info: {self.current_summary}" + ) + raise CollectorProcessingError(f"Processing error: {str(e)}") from e + + self.logger.info( + f"{LOG_PREFIX} Processing cycle completed: {self.current_summary}" + ) diff --git a/template/tests/collector/engines/__init__.py b/template/tests/collector/engines/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/tests/collector/engines/test_basic.py b/template/tests/collector/engines/test_basic.py new file mode 100644 index 00000000..77488f03 --- /dev/null +++ b/template/tests/collector/engines/test_basic.py @@ -0,0 +1,457 @@ +import unittest +from unittest.mock import MagicMock, patch + +import src.collector.engines.basic as module +from src.collector.models.source import SourceHandler + + +class TestBasicCollectorEngine(unittest.TestCase): + def test_minimal_init(self): + """""" + name = "my name is" + collector_id = "1234abcd" + data_fetcher_model = MagicMock() + signatures = MagicMock() + source = MagicMock(spec=module.Source) + source.data_fetcher_model = data_fetcher_model + source.signatures = signatures + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock(spec_set=module.OpenAEV) + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + self.assertEqual(collector_engine.name, name) + self.assertEqual(collector_engine.collector_id, collector_id) + self.assertEqual(collector_engine.source, source) + self.assertEqual(collector_engine.source_handler, source_handler) + self.assertEqual(collector_engine.oaev_api, oaev_api) + self.assertFalse(collector_engine.batching) + self.assertFalse(collector_engine.configured) + self.assertIsNotNone(collector_engine.logger) + self.assertIsNotNone(collector_engine.current_summary) + self.assertIsNone(collector_engine.oaev_detection_helper) + self.assertIsNone(collector_engine.expectation_uploader) + self.assertIsNone(collector_engine.trace_uploader) + self.assertEqual(collector_engine.data_fetcher_model, source.data_fetcher_model) + self.assertEqual(collector_engine.signatures, source.signatures) + + def test_full_init(self): + """""" + name = "my name is" + collector_id = "1234abcd" + data_fetcher_model = MagicMock() + signatures = MagicMock() + source = MagicMock(spec=module.Source) + source.data_fetcher_model = data_fetcher_model + source.signatures = signatures + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock(spec_set=module.OpenAEV) + batching = True + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + batching=batching, + ) + + self.assertEqual(collector_engine.name, name) + self.assertEqual(collector_engine.collector_id, collector_id) + self.assertEqual(collector_engine.source, source) + self.assertEqual(collector_engine.source_handler, source_handler) + self.assertEqual(collector_engine.oaev_api, oaev_api) + self.assertTrue(collector_engine.batching) + self.assertFalse(collector_engine.configured) + self.assertIsNotNone(collector_engine.logger) + self.assertIsNotNone(collector_engine.current_summary) + self.assertIsNone(collector_engine.oaev_detection_helper) + self.assertIsNone(collector_engine.expectation_uploader) + self.assertIsNone(collector_engine.trace_uploader) + + def test_wrong_source_init(self): + """""" + name = "my name is" + collector_id = "1234abcd" + source = MagicMock() + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock(spec_set=module.OpenAEV) + + with self.assertRaises(TypeError): + module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + def test_wrong_source_handler_init(self): + """""" + ... + name = "my name is" + collector_id = "1234abcd" + source = MagicMock(spec_set=module.Source) + source_handler = MagicMock() + oaev_api = MagicMock(spec_set=module.OpenAEV) + + with self.assertRaises(TypeError): + module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + def test_wrong_oaev_api_init(self): + """""" + ... + name = "my name is" + collector_id = "1234abcd" + source = MagicMock(spec_set=module.Source) + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock() + + with self.assertRaises(TypeError): + module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + @patch.object(module.BasicCollectorEngine, "_reset_summary") + def test_configure_engine(self, m_reset_summary): + """""" + ... + name = "my name is" + collector_id = "1234abcd" + source = MagicMock(spec=module.Source) + source.signatures = [MagicMock()] + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock(spec_set=module.OpenAEV) + config = MagicMock() + batching = True + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + collector_engine.configure_engine(config, batching) + + self.assertEqual(collector_engine.config, config) + self.assertTrue(collector_engine.batching) + self.assertIsNotNone(collector_engine.oaev_detection_helper) + self.assertIsNotNone(collector_engine.expectation_uploader) + self.assertIsNotNone(collector_engine.trace_uploader) + self.assertTrue(collector_engine.configured) + m_reset_summary.assert_called_once() + + def test_reset_summary(self): + """""" + name = "my name is" + collector_id = "1234abcd" + source = MagicMock(spec=module.Source) + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock(spec_set=module.OpenAEV) + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + collector_engine.current_summary.received = 50 + collector_engine.current_summary.supported = 30 + self.assertEqual(collector_engine.current_summary.received, 50) + self.assertEqual(collector_engine.current_summary.supported, 30) + collector_engine._reset_summary() + self.assertEqual(collector_engine.current_summary.received, 0) + self.assertEqual(collector_engine.current_summary.supported, 0) + + def test_filter_supported(self): + """""" + name = "my name is" + collector_id = "1234abcd" + source = MagicMock(spec=module.Source) + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock(spec_set=module.OpenAEV) + + expectations = [ + MagicMock(spec=module.DetectionExpectation), + MagicMock(spec=module.PreventionExpectation), + "expectation", + MagicMock(spec=module.PreventionExpectation), + ] + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + supported_expectations = collector_engine._filter_supported(expectations) + self.assertTrue(len(expectations) > len(supported_expectations)) + self.assertEqual(len(supported_expectations), 3) + + def test_fetch_expectations(self): + """""" + name = "my name is" + collector_id = "1234abcd" + source = MagicMock(spec=module.Source) + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock(spec=module.OpenAEV) + + api_expectations = [ + MagicMock(spec=module.DetectionExpectation), + MagicMock(spec=module.PreventionExpectation), + "expectation", + MagicMock(spec=module.PreventionExpectation), + ] + oaev_api.inject_expectation = MagicMock() + oaev_api.inject_expectation.expectations_models_for_source.return_value = ( + api_expectations + ) + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + expectations = collector_engine._fetch_expectations() + + oaev_api.inject_expectation.expectations_models_for_source.assert_called_with( + source_id=collector_id + ) + self.assertEqual(expectations, list(reversed(api_expectations))) + + def test_fetch_expectations_api_failure(self): + """""" + name = "my name is" + collector_id = "1234abcd" + source = MagicMock(spec=module.Source) + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock(spec=module.OpenAEV) + + oaev_api.inject_expectation = MagicMock() + oaev_api.inject_expectation.expectations_models_for_source.side_effect = ( + Exception() + ) + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + expectations = collector_engine._fetch_expectations() + + self.assertEqual(expectations, []) + oaev_api.inject_expectation.expectations_models_for_source.assert_called_with( + source_id=collector_id + ) + + @patch.object(module.BasicCollectorEngine, "_filter_supported") + @patch.object(module.BasicCollectorEngine, "_fetch_expectations") + def test_fetch_and_filter_expectations( + self, m_fetch_expectations, m_filter_supported + ): + """""" + name = "my name is" + collector_id = "1234abcd" + source = MagicMock(spec_set=module.Source) + source_handler = MagicMock(spec_set=SourceHandler) + oaev_api = MagicMock(spec_set=module.OpenAEV) + fetched_expectations = [ + MagicMock, + ] + m_fetch_expectations.return_value = fetched_expectations + supported_expectations = [ + MagicMock, + ] + m_filter_supported.return_value = supported_expectations + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + expectations = collector_engine.fetch_and_filter_expectations() + + m_fetch_expectations.assert_called_once() + m_filter_supported.assert_called_with(fetched_expectations) + self.assertEqual(expectations, supported_expectations) + self.assertEqual(collector_engine.current_summary.received, 1) + self.assertEqual(collector_engine.current_summary.supported, 1) + + @patch.object(module, "ExpectationResult") + @patch.object(module.BasicCollectorEngine, "_reset_summary") + def test_process_batch( + self, + m_reset_summary, + m_expectation_result, + ): + """""" + name = "my name is" + collector_id = "1234abcd" + signature_type = MagicMock(value="parent process name") + data_fetcher_model = MagicMock() + source = MagicMock(spec=module.Source) + source.signatures = [ + signature_type, + ] + source.data_fetcher_model = data_fetcher_model + source_handler = MagicMock(spec=SourceHandler) + data_element = MagicMock() + source_handler.get_source_data.return_value = [ + data_element, + ] + source_handler.match_expectation_and_sourcedata.side_effect = [ + [True, True], + [False, False], + ] + oaev_api = MagicMock(spec_set=module.OpenAEV) + + expectation1 = MagicMock() + expectation2 = MagicMock() + batch = [expectation1, expectation2] + result1 = MagicMock() + result2 = MagicMock() + m_expectation_result.side_effect = [result1, result2] + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + config = MagicMock() + collector_engine.configure_engine(config) + m_reset_summary.assert_called_once() + + batch_results = collector_engine._process_batch(batch) + + source_handler.get_source_data.assert_called_with(source.data_fetcher_model()) + self.assertEqual(source_handler.serialize_as_oaevdata._mock_call_count, 2) + source_handler.serialize_as_oaevdata.assert_called_with(data_element) + self.assertEqual( + source_handler.get_expectation_signature_groups._mock_call_count, 2 + ) + source_handler.get_expectation_signature_groups.assert_any_call( + source.signatures, expectation1 + ) + source_handler.get_expectation_signature_groups.assert_called_with( + source.signatures, expectation2 + ) + source_handler.match_signature_groups_and_oaevdata.assert_any_call( + source_handler.get_expectation_signature_groups.return_value, + source_handler.serialize_as_oaevdata.return_value, + collector_engine.oaev_detection_helper, + ) + source_handler.serialize_as_tracedata.assert_called_with(data_element) + self.assertEqual( + source_handler.match_expectation_and_sourcedata._mock_call_count, 2 + ) + source_handler.match_expectation_and_sourcedata.assert_any_call( + expectation1, data_element + ) + source_handler.match_expectation_and_sourcedata.assert_called_with( + expectation2, data_element + ) + + m_expectation_result.assert_any_call( + expectation_id=str(expectation1.inject_expectation_id), + is_valid=True, + expectation=expectation1, + matched_alerts=[source_handler.serialize_as_tracedata.return_value], + ) + m_expectation_result.assert_any_call( + expectation_id=str(expectation2.inject_expectation_id), + is_valid=False, + expectation=expectation2, + matched_alerts=[source_handler.serialize_as_tracedata.return_value], + ) + self.assertEqual(batch_results, [result1, result2]) + + @patch.object(module, "TraceUploader") + @patch.object(module, "ExpectationUploader") + @patch.object(module.BasicCollectorEngine, "_process_batch") + @patch.object(module.BasicCollectorEngine, "fetch_and_filter_expectations") + @patch.object(module.BasicCollectorEngine, "_reset_summary") + def test_run_engine( + self, + m_reset_summary, + m_fetch_and_filter_expectations, + m_process_batch, + m_expectation_uploader, + m_trace_uploader, + ): + """""" + name = "my name is" + collector_id = "1234abcd" + signature_type = MagicMock(value="parent process name") + data_fetcher_model = MagicMock() + source = MagicMock(spec=module.Source) + source.signatures = [ + signature_type, + ] + source.data_fetcher_model = data_fetcher_model + source_handler = MagicMock(spec=SourceHandler) + oaev_api = MagicMock(spec_set=module.OpenAEV) + + expectation1 = MagicMock() + expectation2 = MagicMock() + m_fetch_and_filter_expectations.return_value = [expectation1, expectation2] + result1 = MagicMock() + result2 = MagicMock() + m_process_batch.return_value = [result1, result2] + + collector_engine = module.BasicCollectorEngine( + name=name, + collector_id=collector_id, + source=source, + source_handler=source_handler, + oaev_api=oaev_api, + ) + + config = MagicMock() + collector_engine.configure_engine(config) + m_reset_summary.assert_called_once() + + collector_engine.run_engine() + + self.assertEqual(m_reset_summary._mock_call_count, 2) + m_reset_summary.assert_any_call() + + m_fetch_and_filter_expectations.assert_called_once() + m_process_batch.assert_called_once_with([expectation1, expectation2]) + m_expectation_uploader.return_value.upload_data.assert_any_call( + [result1, result2] + ) + m_trace_uploader.return_value.upload_data.assert_any_call([result1, result2]) From 1caa57c56c6c6ed354d17279f97e51724fdf9eb2 Mon Sep 17 00:00:00 2001 From: guzmud Date: Thu, 7 May 2026 12:05:46 +0200 Subject: [PATCH 13/25] [template] feat(collector): reworking the common collector elements --- template/src/__main__.py | 29 +- template/src/collector/__init__.py | 3 - .../src/collector/base_models/exception.py | 73 --- .../src/collector/base_models/expectations.py | 168 ------ template/src/collector/collector.py | 157 ++--- template/src/collector/expectation_handler.py | 204 ------- template/src/collector/expectation_manager.py | 426 ------------- .../collector/expectation_service_provider.py | 76 --- .../collector/helpers}/__init__.py | 0 template/src/collector/signature_registry.py | 159 ----- template/src/collector/trace_manager.py | 201 ------- .../src/collector/trace_service_provider.py | 24 - template/src/img/template-logo.png | 1 + template/src/models/settings/__init__.py | 6 +- .../src/models/settings/collector_configs.py | 2 +- template/src/models/settings/config_loader.py | 5 +- .../src/models/settings/template_configs.py | 2 +- template/src/services/converter.py | 103 ---- template/src/services/exception.py | 31 - template/src/services/expectation_service.py | 522 ---------------- template/src/services/fetcher_data.py | 69 --- template/src/services/model_data.py | 15 - template/src/services/trace_service.py | 198 ------ template/src/services/utils/__init__.py | 9 - template/src/services/utils/config_loader.py | 72 --- .../src/services/utils/signature_extractor.py | 127 ---- template/src/services/utils/trace_builder.py | 46 -- .../fixtures => src/source}/__init__.py | 0 template/src/source/template_data_fetcher.py | 11 + template/src/source/template_signatures.py | 7 + template/src/source/template_source_data.py | 35 ++ template/src/template_collector.py | 37 ++ template/tests/collector/__init__.py | 0 template/tests/collector/test_collector.py | 132 ++++ template/tests/conftest.py | 93 --- template/tests/gwt_shared.py | 564 ------------------ template/tests/services/conftest.py | 385 ------------ template/tests/services/fixtures/factories.py | 263 -------- template/tests/services/test_converter.py | 196 ------ .../services/test_expectation_service.py | 481 --------------- template/tests/services/test_fetcher_data.py | 168 ------ template/tests/services/test_trace_service.py | 235 -------- template/tests/test_create_collector.py | 188 ------ template/tests/test_template_collector.py | 32 + 44 files changed, 329 insertions(+), 5226 deletions(-) delete mode 100644 template/src/collector/base_models/exception.py delete mode 100644 template/src/collector/base_models/expectations.py delete mode 100644 template/src/collector/expectation_handler.py delete mode 100644 template/src/collector/expectation_manager.py delete mode 100644 template/src/collector/expectation_service_provider.py rename template/{tests/services => src/collector/helpers}/__init__.py (100%) delete mode 100644 template/src/collector/signature_registry.py delete mode 100644 template/src/collector/trace_manager.py delete mode 100644 template/src/collector/trace_service_provider.py create mode 100644 template/src/img/template-logo.png delete mode 100644 template/src/services/converter.py delete mode 100644 template/src/services/exception.py delete mode 100644 template/src/services/expectation_service.py delete mode 100644 template/src/services/fetcher_data.py delete mode 100644 template/src/services/model_data.py delete mode 100644 template/src/services/trace_service.py delete mode 100644 template/src/services/utils/__init__.py delete mode 100644 template/src/services/utils/config_loader.py delete mode 100644 template/src/services/utils/signature_extractor.py delete mode 100644 template/src/services/utils/trace_builder.py rename template/{tests/services/fixtures => src/source}/__init__.py (100%) create mode 100644 template/src/source/template_data_fetcher.py create mode 100644 template/src/source/template_signatures.py create mode 100644 template/src/source/template_source_data.py create mode 100644 template/src/template_collector.py create mode 100644 template/tests/collector/__init__.py create mode 100644 template/tests/collector/test_collector.py delete mode 100644 template/tests/conftest.py delete mode 100644 template/tests/gwt_shared.py delete mode 100644 template/tests/services/conftest.py delete mode 100644 template/tests/services/fixtures/factories.py delete mode 100644 template/tests/services/test_converter.py delete mode 100644 template/tests/services/test_expectation_service.py delete mode 100644 template/tests/services/test_fetcher_data.py delete mode 100644 template/tests/services/test_trace_service.py delete mode 100644 template/tests/test_create_collector.py create mode 100644 template/tests/test_template_collector.py diff --git a/template/src/__main__.py b/template/src/__main__.py index 29e5b88d..04f1cc1e 100644 --- a/template/src/__main__.py +++ b/template/src/__main__.py @@ -1,33 +1,6 @@ """Main entry point for the collector.""" -import logging -import os -import sys - -from src.collector import Collector -from src.collector.exception import CollectorConfigError - -LOG_PREFIX = "[Main]" - - -def main() -> None: - """Define the main function to run the collector.""" - logger = logging.getLogger(__name__) - - try: - logger.info(f"{LOG_PREFIX} Starting Template collector...") - collector = Collector() - collector.start() - except KeyboardInterrupt: - logger.info(f"{LOG_PREFIX} Collector stopped by user (Ctrl+C)") - os._exit(0) - except CollectorConfigError as e: - logger.error(f"{LOG_PREFIX} Configuration error: {e}") - sys.exit(2) - except Exception as e: - logger.exception(f"{LOG_PREFIX} Fatal error starting collector: {e}") - sys.exit(1) - +from src.template_collector import main if __name__ == "__main__": main() diff --git a/template/src/collector/__init__.py b/template/src/collector/__init__.py index 36918ee8..e69de29b 100644 --- a/template/src/collector/__init__.py +++ b/template/src/collector/__init__.py @@ -1,3 +0,0 @@ -from src.collector.collector import Collector - -__all__ = ["Collector"] diff --git a/template/src/collector/base_models/exception.py b/template/src/collector/base_models/exception.py deleted file mode 100644 index df900901..00000000 --- a/template/src/collector/base_models/exception.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Custom exceptions for the collector.""" - - -class CollectorError(Exception): - """Base exception for the collector.""" - - pass - - -class CollectorConfigError(CollectorError): - """Exception raised when there is an error in the collector configuration.""" - - pass - - -class CollectorSetupError(CollectorError): - """Exception raised when there is an error setting up the collector.""" - - pass - - -class CollectorProcessingError(CollectorError): - """Exception raised when there is an error processing data in the collector.""" - - pass - - -class ExpectationHandlerError(CollectorError): - """Exception raised when there is an error in expectation handling.""" - - pass - - -class ExpectationProcessingError(CollectorError): - """Exception raised when there is an error processing expectations.""" - - pass - - -class ExpectationUpdateError(CollectorError): - """Exception raised when there is an error updating expectations.""" - - pass - - -class BulkUpdateError(ExpectationUpdateError): - """Exception raised when there is an error during bulk update operations.""" - - pass - - -class APIError(CollectorError): - """Exception raised when there is an error with API operations.""" - - pass - - -class TracingError(CollectorError): - """Exception raised when there is an error with tracing operations.""" - - pass - - -class TraceSubmissionError(TracingError): - """Exception raised when there is an error submitting traces.""" - - pass - - -class TraceCreationError(TracingError): - """Exception raised when there is an error creating traces.""" - - pass diff --git a/template/src/collector/base_models/expectations.py b/template/src/collector/base_models/expectations.py deleted file mode 100644 index 86021fe8..00000000 --- a/template/src/collector/base_models/expectations.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Pydantic models for collector data structures.""" - -from typing import Any - -from pydantic import BaseModel, Field, field_validator - - -class ExpectationTrace(BaseModel): - """Pydantic model for expectation trace data. - - This model represents the structure of trace data that gets sent to the - OpenAEV API for expectation tracking and validation. - """ - - inject_expectation_trace_expectation: str = Field( - description="The expectation ID this trace is associated with" - ) - inject_expectation_trace_source_id: str = Field( - description="The collector/source ID that generated this trace" - ) - inject_expectation_trace_alert_name: str = Field( - description="Name of the alert that was matched" - ) - inject_expectation_trace_alert_link: str = Field( - description="Link to the alert in the source system" - ) - inject_expectation_trace_date: str = Field( - description="Date when the trace was created (ISO format string)" - ) - - @field_validator("inject_expectation_trace_expectation") - @classmethod - def expectation_must_not_be_empty(cls, v: str) -> str: - """Validate that expectation ID is not empty. - - Args: - v: The expectation ID value to validate. - - Returns: - The trimmed expectation ID. - - Raises: - ValueError: If the expectation ID is empty or whitespace only. - - """ - if not v or not v.strip(): - raise ValueError("Expectation ID cannot be empty") - return v.strip() - - @field_validator("inject_expectation_trace_source_id") - @classmethod - def source_id_must_not_be_empty(cls, v: str) -> str: - """Validate that source ID is not empty. - - Args: - v: The source ID value to validate. - - Returns: - The trimmed source ID. - - Raises: - ValueError: If the source ID is empty or whitespace only. - - """ - if not v or not v.strip(): - raise ValueError("Source ID cannot be empty") - return v.strip() - - @field_validator("inject_expectation_trace_alert_name") - @classmethod - def alert_name_must_not_be_empty(cls, v: str) -> str: - """Validate that alert name is not empty. - - Args: - v: The alert name value to validate. - - Returns: - The trimmed alert name. - - Raises: - ValueError: If the alert name is empty or whitespace only. - - """ - if not v or not v.strip(): - raise ValueError("Alert name cannot be empty") - return v.strip() - - @field_validator("inject_expectation_trace_alert_link") - @classmethod - def alert_link_must_not_be_empty(cls, v: str) -> str: - """Validate that alert link is not empty. - - Args: - v: The alert link value to validate. - - Returns: - The trimmed alert link. - - Raises: - ValueError: If the alert link is empty or whitespace only. - - """ - if not v or not v.strip(): - raise ValueError("Alert link cannot be empty") - return v.strip() - - @field_validator("inject_expectation_trace_date") - @classmethod - def date_must_not_be_empty(cls, v: str) -> str: - """Validate that date is not empty. - - Args: - v: The date value to validate. - - Returns: - The trimmed date string. - - Raises: - ValueError: If the date is empty or whitespace only. - - """ - if not v or not v.strip(): - raise ValueError("Trace date cannot be empty") - return v.strip() - - def to_api_dict(self) -> dict[str, str]: - """Convert the model to a dictionary suitable for API submission. - - This method ensures all values are strings as expected by the API, - replacing the manual sanitization logic in the expectation manager. - - Returns: - Dict with all values converted to strings. - - """ - return { - key: str(value) if value is not None else "" - for key, value in self.model_dump().items() - } - - -class ExpectationResult(BaseModel): - """Model for expectation processing results.""" - - expectation_id: str = Field(..., description="ID of the processed expectation") - is_valid: bool = Field(..., description="Whether the expectation was validated") - expectation: Any | None = Field(None, description="The original expectation object") - matched_alerts: list[dict[str, Any]] | None = Field( - None, description="List of alerts that matched this expectation" - ) - error_message: str | None = Field( - None, description="Error message if processing failed" - ) - processing_time: float | None = Field( - None, description="Time taken to process this expectation in seconds" - ) - - -class ProcessingSummary(BaseModel): - """Model for expectation processing summary.""" - - processed: int = Field(..., description="Total number of expectations processed") - valid: int = Field(..., description="Number of valid expectations") - invalid: int = Field(..., description="Number of invalid expectations") - skipped: int = Field(..., description="Number of skipped expectations") - total_processing_time: float | None = Field( - None, description="Total processing time in seconds" - ) diff --git a/template/src/collector/collector.py b/template/src/collector/collector.py index b1a99403..34248a15 100644 --- a/template/src/collector/collector.py +++ b/template/src/collector/collector.py @@ -1,64 +1,97 @@ -"""Core collector.""" +import logging -import os - -from pyoaev.daemons import CollectorDaemon # type: ignore[import-untyped] -from pyoaev.helpers import OpenAEVDetectionHelper # type: ignore[import-untyped] -from src.services.expectation_service import TemplateExpectationService -from src.services.trace_service import TemplateTraceService -from src.services.utils import TemplateConfig - -from .exception import ( +from pyoaev.daemons import CollectorDaemon +from src.collector.engines.basic import BasicCollectorEngine +from src.collector.models.exception import ( CollectorConfigError, - CollectorProcessingError, + CollectorEngineConfigError, CollectorSetupError, ) -from .expectation_handler import GenericExpectationHandler -from .expectation_manager import GenericExpectationManager +from src.collector.models.source import ( + Source, + SourceHandler, +) +from src.collector.protocols.engine import CollectorEngineProtocol +from src.collector.protocols.source_handler import SourceHandlerProtocol +from src.models.settings.config_loader import ConfigLoader LOG_PREFIX = "[Collector]" -class Collector(CollectorDaemon): # type: ignore[misc] - """Generic Collector using service provider pattern. - - This collector is use-case agnostic and works with any service provider. +class BaseCollector(CollectorDaemon): # type: ignore[misc] + """ + Generic BaseCollector providing a defined source to a generic collector engine. + This collector is use-case agnostic and works with any source provided. """ - def __init__(self) -> None: + def __init__( + self, + name: str, + source: Source, + source_handler: SourceHandlerProtocol | None = None, + engine_model: type[CollectorEngineProtocol] | None = None, + ) -> None: """Initialize the collector. Raises: CollectorConfigError: If collector initialization fails. - + CollectorEngineConfigError: If collector engine initialization fails. """ + self.name = name + try: - self.config = TemplateConfig() - self.config_instance = self.config.load + if source and not isinstance(source, Source): + raise TypeError("Source provided is not of type Source") + self.source = source + + if source_handler and not isinstance(source_handler, SourceHandlerProtocol): + raise TypeError( + "Source handler provided does not follow source handler protocol" + ) + self.source_handler = source_handler or SourceHandler() + + if engine_model and not issubclass(engine_model, CollectorEngineProtocol): + raise TypeError( + "Engine model provided does not follow collector engine protocol" + ) + self.engine_model = engine_model or BasicCollectorEngine + + self.config = ConfigLoader() super().__init__( - configuration=self.config_instance.to_daemon_config(), - callback=self._process_callback, - collector_type="openaev_template", + configuration=self.config.to_daemon_config(), + collector_type=f"openaev_{self.name}", ) self.logger.info( # type: ignore[has-type] - f"{LOG_PREFIX} Template Collector initialized successfully" + f"{LOG_PREFIX} {self.name} Collector initialized successfully" ) except Exception as err: - import logging - logging.basicConfig(level=logging.ERROR) self.logger = logging.getLogger(__name__) raise CollectorConfigError( - f"Failed to initialize the collector: {err}" + f"Failed to initialize the {self.name} collector: {err}" ) from err - def _setup(self) -> None: + try: + self.engine = self.engine_model( + name=self.name, + collector_id=self.get_id(), + source=self.source, + source_handler=self.source_handler, + oaev_api=self.api, + ) + self.set_callback(self.engine.run_engine) + except Exception as err: + raise CollectorEngineConfigError( + f"Faile to initialize the engine of {self.name} collector: {err}" + ) from err + + def _setup(self, batching=False) -> None: """Set up the collector. - Initializes Template services, expectation handler, expectation manager, + Initializes PaloAltoCortexXDR services, expectation handler, expectation manager, and OpenAEV detection helper. Sets up the collector for processing expectations. Raises: @@ -70,71 +103,11 @@ def _setup(self) -> None: super()._setup() - self.logger.debug(f"{LOG_PREFIX} Initializing Template services...") - - self.template_service = TemplateExpectationService( - config=self.config_instance - ) - - self.trace_service = TemplateTraceService(self.config_instance) - - self.expectation_handler = GenericExpectationHandler(self.template_service) - - self.expectation_manager = GenericExpectationManager( - oaev_api=self.api, - collector_id=self.get_id(), - expectation_handler=self.expectation_handler, - trace_service=self.trace_service, - ) - - supported_signatures = self.template_service.get_supported_signatures() - self.oaev_detection_helper = OpenAEVDetectionHelper( - logger=self.logger, - relevant_signatures_types=supported_signatures, - ) + self.logger.debug(f"{LOG_PREFIX} Configuring the collector engine...") + self.engine.configure_engine(self.config.template, batching=batching) self.logger.info(f"{LOG_PREFIX} Collector setup completed successfully") - self.logger.info( - f"{LOG_PREFIX} Supported signatures: {[sig.value for sig in supported_signatures]}" - ) - - service_info = self.template_service.get_service_info() - self.logger.debug(f"{LOG_PREFIX} Service info: {service_info}") except Exception as err: self.logger.error(f"{LOG_PREFIX} Collector setup failed: {err}") raise CollectorSetupError(f"Failed to setup the collector: {err}") from err - - def _process_callback(self) -> None: - """Process the callback for expectation processing. - - Executes a single processing cycle, handling expectations through the - expectation manager and logging results. Handles keyboard interrupts - and system exits gracefully. - - Raises: - CollectorProcessingError: If processing cycle fails. - - """ - try: - self.logger.info(f"{LOG_PREFIX} Starting processing cycle...") - self.logger.debug( - f"{LOG_PREFIX} Processing expectations using Template services" - ) - - results = self.expectation_manager.process_expectations( - detection_helper=self.oaev_detection_helper - ) - - self.logger.info( - f"{LOG_PREFIX} Processing cycle completed: {results.processed} total, " - f"{results.valid} valid, {results.invalid} invalid, " - f"{results.skipped} skipped" - ) - - except (KeyboardInterrupt, SystemExit): - self.logger.info(f"{LOG_PREFIX} Collector stopping...") - os._exit(0) - except Exception as e: - self.logger.error(f"{LOG_PREFIX} Error during processing cycle: {str(e)}") - raise CollectorProcessingError(f"Processing error: {str(e)}") from e diff --git a/template/src/collector/expectation_handler.py b/template/src/collector/expectation_handler.py deleted file mode 100644 index 7e47aa19..00000000 --- a/template/src/collector/expectation_handler.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Generic Expectation Handler.""" - -import logging -from typing import Any - -from pyoaev.apis.inject_expectation.model import ( # type: ignore[import-untyped] - DetectionExpectation, - PreventionExpectation, -) -from pyoaev.helpers import OpenAEVDetectionHelper # type: ignore[import-untyped] -from pyoaev.signatures.types import SignatureTypes # type: ignore[import-untyped] - -from .exception import ExpectationHandlerError -from .expectation_service_provider import ExpectationServiceProvider -from .models import ExpectationResult -from .signature_registry import ExpectationHandlerType, get_registry - -LOG_PREFIX = "[CollectorExpectationHandler]" - - -class GenericExpectationHandler: - """Generic expectation handler that delegates to service providers. - - This handler is completely agnostic to the specific use case and - delegates all processing logic to the injected service provider. - """ - - def __init__(self, service_provider: ExpectationServiceProvider) -> None: - """Initialize the generic handler. - - Args: - service_provider: Service provider implementing business logic. - - """ - self.logger = logging.getLogger(__name__) - self.service_provider = service_provider - - self.logger.debug(f"{LOG_PREFIX} Initializing generic expectation handler") - self._register_with_registry() - self.logger.info( - f"{LOG_PREFIX} Generic expectation handler initialized successfully" - ) - - def _register_with_registry(self) -> None: - """Register handler capabilities with the signature registry. - - Registers detection and prevention handlers with the signature registry - for all supported signature types from the service provider. - - Raises: - Exception: If registration with registry fails. - - """ - try: - registry = get_registry() - supported_signatures = self.service_provider.get_supported_signatures() - - registry.register_handler( - handler_type=ExpectationHandlerType.DETECTION, - handler_func=self.handle_expectation, - signature_types=supported_signatures, - ) - - registry.register_handler( - handler_type=ExpectationHandlerType.PREVENTION, - handler_func=self.handle_expectation, - signature_types=supported_signatures, - ) - - self.logger.info( - f"{LOG_PREFIX} Registered handler for {len(supported_signatures)} signature types: {[sig.value for sig in supported_signatures]}" - ) - - except Exception as e: - self.logger.error( - f"{LOG_PREFIX} Failed to register handler with registry: {e}" - ) - raise - - def handle_expectation( - self, - expectation: Any, - detection_helper: OpenAEVDetectionHelper, - ) -> ExpectationResult: - """Handle an expectation by delegating to the service provider. - - Args: - expectation: The expectation to process. - detection_helper: OpenAEV detection helper instance. - - Returns: - ExpectationResult containing processing results. - - Raises: - Exception: If expectation handling fails. - - """ - expectation_id = ( - str(expectation.inject_expectation_id) - if hasattr(expectation, "inject_expectation_id") - else "unknown" - ) - - try: - if isinstance(expectation, DetectionExpectation): - self.logger.debug( - f"{LOG_PREFIX} Processing detection expectation: {expectation_id}" - ) - result = self.service_provider.handle_detection_expectation( - expectation, detection_helper - ) - elif isinstance(expectation, PreventionExpectation): - self.logger.debug( - f"{LOG_PREFIX} Processing prevention expectation: {expectation_id}" - ) - result = self.service_provider.handle_prevention_expectation( - expectation, detection_helper - ) - else: - self.logger.warning( - f"{LOG_PREFIX} Unsupported expectation type for {expectation_id}: {type(expectation)}" - ) - result = ExpectationResult( - expectation_id=expectation_id, - is_valid=False, - expectation=expectation, - error_message="Unsupported expectation type", - ) - - return result - - except Exception as e: - self.logger.error( - f"{LOG_PREFIX} Error handling expectation {expectation_id}: {e}" - ) - raise - - def handle_batch_expectations( - self, - expectations: list[Any], - detection_helper: OpenAEVDetectionHelper, - ) -> tuple[list[ExpectationResult], int]: - """Handle a batch of expectations by delegating to the service provider. - - Post-processes results to ensure completeness by filling in missing - expectation IDs and expectation objects. - - Args: - expectations: List of expectations to process. - detection_helper: OpenAEV detection helper instance. - - Returns: - Tuple of (results, skipped_count) where: - - results: List of ExpectationResult objects for processed expectations - - skipped_count: Number of expectations skipped due to missing end_date - - Raises: - ExpectationHandlerError: If batch processing fails. - - """ - try: - self.logger.info( - f"{LOG_PREFIX} Starting batch processing of {len(expectations)} expectations" - ) - - results, skipped_count = self.service_provider.handle_batch_expectations( - expectations, detection_helper - ) - - # Post-process results to ensure completeness - self.logger.debug(f"{LOG_PREFIX} Post-processing batch results...") - for i, result in enumerate(results): - if result.expectation is None and i < len(expectations): - result.expectation = expectations[i] - if not result.expectation_id and result.expectation: - result.expectation_id = str( - result.expectation.inject_expectation_id - ) - - valid_count = sum(1 for r in results if r.is_valid) - invalid_count = len(results) - valid_count - - self.logger.info( - f"{LOG_PREFIX} Batch processing completed: {valid_count} valid, {invalid_count} invalid, {skipped_count} skipped" - ) - - return results, skipped_count - - except Exception as e: - self.logger.error(f"{LOG_PREFIX} Batch processing failed: {e}") - raise ExpectationHandlerError(f"Error in batch processing: {e}") from e - - def get_supported_signatures(self) -> list[SignatureTypes]: - """Get supported signature types from service provider. - - Returns: - List of SignatureTypes supported by the service provider. - - """ - signatures = self.service_provider.get_supported_signatures() - self.logger.debug( - f"{LOG_PREFIX} Supported signatures: {[sig.value for sig in signatures]}" - ) - return signatures diff --git a/template/src/collector/expectation_manager.py b/template/src/collector/expectation_manager.py deleted file mode 100644 index 3fa752cd..00000000 --- a/template/src/collector/expectation_manager.py +++ /dev/null @@ -1,426 +0,0 @@ -"""Generic Expectation Manager.""" - -import logging -from typing import Any - -from pyoaev.apis.inject_expectation.model import ( # type: ignore[import-untyped] - DetectionExpectation, - PreventionExpectation, -) -from pyoaev.client import OpenAEV # type: ignore[import-untyped] -from pyoaev.helpers import OpenAEVDetectionHelper # type: ignore[import-untyped] - -from .exception import ( - APIError, - BulkUpdateError, - ExpectationProcessingError, - ExpectationUpdateError, -) -from .expectation_handler import GenericExpectationHandler -from .models import ExpectationResult, ProcessingSummary -from .trace_manager import TraceManager -from .trace_service_provider import TraceServiceProvider - -LOG_PREFIX = "[CollectorExpectationManager]" - - -class GenericExpectationManager: - """Generic expectation manager that works with any service provider. - - This manager is completely agnostic to the specific use case and - delegates all processing logic to the injected service providers. - """ - - def __init__( - self, - oaev_api: OpenAEV, - collector_id: str, - expectation_handler: GenericExpectationHandler, - trace_service: TraceServiceProvider | None = None, - ) -> None: - """Initialize generic expectation manager. - - Args: - oaev_api: OpenAEV API client. - collector_id: ID of the collector. - expectation_handler: Handler for processing expectations. - trace_service: Optional service for creating traces. - - Raises: - ValueError: If required parameters are None or empty. - - """ - if not oaev_api: - raise ValueError("oaev_api cannot be None") - if not collector_id: - raise ValueError("collector_id cannot be empty") - if not expectation_handler: - raise ValueError("expectation_handler cannot be None") - - self.logger = logging.getLogger(__name__) - self.oaev_api = oaev_api - self.collector_id = collector_id - self.expectation_handler = expectation_handler - self.trace_manager = TraceManager( - oaev_api=oaev_api, - collector_id=collector_id, - trace_service=trace_service, - ) - - self.logger.info( - f"{LOG_PREFIX} Expectation manager initialized for collector: {collector_id}" - ) - - def process_expectations( - self, detection_helper: OpenAEVDetectionHelper - ) -> ProcessingSummary: - """Process all expectations using the injected handler. - - Fetches expectations from OpenAEV, processes them through the handler, - updates expectations in OpenAEV, and creates traces. - - Args: - detection_helper: OpenAEV detection helper. - - Returns: - ProcessingSummary containing processing results. - - Raises: - ExpectationProcessingError: If processing fails. - - """ - try: - self.logger.info(f"{LOG_PREFIX} Starting expectation processing cycle") - - self.logger.debug(f"{LOG_PREFIX} Fetching expectations from OpenAEV...") - expectations = self._fetch_expectations() - - if not expectations: - self.logger.warning(f"{LOG_PREFIX} No expectations found to process") - return ProcessingSummary( - processed=0, - valid=0, - invalid=0, - skipped=0, - total_processing_time=None, - ) - - supported_expectations = [ - exp - for exp in expectations - if isinstance(exp, (DetectionExpectation, PreventionExpectation)) - ] - - total_expectations = len(expectations) - supported_count = len(supported_expectations) - skipped_count = total_expectations - supported_count - - self.logger.info( - f"{LOG_PREFIX} Found {total_expectations} total expectations: " - f"{supported_count} supported, {skipped_count} skipped" - ) - - if skipped_count > 0: - self.logger.debug( - f"{LOG_PREFIX} Skipped {skipped_count} unsupported expectation types" - ) - - self.logger.debug( - f"{LOG_PREFIX} Processing expectations through handler..." - ) - results, service_skipped_count = ( - self.expectation_handler.handle_batch_expectations( - supported_expectations, detection_helper - ) - ) - - self.logger.debug(f"{LOG_PREFIX} Updating expectations in OpenAEV...") - self._bulk_update_expectations(results) - - self.logger.debug(f"{LOG_PREFIX} Creating and submitting traces...") - self.trace_manager.create_and_submit_traces(results) - - valid_count = sum(1 for r in results if r.is_valid) - invalid_count = len(results) - valid_count - - total_skipped = skipped_count + service_skipped_count - - summary = ProcessingSummary( - processed=len(results), - valid=valid_count, - invalid=invalid_count, - skipped=total_skipped, - total_processing_time=None, - ) - - self.logger.info( - f"{LOG_PREFIX} Expectation processing: processed {total_expectations} items -> {len(results)} results" - ) - - self.logger.info( - f"{LOG_PREFIX} Processing cycle completed: {valid_count} valid, " - f"{invalid_count} invalid, {total_skipped} skipped ({skipped_count} unsupported types, {service_skipped_count} no end_date)" - ) - - return summary - - except (BulkUpdateError, APIError) as e: - self.logger.error(f"{LOG_PREFIX} API operation failed: {e}") - raise ExpectationProcessingError(f"API error during processing: {e}") from e - except Exception as e: - self.logger.error(f"{LOG_PREFIX} Unexpected error during processing: {e}") - raise ExpectationProcessingError( - f"Unexpected error processing expectations: {e}" - ) from e - - def _bulk_update_expectations(self, results: list[ExpectationResult]) -> None: - """Bulk update expectations in OpenAEV. - - Prepares bulk data from results and attempts to update expectations - using the OpenAEV bulk update API. - - Args: - results: List of ExpectationResult objects to update. - - Raises: - BulkUpdateError: If bulk update fails. - - """ - if not results: - self.logger.debug( - f"{LOG_PREFIX} No results to update, skipping bulk update" - ) - return - - try: - self.logger.debug( - f"{LOG_PREFIX} Preparing bulk data for {len(results)} results..." - ) - bulk_data = self._prepare_bulk_data(results) - - if bulk_data: - self.logger.debug( - f"{LOG_PREFIX} Attempting bulk update of {len(bulk_data)} expectations..." - ) - self._attempt_bulk_update(bulk_data) - else: - self.logger.debug( - f"{LOG_PREFIX} No valid bulk data prepared, skipping update" - ) - - except Exception as e: - self.logger.error(f"{LOG_PREFIX} Bulk update failed: {e}") - raise BulkUpdateError(f"Error in bulk update: {e}") from e - - def _prepare_bulk_data( - self, results: list[ExpectationResult] - ) -> dict[str, dict[str, Any]]: - """Prepare bulk data from results. - - Transforms ExpectationResult objects into dictionary format - required by the OpenAEV bulk update API. - - Args: - results: List of ExpectationResult objects. - - Returns: - Dictionary mapping expectation IDs to update data. - - """ - bulk_data = {} - skipped_count = 0 - - for result in results: - try: - expectation_id = result.expectation_id - if not expectation_id: - skipped_count += 1 - self.logger.debug( - f"{LOG_PREFIX} Skipping result without expectation_id" - ) - continue - - is_valid = result.is_valid - expectation = result.expectation - if expectation: - result_text = self._get_result_text(expectation, is_valid) - bulk_data[expectation_id] = { - "collector_id": self.collector_id, - "result": result_text, - "is_success": is_valid, - } - self.logger.debug( - f"{LOG_PREFIX} Prepared update for expectation {expectation_id}: " - f"result='{result_text}', success={is_valid}" - ) - else: - skipped_count += 1 - self.logger.debug( - f"{LOG_PREFIX} Skipping result {expectation_id} without expectation object" - ) - except Exception as e: - skipped_count += 1 - self.logger.warning(f"{LOG_PREFIX} Error processing result: {e}") - - if skipped_count > 0: - self.logger.debug( - f"{LOG_PREFIX} Skipped {skipped_count} results during bulk data preparation" - ) - return bulk_data - - def _get_result_text( - self, expectation: DetectionExpectation | PreventionExpectation, is_valid: bool - ) -> str: - """Get result text based on expectation type and validity. - - Args: - expectation: The expectation object (Detection or Prevention). - is_valid: Whether the expectation was successfully validated. - - Returns: - Human-readable result text for the expectation. - - """ - try: - base_text = ( - "Detected" - if isinstance(expectation, DetectionExpectation) - else "Prevented" - ) - result_text = base_text if is_valid else f"Not {base_text}" - - self.logger.debug( - f"{LOG_PREFIX} Generated result text: '{result_text}' for {type(expectation).__name__}" - ) - return result_text - except Exception as e: - self.logger.warning(f"{LOG_PREFIX} Error generating result text: {e}") - return "Unknown" - - def _attempt_bulk_update(self, bulk_data: dict[str, dict[str, Any]]) -> None: - """Attempt bulk update with fallback to individual updates. - - Tries to use the bulk update API first, then falls back to individual - updates if the bulk operation fails. - - Args: - bulk_data: Dictionary of expectation updates to apply. - - Raises: - BulkUpdateError: If both bulk and individual updates fail. - - """ - try: - self.logger.debug(f"{LOG_PREFIX} Attempting bulk update via OpenAEV API...") - self.oaev_api.inject_expectation.bulk_update( - inject_expectation_input_by_id=bulk_data - ) - self.logger.info( - f"{LOG_PREFIX} Successfully bulk updated {len(bulk_data)} expectations" - ) - - except Exception as bulk_error: - self.logger.warning( - f"{LOG_PREFIX} Bulk update failed, falling back to individual updates: {bulk_error}" - ) - try: - self._fallback_individual_updates(bulk_data) - except Exception as fallback_error: - raise BulkUpdateError( - f"Both bulk and individual updates failed: {fallback_error}" - ) from fallback_error - - def _fallback_individual_updates( - self, bulk_data: dict[str, dict[str, Any]] - ) -> None: - """Fallback to individual expectation updates. - - Updates expectations one by one when bulk update fails. - - Args: - bulk_data: Dictionary of expectation updates to apply. - - """ - self.logger.info( - f"{LOG_PREFIX} Attempting individual updates for {len(bulk_data)} expectations" - ) - success_count = 0 - error_count = 0 - - for expectation_id, update_data in bulk_data.items(): - try: - self._update_expectation(expectation_id, update_data) - success_count += 1 - except (APIError, ExpectationUpdateError) as e: - error_count += 1 - self.logger.error( - f"{LOG_PREFIX} Failed to update expectation {expectation_id}: {e}" - ) - except Exception as e: - error_count += 1 - self.logger.error( - f"{LOG_PREFIX} Unexpected error updating expectation {expectation_id}: {e}" - ) - - self.logger.info( - f"{LOG_PREFIX} Individual updates completed: {success_count} successful, {error_count} failed" - ) - - def _update_expectation( - self, expectation_id: str, update_data: dict[str, Any] - ) -> None: - """Update a single expectation. - - Args: - expectation_id: ID of the expectation to update. - update_data: Update data to apply to the expectation. - - Raises: - ExpectationUpdateError: If the update fails. - - """ - self.logger.debug( - f"{LOG_PREFIX} Updating individual expectation: {expectation_id}" - ) - - try: - self.oaev_api.inject_expectation.update( - inject_expectation_id=expectation_id, - inject_expectation=update_data, - ) - self.logger.debug( - f"{LOG_PREFIX} Successfully updated expectation {expectation_id}" - ) - - except Exception as individual_error: - raise ExpectationUpdateError( - f"Failed to update expectation {expectation_id}: {individual_error}" - ) from individual_error - - def _fetch_expectations( - self, - ) -> list[DetectionExpectation | PreventionExpectation]: - """Fetch expectations from OpenAEV. - - Returns: - List of expectations. - - """ - self.logger.debug( - f"{LOG_PREFIX} Fetching expectations for collector: {self.collector_id}" - ) - - try: - expectations = ( - self.oaev_api.inject_expectation.expectations_models_for_source( - source_id=self.collector_id - ) - ) - self.logger.debug( - f"{LOG_PREFIX} Fetched {len(expectations)} expectations, reversing order..." - ) - expectations = list(reversed(expectations)) - return expectations - except Exception as e: - self.logger.error(f"{LOG_PREFIX} Error fetching expectations: {e}") - return [] diff --git a/template/src/collector/expectation_service_provider.py b/template/src/collector/expectation_service_provider.py deleted file mode 100644 index 627bc4c5..00000000 --- a/template/src/collector/expectation_service_provider.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Protocol defining the interface for expectation service providers.""" - -from typing import Any, Protocol - -from pyoaev.apis.inject_expectation.model import ( # type: ignore[import-untyped] - DetectionExpectation, - PreventionExpectation, -) -from pyoaev.helpers import OpenAEVDetectionHelper # type: ignore[import-untyped] -from pyoaev.signatures.types import SignatureTypes # type: ignore[import-untyped] - -from .models import ExpectationResult - - -class ExpectationServiceProvider(Protocol): - """Protocol defining the interface for expectation service providers.""" - - def get_supported_signatures(self) -> list[SignatureTypes]: - """Get list of signature types this provider supports. - - Returns: - List of SignatureTypes that this provider can handle. - - """ - ... - - def handle_detection_expectation( - self, - expectation: DetectionExpectation, - detection_helper: OpenAEVDetectionHelper, - ) -> ExpectationResult: - """Handle a detection expectation. - - Args: - expectation: The detection expectation to process. - detection_helper: OpenAEV detection helper instance. - - Returns: - ExpectationResult containing the processing outcome. - - """ - ... - - def handle_prevention_expectation( - self, - expectation: PreventionExpectation, - detection_helper: OpenAEVDetectionHelper, - ) -> ExpectationResult: - """Handle a prevention expectation. - - Args: - expectation: The prevention expectation to process. - detection_helper: OpenAEV detection helper instance. - - Returns: - ExpectationResult containing the processing outcome. - - """ - ... - - def handle_batch_expectations( - self, expectations: list[Any], detection_helper: OpenAEVDetectionHelper - ) -> tuple[list[ExpectationResult], int]: - """Handle a batch of expectations efficiently. - - Args: - expectations: List of expectations to process in batch. - detection_helper: OpenAEV detection helper instance. - - Returns: - Tuple of (results, skipped_count) where: - - results: List of ExpectationResult objects for processed expectations - - skipped_count: Number of expectations skipped due to missing end_date - - """ - ... diff --git a/template/tests/services/__init__.py b/template/src/collector/helpers/__init__.py similarity index 100% rename from template/tests/services/__init__.py rename to template/src/collector/helpers/__init__.py diff --git a/template/src/collector/signature_registry.py b/template/src/collector/signature_registry.py deleted file mode 100644 index ec5675b0..00000000 --- a/template/src/collector/signature_registry.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Signature Registry for dynamic expectation handling.""" - -from enum import Enum -from typing import Any, Callable - -from pyoaev.signatures.types import SignatureTypes # type: ignore[import-untyped] - -from .models import ExpectationResult - - -class ExpectationHandlerType(Enum): - """Types of expectation handlers.""" - - DETECTION = "detection" - PREVENTION = "prevention" - - -class SignatureRegistry: - """Simple registry for managing signature subscriptions and expectation handlers. - - This registry allows components to dynamically register: - - Which signature types they're interested in - - How to handle different types of expectations - - Keeps it simple by using basic data structures and clear interfaces. - """ - - def __init__(self) -> None: - """Initialize the registry. - - Creates empty data structures for managing signature subscriptions - and expectation handlers. - """ - self._subscribed_signatures: set[SignatureTypes] = set() - self._handlers: dict[ - ExpectationHandlerType, Callable[[Any, Any], ExpectationResult] - ] = {} - self._handler_signatures: dict[ExpectationHandlerType, set[SignatureTypes]] = {} - - def subscribe_to_signatures(self, signature_types: list[SignatureTypes]) -> None: - """Subscribe to specific signature types. - - Args: - signature_types: List of signature types to subscribe to. - - """ - self._subscribed_signatures.update(signature_types) - - def register_handler( - self, - handler_type: ExpectationHandlerType, - handler_func: Callable[[Any, Any], ExpectationResult], - signature_types: list[SignatureTypes], - ) -> None: - """Register an expectation handler for specific signature types. - - Args: - handler_type: Type of handler (detection/prevention). - handler_func: Function to handle expectations. - signature_types: Signature types this handler supports. - - """ - self._handlers[handler_type] = handler_func - self._handler_signatures[handler_type] = set(signature_types) - - self.subscribe_to_signatures(signature_types) - - def get_subscribed_signatures(self) -> list[SignatureTypes]: - """Get all subscribed signature types. - - Returns: - List of subscribed signature types. - - """ - return list(self._subscribed_signatures) - - def has_handler_for_signatures( - self, - handler_type: ExpectationHandlerType, - signature_types: list[SignatureTypes], - ) -> bool: - """Check if a handler supports the given signature types. - - Args: - handler_type: Type of handler to check. - signature_types: Signature types to check. - - Returns: - True if handler supports any of the signature types. - - """ - if handler_type not in self._handler_signatures: - return False - - handler_sigs = self._handler_signatures[handler_type] - return any(sig in handler_sigs for sig in signature_types) - - def get_handler( - self, handler_type: ExpectationHandlerType - ) -> Callable[[Any, Any], ExpectationResult]: - """Get handler function for the given type. - - Args: - handler_type: Type of handler to retrieve. - - Returns: - Handler function. - - Raises: - KeyError: If no handler registered for the type. - - """ - if handler_type not in self._handlers: - raise KeyError(f"No handler registered for type: {handler_type}") - return self._handlers[handler_type] - - def is_signature_supported(self, signature_type: SignatureTypes) -> bool: - """Check if a signature type is supported by any registered handler. - - Args: - signature_type: Signature type to check. - - Returns: - True if supported. - - """ - return signature_type in self._subscribed_signatures - - def get_handler_types(self) -> list[ExpectationHandlerType]: - """Get all registered handler types. - - Returns: - List of registered handler types. - - """ - return list(self._handlers.keys()) - - def clear(self) -> None: - """Clear all registrations. - - Removes all signature subscriptions and handler registrations. - Useful for testing and cleanup scenarios. - """ - self._subscribed_signatures.clear() - self._handlers.clear() - self._handler_signatures.clear() - - -_registry = SignatureRegistry() - - -def get_registry() -> SignatureRegistry: - """Get the global signature registry instance. - - Returns: - The global registry instance. - - """ - return _registry diff --git a/template/src/collector/trace_manager.py b/template/src/collector/trace_manager.py deleted file mode 100644 index 983859bf..00000000 --- a/template/src/collector/trace_manager.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Trace Manager for handling expectation traces. - -This module provides the TraceManager class which handles all trace-related operations -for expectation processing. It separates trace concerns from the main expectation -""" - -import logging -from typing import Any - -from pyoaev.client import OpenAEV # type: ignore[import-untyped] - -from .exception import TraceCreationError, TraceSubmissionError, TracingError -from .models import ExpectationResult -from .trace_service_provider import TraceServiceProvider - -LOG_PREFIX = "[CollectorTraceManager]" - - -class TraceManager: - """Manages trace creation and submission for expectations. - - This manager handles all trace-related operations, including creating traces - from expectation results and submitting them to the OpenAEV API. - """ - - def __init__( - self, - oaev_api: OpenAEV, - collector_id: str, - trace_service: TraceServiceProvider | None = None, - ) -> None: - """Initialize trace manager. - - Args: - oaev_api: OpenAEV API client. - collector_id: ID of the collector. - trace_service: Service for creating traces from results. - - """ - self.logger = logging.getLogger(__name__) - self.oaev_api = oaev_api - self.collector_id = collector_id - self.trace_service = trace_service - - self.logger.info( - f"{LOG_PREFIX} Trace manager initialized for collector: {collector_id}" - ) - if trace_service: - self.logger.debug( - f"{LOG_PREFIX} Trace service available for trace creation" - ) - else: - self.logger.debug( - f"{LOG_PREFIX} No trace service provided - traces will be skipped" - ) - - def create_and_submit_traces(self, results: list[ExpectationResult]) -> None: - """Create and submit traces from expectation results. - - Creates traces from the provided expectation results using the trace service - and submits them to the OpenAEV API. - - Args: - results: List of ExpectationResult objects. - - Raises: - TracingError: If trace creation or submission fails. - - """ - try: - if not self.trace_service: - self.logger.debug( - f"{LOG_PREFIX} No trace service provided, skipping trace creation" - ) - return - - self.logger.debug( - f"{LOG_PREFIX} Creating traces from {len(results)} expectation results..." - ) - traces = self.trace_service.create_traces_from_results( - results, self.collector_id - ) - - if not traces: - self.logger.info(f"{LOG_PREFIX} No traces created from results") - return - - self.logger.info( - f"{LOG_PREFIX} Created {len(traces)} traces, submitting to OpenAEV..." - ) - self._submit_traces(traces) - - except Exception as e: - self.logger.error( - f"{LOG_PREFIX} Error creating and submitting traces: {e} (Context: results_count={len(results)}, collector_id={self.collector_id})" - ) - raise TracingError(f"Error creating and submitting traces: {e}") from e - - def _submit_traces(self, traces: list[Any]) -> None: - """Submit traces to the OpenAEV API. - - Converts traces to API format and submits them using bulk creation. - Falls back to individual creation if bulk submission fails. - - Args: - traces: List of trace objects to submit. - - Raises: - TraceSubmissionError: If trace submission fails. - - """ - try: - self.logger.debug(f"{LOG_PREFIX} Converting traces to API format...") - trace_dicts = [trace.to_api_dict() for trace in traces] - - if not trace_dicts: - self.logger.warning( - f"{LOG_PREFIX} No trace dictionaries generated from traces" - ) - return - - self.logger.debug( - f"{LOG_PREFIX} Submitting {len(trace_dicts)} trace dictionaries to OpenAEV" - ) - self.logger.debug( - f"{LOG_PREFIX} Trace data preview: {trace_dicts[:2] if len(trace_dicts) > 2 else trace_dicts}" - ) - - response = self.oaev_api.inject_expectation_trace.bulk_create( - payload={"expectation_traces": trace_dicts} - ) - - self.logger.info( - f"{LOG_PREFIX} Successfully created {len(trace_dicts)} expectation traces" - ) - self.logger.debug(f"{LOG_PREFIX} OpenAEV response: {response}") - - except Exception as e: - self.logger.error(f"{LOG_PREFIX} Bulk trace submission failed: {e}") - try: - self.logger.info( - f"{LOG_PREFIX} Attempting individual trace creation as fallback..." - ) - self._fallback_individual_trace_creation(traces) - except TraceCreationError as fallback_error: - self.logger.error( - f"{LOG_PREFIX} Fallback trace creation also failed: {fallback_error}" - ) - raise TraceSubmissionError(f"Error submitting traces: {e}") from e - - def _fallback_individual_trace_creation(self, traces: list[Any]) -> None: - """Fallback method to create traces individually if bulk creation fails. - - Creates traces one by one when bulk creation fails, providing - resilience for trace submission. - - Args: - traces: List of trace objects to create individually. - - Raises: - TraceCreationError: If all individual trace creations fail. - - """ - try: - self.logger.info( - f"{LOG_PREFIX} Creating {len(traces)} traces individually as fallback" - ) - success_count = 0 - error_count = 0 - - for i, trace in enumerate(traces, 1): - try: - self.logger.debug( - f"{LOG_PREFIX} Creating individual trace {i}/{len(traces)}" - ) - r = self.oaev_api.inject_expectation_trace.create( - trace.to_api_dict() - ) - success_count += 1 - self.logger.debug( - f"{LOG_PREFIX} Individual trace {i} created successfully" - ) - self.logger.debug(f"{LOG_PREFIX} single Response: {r}") - except Exception as individual_error: - error_count += 1 - self.logger.error( - f"{LOG_PREFIX} Failed to create individual trace {i}: {individual_error}" - ) - - self.logger.info( - f"{LOG_PREFIX} Individual trace creation completed: {success_count} successful, {error_count} failed" - ) - - if success_count == 0: - raise TraceCreationError("All individual trace creations failed") - - except Exception as e: - self.logger.error( - f"{LOG_PREFIX} Error in fallback trace creation: {e} (Context: traces_count={len(traces)}, success_count={success_count})" - ) - raise TraceCreationError(f"Error in fallback trace creation: {e}") from e diff --git a/template/src/collector/trace_service_provider.py b/template/src/collector/trace_service_provider.py deleted file mode 100644 index 9c4abe4f..00000000 --- a/template/src/collector/trace_service_provider.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Protocol for trace creation services.""" - -from typing import Protocol - -from .models import ExpectationResult, ExpectationTrace - - -class TraceServiceProvider(Protocol): - """Protocol for trace creation services.""" - - def create_traces_from_results( - self, results: list[ExpectationResult], collector_id: str - ) -> list[ExpectationTrace]: - """Create trace data from processing results. - - Args: - results: List of ExpectationResult objects to create traces from. - collector_id: ID of the collector creating the traces. - - Returns: - List of ExpectationTrace objects for successful expectations. - - """ - ... diff --git a/template/src/img/template-logo.png b/template/src/img/template-logo.png new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/template/src/img/template-logo.png @@ -0,0 +1 @@ +1 diff --git a/template/src/models/settings/__init__.py b/template/src/models/settings/__init__.py index 12de273c..c5c9ad18 100644 --- a/template/src/models/settings/__init__.py +++ b/template/src/models/settings/__init__.py @@ -1,9 +1,9 @@ -from src.models.configs.base_settings import ConfigBaseSettings -from src.models.configs.collector_configs import ( +from src.models.settings.base_settings import ConfigBaseSettings +from src.models.settings.collector_configs import ( _ConfigLoaderCollector, _ConfigLoaderOAEV, ) -from src.models.configs.template_configs import _ConfigLoaderTemplate +from src.models.settings.template_configs import _ConfigLoaderTemplate __all__ = [ "ConfigBaseSettings", diff --git a/template/src/models/settings/collector_configs.py b/template/src/models/settings/collector_configs.py index 7ef16306..e39b9c67 100644 --- a/template/src/models/settings/collector_configs.py +++ b/template/src/models/settings/collector_configs.py @@ -4,7 +4,7 @@ from typing import Annotated, Literal from pydantic import Field, HttpUrl, PlainSerializer -from src.models.configs import ConfigBaseSettings +from src.models.settings import ConfigBaseSettings LogLevelToLower = Annotated[ Literal["debug", "info", "warn", "error"], diff --git a/template/src/models/settings/config_loader.py b/template/src/models/settings/config_loader.py index 9db00642..dbba6622 100644 --- a/template/src/models/settings/config_loader.py +++ b/template/src/models/settings/config_loader.py @@ -11,7 +11,7 @@ YamlConfigSettingsSource, ) from pyoaev.configuration import Configuration -from src.models.configs import ( +from src.models.settings import ( ConfigBaseSettings, _ConfigLoaderCollector, _ConfigLoaderOAEV, @@ -98,7 +98,8 @@ def settings_customise_sources( env_file_encoding="utf-8", ), ) - elif yaml_path.exists(): + + if yaml_path.exists(): return ( YamlConfigSettingsSource( settings_cls, diff --git a/template/src/models/settings/template_configs.py b/template/src/models/settings/template_configs.py index f4570648..142e0e6a 100644 --- a/template/src/models/settings/template_configs.py +++ b/template/src/models/settings/template_configs.py @@ -3,7 +3,7 @@ from datetime import timedelta from pydantic import Field -from src.models.configs import ConfigBaseSettings +from src.models.settings import ConfigBaseSettings class _ConfigLoaderTemplate(ConfigBaseSettings): diff --git a/template/src/services/converter.py b/template/src/services/converter.py deleted file mode 100644 index 1f42d005..00000000 --- a/template/src/services/converter.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Template Data Converter to OAEV format.""" - -import logging -from typing import Any - -from .exception import TemplateDataConversionError, TemplateValidationError -from .model_data import TemplateData - -LOG_PREFIX = "[TemplateConverter]" - - -class TemplateConverter: - """Converter for Template data to OAEV format.""" - - def __init__(self) -> None: - """Initialize the Template data converter.""" - self.logger = logging.getLogger(__name__) - self.logger.debug(f"{LOG_PREFIX} Template converter initialized") - - def convert_data_to_oaev(self, data: list[TemplateData]) -> list[dict[str, Any]]: - """Convert Template data to OAEV format. - - Args: - data: List of TemplateData objects. - - Returns: - List of OAEV data dictionaries. - - Raises: - TemplateValidationError: If data format is invalid. - TemplateDataConversionError: If conversion fails. - - """ - if not data: - self.logger.debug(f"{LOG_PREFIX} No data to convert") - return [] - - if not isinstance(data, list): - raise TemplateValidationError("data must be a list") - - try: - self.logger.debug( - f"{LOG_PREFIX} Converting {len(data)} data to OAEV format" - ) - - oaev_data_list = [] - converted_count = 0 - - for i, single_data in enumerate(data, 1): - if not isinstance(single_data, TemplateData): - self.logger.warning( - f"{LOG_PREFIX} Item {i} is not a TemplateData: {type(single_data)}" - ) - continue - - try: - oaev_data = self._convert_data_to_oaev(single_data) - if oaev_data: - oaev_data_list.append(oaev_data) - converted_count += 1 - self.logger.debug( - f"{LOG_PREFIX} Converted data {i}/{len(data)}" - ) - except Exception as e: - self.logger.warning(f"{LOG_PREFIX} Failed to convert data {i}: {e}") - - self.logger.info( - f"{LOG_PREFIX} Conversion completed: {converted_count} data -> {len(oaev_data_list)} OAEV items" - ) - - return oaev_data_list - - except Exception as e: - raise TemplateDataConversionError( - f"Failed to convert data to OAEV format: {e}" - ) from e - - def _convert_data_to_oaev(self, data: TemplateData) -> dict[str, Any]: - """Convert a single data to OAEV format. - - Args: - data: TemplateData object to convert. - - Returns: - OAEV formatted data dictionary. - - Raises: - TemplateValidationError: If data is invalid. - - """ - try: - oaev_data = {"change-me-key": "change-me-value"} - # oaev_data to update according to the custom data object for your collector - - self.logger.debug( - f"{LOG_PREFIX} Successfully converted data to OAEV format" - ) - return oaev_data - - except Exception as e: - raise TemplateDataConversionError( - f"Error converting data to OAEV: {e}" - ) from e diff --git a/template/src/services/exception.py b/template/src/services/exception.py deleted file mode 100644 index 12686b44..00000000 --- a/template/src/services/exception.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Template Service Exceptions.""" - - -class TemplateServiceError(Exception): - """Base exception for all Template service errors.""" - - pass - - -class TemplateExpectationError(TemplateServiceError): - """Raised when there's an error processing expectations.""" - - pass - - -class TemplateDataConversionError(TemplateServiceError): - """Raised when there's an error converting data.""" - - pass - - -class TemplateFetcherError(TemplateServiceError): - """Raised when there's an error with Template fetcher operations.""" - - pass - - -class TemplateValidationError(TemplateServiceError): - """Raised when input validation fails.""" - - pass diff --git a/template/src/services/expectation_service.py b/template/src/services/expectation_service.py deleted file mode 100644 index a1214c14..00000000 --- a/template/src/services/expectation_service.py +++ /dev/null @@ -1,522 +0,0 @@ -"""Template Expectation Service with batch-based processing.""" - -import logging -from datetime import datetime, timezone -from typing import Any - -from pydantic import BaseModel, Field -from pyoaev.apis.inject_expectation.model.expectation import ( - DetectionExpectation, - PreventionExpectation, -) -from pyoaev.signatures.types import SignatureTypes - -from .converter import TemplateConverter -from .exception import TemplateExpectationError, TemplateFetcherError -from .fetcher_data import FetcherData -from .model_data import TemplateData -from .utils import SignatureExtractor, TraceBuilder - -LOG_PREFIX = "[TemplateExpectationService]" - - -class ExpectationResult(BaseModel): - """Model for expectation processing results.""" - - expectation_id: str = Field(..., description="ID of the processed expectation") - is_valid: bool = Field(..., description="Whether the expectation was validated") - expectation: Any | None = Field(None, description="The original expectation object") - matched_alerts: list[dict[str, Any]] | None = Field( - None, description="List of alerts that matched this expectation" - ) - error_message: str | None = Field( - None, description="Error message if processing failed" - ) - processing_time: float | None = Field( - None, description="Time taken to process this expectation in seconds" - ) - - -class TemplateExpectationService: - """Service for processing Template expectations in batches.""" - - def __init__( - self, - config: Any | None = None, - ) -> None: - """Initialize the Template expectation service. - - Args: - config: Configuration loader for alternative initialization. - - Raises: - TemplateValidationError: If required parameters are None. - - """ - self.logger: logging.Logger = logging.getLogger(__name__) - - self.converter: TemplateConverter = TemplateConverter() - self.batch_size: int = config.template.expectation_batch_size - self.time_window = config.template.time_window - - self.data_fetcher: FetcherData = FetcherData() - - self.logger.info( - f"{LOG_PREFIX} Service initialized with batch size: {self.batch_size}" - ) - - def get_supported_signatures(self) -> list[SignatureTypes]: - """Get list of supported signature types. - - Returns: - List of supported SignatureTypes enum values. - - """ - return [ - SignatureTypes.SIG_TYPE_PARENT_PROCESS_NAME, - SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, - SignatureTypes.SIG_TYPE_END_DATE, - ] - - def handle_batch_expectations( - self, - expectations: list[DetectionExpectation | PreventionExpectation], - detection_helper: Any, - ) -> tuple[list[ExpectationResult], int]: - """Handle a batch of expectations. - - Args: - expectations: List of expectations to process. - detection_helper: OpenAEV detection helper instance. - - Returns: - Tuple of (results, skipped_count) where: - - results: List of ExpectationResult objects for processed expectations - - skipped_count: Number of expectations skipped due to missing end_date - - Raises: - TemplateExpectationError: If batch processing fails. - - """ - if not expectations: - self.logger.info(f"{LOG_PREFIX} No expectations to process") - return [], 0 - - try: - self.logger.info( - f"{LOG_PREFIX} Starting new batch processing of {len(expectations)} expectations" - ) - - batches, skipped_count = self._create_expectation_batches(expectations) - self.logger.info( - f"{LOG_PREFIX} Created {len(batches)} batches of size {self.batch_size} (skipped {skipped_count} expectations without end_date)" - ) - - all_results = [] - - for batch_idx, batch in enumerate(batches, 1): - self.logger.info( - f"{LOG_PREFIX} Processing batch {batch_idx}/{len(batches)} with {len(batch)} expectations" - ) - - try: - batch_results = self._process_expectation_batch( - batch, detection_helper, batch_idx - ) - all_results.extend(batch_results) - - self.logger.info( - f"{LOG_PREFIX} Batch {batch_idx} completed: {len(batch_results)} results" - ) - except Exception as e: - self.logger.error( - f"{LOG_PREFIX} Error processing batch {batch_idx}: {e}" - ) - error_results = [ - self._create_error_result_object( - TemplateExpectationError(f"Batch processing error: {e}"), - expectation, - ) - for expectation in batch - ] - all_results.extend(error_results) - - valid_count = sum(1 for r in all_results if r.is_valid) - invalid_count = len(all_results) - valid_count - - self.logger.info( - f"{LOG_PREFIX} New batch processing completed: {valid_count} valid, {invalid_count} invalid, {skipped_count} skipped (no end_date)" - ) - - return all_results, skipped_count - - except Exception as e: - raise TemplateExpectationError( - f"Error in handle_batch_expectations: {e}" - ) from e - - def _create_expectation_batches( - self, expectations: list[DetectionExpectation | PreventionExpectation] - ) -> tuple[list[list[DetectionExpectation | PreventionExpectation]], int]: - """Group expectations into batches, filtering out those without end_date. - - Args: - expectations: List of expectations to batch. - - Returns: - Tuple of (batches, skipped_count) where: - - batches: List of expectation batches that have end_date signatures - - skipped_count: Number of expectations skipped due to missing end_date - - """ - valid_expectations = [] - skipped_count = 0 - - for expectation in expectations: - has_end_date = ( - SignatureExtractor.extract_end_date([expectation]) is not None - ) - - if has_end_date: - valid_expectations.append(expectation) - else: - skipped_count += 1 - self.logger.debug( - f"{LOG_PREFIX} Skipping expectation {expectation.inject_expectation_id} - no end_date signature found" - ) - - if skipped_count > 0: - self.logger.info( - f"{LOG_PREFIX} Filtered out {skipped_count} expectations without end_date signatures" - ) - - batches = [] - for i in range(0, len(valid_expectations), self.batch_size): - batch = valid_expectations[i : i + self.batch_size] - batches.append(batch) - - self.logger.debug( - f"{LOG_PREFIX} Created {len(batches)} batches from {len(valid_expectations)} valid expectations (skipped {skipped_count})" - ) - return batches, skipped_count - - def _process_expectation_batch( - self, - batch: list[DetectionExpectation | PreventionExpectation], - detection_helper: Any, - batch_idx: int, - ) -> list[ExpectationResult]: - """Process a single batch of expectations. - - Args: - batch: Batch of expectations to process. - detection_helper: OpenAEV detection helper. - batch_idx: Batch index for logging. - - Returns: - List of ExpectationResult objects for this batch. - - """ - try: - process_names = self._extract_process_names_from_batch(batch) - - self.logger.debug( - f"{LOG_PREFIX} Batch {batch_idx}: Found {len(process_names)} unique process names" - ) - - data = self._fetch_data_for_time_window(batch) - self.logger.info( - f"{LOG_PREFIX} Batch {batch_idx}: Fetched {len(data)} data from time window" - ) - - results = self._match_data_to_expectations(batch, data, detection_helper) - - return results - - except Exception as e: - raise TemplateExpectationError( - f"Error processing batch {batch_idx}: {e}" - ) from e - - def _extract_end_date_from_batch( - self, batch: list[DetectionExpectation | PreventionExpectation] | None = None - ) -> datetime | None: - """Extract end_date from batch signatures. - - Args: - batch: Batch of expectations to extract end_date from. - - Returns: - end_date as datetime or None if no end_date signature found. - - """ - end_date = SignatureExtractor.extract_end_date(batch) - if end_date: - self.logger.debug( - f"{LOG_PREFIX} Extracted end_date from signatures: {end_date}, start_date will be calculated from time_window" - ) - return end_date - - def _fetch_data_for_time_window( - self, batch: list[DetectionExpectation | PreventionExpectation] | None = None - ) -> list[TemplateData]: - """Fetch alldata from the configured time window or date signatures. - - Args: - batch: Optional batch of expectations to extract date filters from. - - Returns: - List of TemplateData objects from the time window. - - Raises: - TemplateFetcherError: If fetcher fails. - - """ - try: - end_time = self._extract_end_date_from_batch(batch) - - if end_time is None: - end_time = datetime.now(timezone.utc) - - start_time = end_time - self.time_window - - self.logger.debug( - f"{LOG_PREFIX} Delegating data fetching to FetcherData for time window: {start_time} to {end_time}" - ) - - return self.data_fetcher.fetch_data_for_time_window( - start_time=start_time, - end_time=end_time, - limit=1000, - ) - - except Exception as e: - raise TemplateFetcherError( - f"Error fetching data for time window: {e}" - ) from e - - def _match_data_to_expectations( - self, - batch: list[DetectionExpectation | PreventionExpectation], - data: list[TemplateData], - detection_helper: Any, - ) -> list[ExpectationResult]: - """Match data to expectations and create results. - - Args: - batch: Batch of expectations. - data: List of filtered data. - detection_helper: OpenAEV detection helper. - - Returns: - List of ExpectationResult objects. - - """ - results = [] - - for expectation in batch: - try: - traces = [] - - for single_data in data: - if self._expectation_matches_data( - expectation, single_data, detection_helper - ): - trace = TraceBuilder.create_data_trace(data) - traces.append(trace) - - if isinstance(expectation, PreventionExpectation): - self.logger.debug( - f"{LOG_PREFIX} Prevention expectation {expectation.inject_expectation_id}: " - f"data {data} matched signature and is mitigated -> expectation satisfied" - ) - break - else: - self.logger.debug( - f"{LOG_PREFIX} Detection expectation {expectation.inject_expectation_id}: " - f"data {data} matched signature -> expectation satisfied" - ) - break - - result_dict = { - "is_valid": True, - "traces": traces, - "expectation_type": ( - "detection" - if isinstance(expectation, DetectionExpectation) - else "prevention" - ), - } - - result = self._convert_dict_to_result(result_dict, expectation) - results.append(result) - - self.logger.debug( - f"{LOG_PREFIX} Expectation {expectation.inject_expectation_id}: " - f"matched=true, traces={len(traces)}" - ) - - except Exception as e: - self.logger.error( - f"{LOG_PREFIX} Error matching expectation {expectation.inject_expectation_id}: {e}" - ) - error_result = self._create_error_result_object( - TemplateExpectationError(f"Matching error: {e}"), expectation - ) - results.append(error_result) - - return results - - def _expectation_matches_data( - self, - expectation: DetectionExpectation | PreventionExpectation, - data: TemplateData, - detection_helper: Any, - ) -> bool: - """Check if an expectation matches the given data using converter and detection helper. - - Args: - expectation: The expectation to match. - data: The data. - detection_helper: OpenAEV detection helper for matching. - - Returns: - True if the expectation matches, False otherwise. - - """ - try: - oaev_data_list = self.converter.convert_data_to_oaev([data]) - - if not oaev_data_list: - self.logger.debug( - f"{LOG_PREFIX} No OAEV data generated for data {data}" - ) - return False - - oaev_data = oaev_data_list[0] - - supported_signatures = self.get_supported_signatures() - self.logger.debug( - f"{LOG_PREFIX} Supported signature types: {[s.value for s in supported_signatures]}" - ) - - signature_groups = SignatureExtractor.group_signatures_by_type( - expectation, supported_signatures - ) - self.logger.debug( - f"{LOG_PREFIX} Filtered signature groups: {list(signature_groups.keys())}" - ) - - supported_sig_names = { - sig_type.value if hasattr(sig_type, "value") else str(sig_type) - for sig_type in supported_signatures - } - filtered_oaev_data = { - key: value - for key, value in oaev_data.items() - if key in supported_sig_names - } - self.logger.debug( - f"{LOG_PREFIX} Available OAEV data: {list(oaev_data.keys())}" - ) - self.logger.debug( - f"{LOG_PREFIX} Filtered OAEV data: {list(filtered_oaev_data.keys())}" - ) - - for sig_type, signatures in signature_groups.items(): - filtered_data = {sig_type: filtered_oaev_data[sig_type]} - self.logger.debug( - f"{LOG_PREFIX} Detection helper input - sig_type: {sig_type}" - ) - self.logger.debug( - f"{LOG_PREFIX} Detection helper input - signatures: {signatures}" - ) - self.logger.debug( - f"{LOG_PREFIX} Detection helper input - filtered_data: {filtered_data}" - ) - - # breakpoint() - match_result = detection_helper.match_alert_elements( - signatures, filtered_data - ) - - self.logger.debug( - f"{LOG_PREFIX} Detection helper result for {sig_type}: {match_result}" - ) - - if not match_result: - self.logger.debug( - f"{LOG_PREFIX} {sig_type} signature failed for data {data}" - ) - return False - - self.logger.debug( - f"{LOG_PREFIX} All signatures matched for expectation {expectation.inject_expectation_id} vs data {data}" - ) - return True - - except Exception as e: - self.logger.warning(f"{LOG_PREFIX} Error in expectation matching: {e}") - return False - - def _create_error_result_object( - self, - error: Exception, - expectation: DetectionExpectation | PreventionExpectation, - ) -> ExpectationResult: - """Create an error result object. - - Args: - error: The error that occurred. - expectation: The expectation that failed. - - Returns: - ExpectationResult object representing the error. - - """ - return ExpectationResult( - expectation_id=str(expectation.inject_expectation_id), - is_valid=False, - expectation=expectation, - matched_alerts=None, - error_message=str(error), - processing_time=None, - ) - - def _convert_dict_to_result( - self, - result_dict: dict[str, Any], - expectation: DetectionExpectation | PreventionExpectation, - ) -> ExpectationResult: - """Convert result dictionary to ExpectationResult object. - - Args: - result_dict: Dictionary containing result data. - expectation: The associated expectation. - - Returns: - ExpectationResult object. - - """ - return ExpectationResult( - expectation_id=str(expectation.inject_expectation_id), - is_valid=result_dict.get("is_valid", False), - expectation=expectation, - matched_alerts=result_dict.get("traces", []), - error_message=result_dict.get("error"), - processing_time=None, - ) - - def get_service_info(self) -> dict[str, Any]: - """Get service information. - - Returns: - Dictionary containing service information. - - """ - return { - "service_name": "TemplateExpectationService", - "batch_size": self.batch_size, - "supported_signatures": self.get_supported_signatures(), - "flow_type": "batch_based", - } diff --git a/template/src/services/fetcher_data.py b/template/src/services/fetcher_data.py deleted file mode 100644 index 5cfb8ca8..00000000 --- a/template/src/services/fetcher_data.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Template Data Fetcher.""" - -import logging -from datetime import datetime - -from .exception import ( - TemplateFetcherError, - TemplateValidationError, -) -from .model_data import TemplateData - -LOG_PREFIX = "[TemplateDataFetcher]" - - -class FetcherData: - """Fetcher for Template data using time-window based queries.""" - - def __init__(self) -> None: - """Initialize the Threat fetcher.""" - self.logger = logging.getLogger(__name__) - self.logger.debug(f"{LOG_PREFIX} Data fetcher initialized") - - def fetch_data_for_time_window( - self, - start_time: datetime, - end_time: datetime, - limit: int = 1000, - ) -> list[TemplateData]: - """Fetch all data for a given time window. - - Args: - start_time: Start time as datetime object. - end_time: End time as datetime object. - limit: Maximum number of threats to fetch. - - Returns: - List of TemplateData objects. - - Raises: - TemplateFetcherError: If fetcher fails. - TemplateValidationError: If parameters are invalid. - - """ - if not isinstance(start_time, datetime) or not isinstance(end_time, datetime): - raise TemplateValidationError( - "start_time and end_time must be datetime objects" - ) - - if start_time >= end_time: - raise TemplateValidationError("start_time must be before end_time") - - if limit <= 0: - raise TemplateValidationError("limit must be positive") - - try: - self.logger.debug( - f"{LOG_PREFIX} Fetching data for time window: {start_time} to {end_time}" - ) - - data = [TemplateData()] - # to fill with the relevant data according to your collector - - self.logger.info(f"{LOG_PREFIX} Fetched {len(data)} data for time window") - return data - - except Exception as e: - raise TemplateFetcherError( - f"Error fetching data for time window: {e}" - ) from e diff --git a/template/src/services/model_data.py b/template/src/services/model_data.py deleted file mode 100644 index cb17ca6f..00000000 --- a/template/src/services/model_data.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Template Data Models.""" - -from typing import Optional - -from pydantic import BaseModel, Field - - -class TemplateData(BaseModel): - """Template data model.""" - - key: Optional[str] = Field(None, description="Example key value") - - def __str__(self) -> str: - """Detaield representation with key debugging information.""" - return f"TemplateData(key='{self.value}'" diff --git a/template/src/services/trace_service.py b/template/src/services/trace_service.py deleted file mode 100644 index e8389e27..00000000 --- a/template/src/services/trace_service.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Template Trace Service Provider.""" - -import logging -from datetime import UTC, datetime -from typing import Any - -from ..collector.models import ExpectationResult, ExpectationTrace -from ..models.settings.config_loader import ConfigLoader -from .exception import TemplateDataConversionError, TemplateValidationError - -LOG_PREFIX = "[TemplateTraceService]" - - -class TemplateTraceService: - """Template-specific trace service provider. - - This service extracts trace information from expectation processing results - and converts them into OpenAEV expectation traces using proper Pydantic models. - """ - - def __init__(self, config: ConfigLoader | None = None) -> None: - """Initialize the Template trace service. - - Args: - config: Configuration loader instance for trace service settings. - - Raises: - TemplateValidationError: If config is None. - - """ - if config is None: - raise TemplateValidationError("Config is required for trace service") - - self.logger = logging.getLogger(__name__) - self.config = config - self.logger.debug(f"{LOG_PREFIX} Template trace service initialized") - - def create_traces_from_results( - self, results: list[ExpectationResult], collector_id: str - ) -> list[ExpectationTrace]: - """Create trace data from processing results. - - Args: - results: List of expectation processing results. - collector_id: ID of the collector. - - Returns: - List of ExpectationTrace models for OpenAEV. - - Raises: - TemplateValidationError: If inputs are invalid. - TemplateDataConversionError: If trace creation fails. - - """ - if not collector_id: - raise TemplateValidationError("collector_id cannot be empty") - - if not isinstance(results, list): - raise TemplateValidationError("results must be a list") - - try: - valid_results = [r for r in results if r.is_valid and r.matched_alerts] - - if not valid_results: - self.logger.info( - f"{LOG_PREFIX} No valid results with matching data for traces out of {len(results)} results" - ) - return [] - - self.logger.info( - f"{LOG_PREFIX} Creating traces for {len(valid_results)} valid results out of {len(results)} total" - ) - - traces = [] - - for i, result in enumerate(valid_results, 1): - expectation_id = result.expectation_id - if not expectation_id: - self.logger.warning( - f"{LOG_PREFIX} Skipping result {i} - missing expectation_id" - ) - continue - - self.logger.debug( - f"{LOG_PREFIX} Creating trace {i}/{len(valid_results)} for expectation {expectation_id}" - ) - - try: - trace = self._create_expectation_trace( - result, expectation_id, collector_id - ) - - if trace: - traces.append(trace) - self.logger.debug( - f"{LOG_PREFIX} Created trace for expectation {expectation_id}: {trace.inject_expectation_trace_alert_name}" - ) - else: - self.logger.warning( - f"{LOG_PREFIX} Trace creation returned None for expectation {expectation_id}" - ) - except Exception as e: - raise TemplateDataConversionError( - f"Error creating trace for expectation {expectation_id}: {e}" - ) from e - - self.logger.info( - f"{LOG_PREFIX} Successfully created {len(traces)} traces from {len(valid_results)} valid results" - ) - return traces - - except TemplateDataConversionError: - raise - except Exception as e: - raise TemplateDataConversionError( - f"Unexpected error creating traces from results: {e}" - ) from e - - def _create_expectation_trace( - self, result: ExpectationResult, expectation_id: str, collector_id: str - ) -> ExpectationTrace: - """Create ExpectationTrace model from a single result. - - Args: - result: Processing result dictionary. - expectation_id: ID of the expectation. - collector_id: ID of the collector. - - Returns: - ExpectationTrace model for OpenAEV. - - Raises: - TemplateValidationError: If inputs are invalid. - TemplateDataConversionError: If trace creation fails. - - """ - if not expectation_id: - raise TemplateValidationError("expectation_id cannot be empty") - - if not collector_id: - raise TemplateValidationError("collector_id cannot be empty") - - if not result.matched_alerts: - raise TemplateValidationError( - "result must have matched_alerts for trace creation" - ) - - try: - matching_data = result.matched_alerts[0] or {} - self.logger.debug( - f"{LOG_PREFIX} Processing matching data with {len(matching_data)} fields" - ) - - alert_name = matching_data.get("alert_name", "Template Alert") - - trace_link = matching_data.get("alert_link", "") - self.logger.debug(f"{LOG_PREFIX} Using trace builder URL: {trace_link}") - - trace_date = datetime.now(UTC).replace(microsecond=0) - date_str = trace_date.isoformat().replace("+00:00", "Z") - self.logger.debug(f"{LOG_PREFIX} Generated trace date: {date_str}") - - trace = ExpectationTrace( - inject_expectation_trace_expectation=str(expectation_id), - inject_expectation_trace_source_id=str(collector_id), - inject_expectation_trace_alert_name=alert_name, - inject_expectation_trace_alert_link=trace_link, - inject_expectation_trace_date=date_str, - ) - - self.logger.debug( - f"{LOG_PREFIX} Created ExpectationTrace with alert name: {alert_name}" - ) - return trace - - except TemplateValidationError: - raise - except Exception as e: - raise TemplateDataConversionError( - f"Error creating expectation trace: {e}" - ) from e - - def get_service_info(self) -> dict[str, Any]: - """Get information about this trace service. - - Returns: - Dictionary containing service metadata and capabilities. - - """ - info = { - "service_type": "template_trace", - "supported_result_types": ["Template processing results"], - "creates_detection_traces": True, - "creates_prevention_traces": True, - "description": "Creates traces from Template expectation processing results using trace builder URLs", - } - self.logger.debug(f"{LOG_PREFIX} Trace service info: {info}") - return info diff --git a/template/src/services/utils/__init__.py b/template/src/services/utils/__init__.py deleted file mode 100644 index def93d4a..00000000 --- a/template/src/services/utils/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from src.services.utils.config_loader import TemplateConfig -from src.services.utils.signature_extractor import SignatureExtractor -from src.services.utils.trace_builder import TraceBuilder - -__all__ = [ - "TemplateConfig", - "SignatureExtractor", - "TraceBuilder", -] diff --git a/template/src/services/utils/config_loader.py b/template/src/services/utils/config_loader.py deleted file mode 100644 index 8bac7298..00000000 --- a/template/src/services/utils/config_loader.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Configuration loader.""" - -import logging - -from pydantic import ValidationError -from src.models import ConfigLoader - -LOG_PREFIX = "[CollectorConfig]" - - -class TemplateConfig: - """Class for loading Template configuration.""" - - def __init__(self) -> None: - """Initialize Template configuration loader. - - Loads configuration from YAML files, environment variables, and defaults. - Sets up logging and validates the configuration structure. - - Raises: - ValueError: If configuration loading or validation fails. - - """ - self.logger = logging.getLogger(__name__) - self.logger.debug(f"{LOG_PREFIX} Initializing Template configuration loader") - self.load = self._load_config() - self.logger.info(f"{LOG_PREFIX} Template configuration loaded successfully") - - def _load_config(self) -> ConfigLoader: - """Load configuration with proper error handling and logging. - - Loads configuration from multiple sources and validates the structure. - Logs configuration details for debugging purposes. - - Returns: - ConfigLoader instance with validated configuration. - - Raises: - ValueError: If configuration validation or loading fails. - - """ - try: - self.logger.debug( - f"{LOG_PREFIX} Loading configuration from sources (YAML/ENV/defaults)" - ) - load_settings = ConfigLoader() - - self.logger.debug( - f"{LOG_PREFIX} Collector ID: {load_settings.collector.id}" - ) - self.logger.debug( - f"{LOG_PREFIX} Collector name: {load_settings.collector.name}" - ) - self.logger.debug( - f"{LOG_PREFIX} Log level: {load_settings.collector.log_level}" - ) - self.logger.debug(f"{LOG_PREFIX} OpenAEV URL: {load_settings.openaev.url}") - self.logger.debug( - f"{LOG_PREFIX} Template key: {load_settings.template.key}" - ) - - return load_settings - except ValidationError as err: - self.logger.error( - f"{LOG_PREFIX} Error in configuration validation: {err} (Context: error_type=ValidationError)" - ) - raise ValueError(f"Configuration validation failed: {err}") from err - except Exception as err: - self.logger.error( - f"{LOG_PREFIX} Error in configuration loading: {err} (Context: error_type={type(err).__name__})" - ) - raise ValueError(f"Configuration loading failed: {err}") from err diff --git a/template/src/services/utils/signature_extractor.py b/template/src/services/utils/signature_extractor.py deleted file mode 100644 index 77bd203f..00000000 --- a/template/src/services/utils/signature_extractor.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Signature extraction utilities for Template expectation processing.""" - -from datetime import datetime, timezone -from typing import TYPE_CHECKING - -from pyoaev.signatures.types import SignatureTypes - -if TYPE_CHECKING: - from pyoaev.apis.inject_expectation.model.expectation import ( - DetectionExpectation, - PreventionExpectation, - ) - - -class SignatureExtractor: - """Utility class for extracting signatures from expectations.""" - - @staticmethod - def extract_hostnames( - batch: list["DetectionExpectation | PreventionExpectation"], - ) -> list[str]: - """Extract unique hostnames from a batch of expectations. - - Args: - batch: List of expectations to extract hostnames from. - - Returns: - List of unique hostname values. - - """ - hostnames = set() - for expectation in batch: - for signature in expectation.inject_expectation_signatures: - if signature.type == SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS: - hostnames.add(signature.value) - return list(hostnames) - - @staticmethod - def extract_process_names( - batch: list["DetectionExpectation | PreventionExpectation"], - ) -> list[str]: - """Extract unique parent process names from a batch of expectations. - - Args: - batch: List of expectations to extract process names from. - - Returns: - List of unique parent process name values. - - """ - process_names = set() - for expectation in batch: - for signature in expectation.inject_expectation_signatures: - if signature.type.value == "parent_process_name": - process_names.add(signature.value) - return list(process_names) - - @staticmethod - def extract_end_date( - batch: list["DetectionExpectation | PreventionExpectation"] | None = None, - ) -> datetime | None: - """Extract end_date from batch signatures. - - Args: - batch: List of expectations to extract end_date from. If None, returns None. - - Returns: - Parsed end_date as datetime or None if no valid end_date signature found. - - """ - if not batch: - return None - - for expectation in batch: - for signature in expectation.inject_expectation_signatures: - if signature.type.value == "end_date": - try: - end_date = datetime.fromisoformat( - signature.value.replace("Z", "+00:00") - ) - if end_date.tzinfo is None: - end_date = end_date.replace(tzinfo=timezone.utc) - return end_date - except (ValueError, AttributeError): - continue - return None - - @staticmethod - def group_signatures_by_type( - expectation: "DetectionExpectation | PreventionExpectation", - supported_signatures: list[SignatureTypes] | None = None, - ) -> dict[str, list[dict[str, str]]]: - """Group signatures by type for detection helper matching. - - Args: - expectation: Single expectation to group signatures from. - supported_signatures: List of supported signature types to filter by. - If None, all signature types are included. - - Returns: - Dictionary mapping signature types to lists of signature dictionaries - in the format expected by detection helper (with 'value' and 'type' keys). - Only includes signature types that are in the supported list. - Excludes end_date as it's only used for query criteria, not matching. - - """ - supported_types = None - if supported_signatures: - supported_types = { - sig_type.value if hasattr(sig_type, "value") else str(sig_type) - for sig_type in supported_signatures - } - - signature_groups = {} - for sig in expectation.inject_expectation_signatures: - sig_type = sig.type.value if hasattr(sig.type, "value") else str(sig.type) - - if supported_types and sig_type not in supported_types: - continue - - if sig_type == "end_date": - continue - - if sig_type not in signature_groups: - signature_groups[sig_type] = [] - signature_groups[sig_type].append({"type": sig_type, "value": sig.value}) - return signature_groups diff --git a/template/src/services/utils/trace_builder.py b/template/src/services/utils/trace_builder.py deleted file mode 100644 index e2f79551..00000000 --- a/template/src/services/utils/trace_builder.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Trace building utilities for Template expectation processing.""" - -import logging -from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from ..model_data import TemplateData - -LOG_PREFIX = "[TemplateTraceBuilder]" - - -class TraceBuilder: - """Utility class for building trace information.""" - - @staticmethod - def create_data_trace( - data: "TemplateData", - ) -> dict[str, Any]: - """Create trace information for a data. - - Args: - data: Template data object. - - Returns: - Dictionary containing trace information with alert name, link, date, - and additional metadata. - - """ - logger = logging.getLogger(__name__) - - alert_name = "Template Alert" - alert_link = "http://foo.bar" - - trace_data = { - "alert_name": alert_name, - "alert_link": alert_link, - "alert_date": datetime.now(timezone.utc).isoformat(), - "additional_data": { - "data_key_value": data.key, - "data_source": "template", - }, - } - - logger.debug(f"{LOG_PREFIX} Created trace data: {trace_data}") - return trace_data diff --git a/template/tests/services/fixtures/__init__.py b/template/src/source/__init__.py similarity index 100% rename from template/tests/services/fixtures/__init__.py rename to template/src/source/__init__.py diff --git a/template/src/source/template_data_fetcher.py b/template/src/source/template_data_fetcher.py new file mode 100644 index 00000000..47dad26f --- /dev/null +++ b/template/src/source/template_data_fetcher.py @@ -0,0 +1,11 @@ +from src.source.template_source_data import TemplateSourceData + + +class TemplateDataFetcher: + """ + Placeholder data fetcher class, meant to follow the data fetcher protocol + """ + + def fetch_data(self): + """return placeholder data in the source data format""" + return [TemplateSourceData(), TemplateSourceData(), TemplateSourceData()] diff --git a/template/src/source/template_signatures.py b/template/src/source/template_signatures.py new file mode 100644 index 00000000..f02a34f8 --- /dev/null +++ b/template/src/source/template_signatures.py @@ -0,0 +1,7 @@ +from pyoaev.signatures.types import SignatureTypes + +SUPPORTED_SIGNATURES = [ + SignatureTypes.SIG_TYPE_END_DATE, + SignatureTypes.SIG_TYPE_START_DATE, + SignatureTypes.SIG_TYPE_PARENT_PROCESS_NAME, +] diff --git a/template/src/source/template_source_data.py b/template/src/source/template_source_data.py new file mode 100644 index 00000000..f3f9a68c --- /dev/null +++ b/template/src/source/template_source_data.py @@ -0,0 +1,35 @@ +import secrets + +from src.collector.models.data import OAEVData, TraceData + + +class TemplateSourceData: + """ + Placeholder source data, meant to follow the source data protocol + """ + + def __init__(self): + """Generate random placeholder data""" + self.value = secrets.token_hex(8) + + def to_oaev_data(self): + """Serialize source data into OAEVData""" + return OAEVData(parent_process_name=f"{self.value}") + + def to_traces_data(self): + """Serialize traces data into TraceData""" + return TraceData( + alert_name=f"Alert {self.value}", alert_link=f"http://fake.url/{self.value}" + ) + + def is_prevented(self): + """Placeholder analysis of the data to determine if the threat is prevented""" + return bool(secrets.randbits(1)) + + def is_detected(self): + """Placeholder analysis of the data to determine if the threat is detected""" + return bool(secrets.randbits(1)) + + def __str__(self): + """Str output of the source data for logging purposes""" + return f"{self.value}" diff --git a/template/src/template_collector.py b/template/src/template_collector.py new file mode 100644 index 00000000..6843bed5 --- /dev/null +++ b/template/src/template_collector.py @@ -0,0 +1,37 @@ +""" +Template collector meant to easier collector development, +based on the distinction between the normalized collector engine +and the custom source related to the implemented tool/service +""" + +import os +import sys + +from src.collector.collector import BaseCollector +from src.collector.models.source import Source +from src.source.template_data_fetcher import TemplateDataFetcher +from src.source.template_signatures import SUPPORTED_SIGNATURES +from src.source.template_source_data import TemplateSourceData + + +def main() -> None: + """ + defining a source, feeding it into the base collector, + then starting said collector + """ + try: + source = Source( + data_fetcher_model=TemplateDataFetcher, + source_data_model=TemplateSourceData, + signatures=SUPPORTED_SIGNATURES, + ) + base_collector = BaseCollector( + name="Template collector", + source=source, + ) + base_collector.start() + except KeyboardInterrupt: + os._exit(0) + except Exception as err: + print(err) + sys.exit(1) diff --git a/template/tests/collector/__init__.py b/template/tests/collector/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/template/tests/collector/test_collector.py b/template/tests/collector/test_collector.py new file mode 100644 index 00000000..4b489269 --- /dev/null +++ b/template/tests/collector/test_collector.py @@ -0,0 +1,132 @@ +import unittest +from unittest.mock import MagicMock, patch + +import src.collector.collector as module + +daemon_config_data = { + "openaev_url": "http://fake.url", + "openaev_token": "my_awesome_token", +} + + +@patch.object(module, "BasicCollectorEngine", spec_set=module.BasicCollectorEngine) +@patch.object(module, "ConfigLoader") +class TestBaseCollector(unittest.TestCase): + def test_minimal_init(self, m_configloader, m_basiccollectorengine): + """""" + m_configloader.return_value.to_daemon_config.return_value = daemon_config_data + name = "my collector" + source = MagicMock(spec_set=module.Source) + + collector = module.BaseCollector(name=name, source=source) + + self.assertEqual(collector.name, name) + self.assertEqual(collector.source, source) + self.assertIsNotNone(collector.source_handler) + self.assertIsNotNone(collector.engine) + self.assertIsNotNone(collector.api) + m_configloader.return_value.to_daemon_config.assert_called_once() + m_basiccollectorengine.assert_called_with( + name=name, + collector_id=collector.get_id(), + source=collector.source, + source_handler=collector.source_handler, + oaev_api=collector.api, + ) + + def test_init_with_custom_handler(self, m_configloader, m_basiccollectorengine): + """""" + m_configloader.return_value.to_daemon_config.return_value = daemon_config_data + name = "my collector" + source = MagicMock(spec_set=module.Source) + source_handler = MagicMock(spec_set=module.SourceHandler) + + collector = module.BaseCollector( + name=name, source=source, source_handler=source_handler + ) + + self.assertEqual(collector.name, name) + self.assertEqual(collector.source, source) + self.assertEqual(source_handler, collector.source_handler) + self.assertIsNotNone(collector.engine) + self.assertIsNotNone(collector.api) + m_configloader.return_value.to_daemon_config.assert_called_once() + m_basiccollectorengine.assert_called_with( + name=name, + collector_id=collector.get_id(), + source=collector.source, + source_handler=collector.source_handler, + oaev_api=collector.api, + ) + + def test_init_with_wrong_handler(self, m_configloader, m_basiccollectorengine): + """""" + m_configloader.return_value.to_daemon_config.return_value = daemon_config_data + name = "my collector" + source = MagicMock(spec_set=module.Source) + source_handler = MagicMock() + + with self.assertRaises(module.CollectorConfigError): + module.BaseCollector( + name=name, source=source, source_handler=source_handler + ) + + def test_init_with_alternative_engine(self, m_configloader, m_basiccollectorengine): + """""" + + class NoRunEngine: + def __init__(self, **kwargs): + pass + + def configure_engine(self, config, batching=False): + pass + + def run_engine(self): + pass + + m_configloader.return_value.to_daemon_config.return_value = daemon_config_data + name = "my collector" + source = MagicMock(spec_set=module.Source) + engine_model = NoRunEngine + + collector = module.BaseCollector( + name=name, source=source, engine_model=engine_model + ) + + self.assertEqual(collector.name, name) + self.assertEqual(collector.source, source) + self.assertEqual(engine_model, collector.engine_model) + self.assertIsNotNone(collector.engine) + self.assertIsNotNone(collector.api) + m_configloader.return_value.to_daemon_config.assert_called_once() + + def test_init_with_wrong_engine(self, m_configloader, m_basiccollectorengine): + """""" + m_configloader.return_value.to_daemon_config.return_value = daemon_config_data + name = "my collector" + source = MagicMock(spec_set=module.Source) + engine_model = MagicMock() + + with self.assertRaises(module.CollectorConfigError): + module.BaseCollector(name=name, source=source, engine_model=engine_model) + + @patch.object(module.CollectorDaemon, "_setup") + def test_setup( + self, m_collectordaemon_setup, m_configloader, m_basiccollectorengine + ): + """""" + m_configloader.return_value.to_daemon_config.return_value = daemon_config_data + name = "my collector" + source = MagicMock(spec_set=module.Source) + source_handler = MagicMock(spec_set=module.SourceHandler) + batching = True + + collector = module.BaseCollector( + name=name, source=source, source_handler=source_handler + ) + collector._setup(batching=batching) + + m_collectordaemon_setup.assert_called_once() + m_basiccollectorengine.return_value.configure_engine.assert_called_with( + collector.config.template, batching=batching + ) diff --git a/template/tests/conftest.py b/template/tests/conftest.py deleted file mode 100644 index 4b52f820..00000000 --- a/template/tests/conftest.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Conftest file for Pytest fixtures.""" - -from typing import TYPE_CHECKING, Any -from unittest.mock import patch - -from pytest import fixture - -if TYPE_CHECKING: - from os import _Environ - - -def mock_env_vars(os_environ: "_Environ[str]", wanted_env: dict[str, str]) -> Any: - """Fixture to mock environment variables dynamically and clean up after. - - Args: - os_environ: The os.environ object to patch. - wanted_env: Dictionary of environment variables to mock. - - Returns: - Mock object for environment variable patching. - - """ - mock_env = patch.dict(os_environ, wanted_env) - mock_env.start() - - return mock_env - - -@fixture(autouse=True) -def mock_openaev_client() -> Any: - """Fixture to mock OpenAEV calls and clean up after. - - Auto-applies to all tests to prevent actual OpenAEV API calls. - Mocks urllib3, pyoaev client, and collector daemon setup. - - Yields: - Tuple of mock objects (urllib, pyoaev, daemon_setup). - - """ - mock_urllib = patch("urllib3.connectionpool.HTTPConnectionPool.urlopen") - mock_pyoaev = patch("pyoaev.client.OpenAEV.http_request") - mock_daemon_setup = patch("pyoaev.daemons.collector_daemon.CollectorDaemon._setup") - - mock_urllib.start() - mock_pyoaev.start() - mock_daemon_setup.start() - - yield mock_urllib, mock_pyoaev, mock_daemon_setup - - mock_urllib.stop() - mock_pyoaev.stop() - mock_daemon_setup.stop() - - -@fixture(autouse=True) -def disable_config_yml() -> Any: - """Fixture to disable config.yml and .env files for tests, forcing environment variable usage only. - - Auto-applies to all tests to ensure consistent configuration loading - from environment variables instead of config files. - - Yields: - Patcher object for the settings customization. - - """ - - def fake_settings_customise_sources( - cls, - settings_cls, - init_settings, - env_settings, - dotenv_settings, - file_secret_settings, - ): - from pydantic_settings import EnvSettingsSource - - # Return only environment settings source, ignoring files - return ( - EnvSettingsSource( - settings_cls, - env_ignore_empty=True, - ), - ) - - patcher = patch( - "src.models.configs.config_loader.ConfigLoader.settings_customise_sources", - new=classmethod(fake_settings_customise_sources), - ) - patcher.start() - - yield patcher - - patcher.stop() diff --git a/template/tests/gwt_shared.py b/template/tests/gwt_shared.py deleted file mode 100644 index 2f541ba4..00000000 --- a/template/tests/gwt_shared.py +++ /dev/null @@ -1,564 +0,0 @@ -"""Shared Given-When-Then methods for Gherkin-style tests. - -This module provides reusable Given, When, and Then methods that can be used -across multiple test files to maximize code reusability and maintain consistency -in test structure. -""" - -from os import environ as os_environ -from typing import Any, Dict, List -from unittest.mock import Mock - -import pytest -from src.collector import Collector -from src.services.converter import TemplateConverter -from src.services.exception import TemplateValidationError -from src.services.expectation_service import TemplateExpectationService -from src.services.model_data import TemplateData -from tests.conftest import mock_env_vars -from tests.services.fixtures.factories import create_test_config - -# ======================================================================== -# SHARED GIVEN METHODS - Set up test preconditions -# ======================================================================== - - -# Configuration Setup Given Methods -# --------------------------------- - - -def given_valid_collector_config(config_data: Dict[str, str]) -> Any: - """Set up valid collector configuration environment. - - Args: - config_data: Dictionary of environment variables to mock. - - Returns: - Mock environment variable patcher object. - - """ - return mock_env_vars(os_environ, config_data) - - -def given_test_config(): - """Create a standard test configuration. - - Returns: - Test configuration object. - - """ - return create_test_config() - - -def given_config_missing_field( - base_config: Dict[str, str], field_name: str -) -> Dict[str, str]: - """Create configuration with a missing required field. - - Args: - base_config: Base configuration dictionary. - field_name: Name of the field to remove. - - Returns: - Configuration dictionary without the specified field. - - """ - config = base_config.copy() - config.pop(field_name, None) - - if field_name in os_environ: - del os_environ[field_name] - - return config - - -def given_config_with_invalid_value( - base_config: Dict[str, str], field_name: str, invalid_value: str -) -> Dict[str, str]: - """Create configuration with an invalid field value. - - Args: - base_config: Base configuration dictionary. - field_name: Name of the field to modify. - invalid_value: Invalid value to set. - - Returns: - Configuration dictionary with invalid value. - - """ - config = base_config.copy() - config[field_name] = invalid_value - return config - - -# Object Creation Given Methods -# ----------------------------- - - -def given_initialized_converter() -> TemplateConverter: - """Create an initialized converter. - - Returns: - Initialized TemplateConverter instance. - - """ - return TemplateConverter() - - -def given_initialized_expectation_service(): - """Create an initialized expectation service. - - Returns: - Initialized TemplateExpectationService instance. - - """ - config = given_test_config() - return TemplateExpectationService(config=config) - - -# Data Creation Given Methods -# --------------------------- - - -def given_data_with_complete_data(key: str = "value123") -> TemplateData: - """Create a data with complete data. - - Args: - key: example value. - - Returns: - TemplateData with complete data. - - """ - return TemplateData(key=key) - - -def given_data_with_empty_key() -> TemplateData: - """Create a data with empty key. - - Returns: - TemplateData with empty key. - - """ - return TemplateData(key="") - - -def given_multiple_data(count: int = 3) -> List[TemplateData]: - """Create multiple data with different data combinations. - - Args: - count: Number of data to create. - - Returns: - List of TemplateData objects. - - """ - data = [] - for i in range(count): - key = f"multi_data_{i + 1}" - data.append(TemplateData(key=key)) - return data - - -def given_large_batch_of_data(count: int = 100) -> List[TemplateData]: - """Create a large batch of data for performance testing. - - Args: - count: Number of data to create. - - Returns: - List of TemplateData objects. - - """ - return [ - TemplateData( - key=f"bulk_data_{i}", - ) - for i in range(count) - ] - - -def given_mixed_valid_invalid_objects(valid_data_key: str = "valid_mixed_123") -> List: - """Create a list with mixed valid and invalid objects. - - Args: - valid_data_key: key for the valid data in the list. - - Returns: - List containing valid data and invalid objects. - - """ - valid_data = TemplateData( - key=valid_data_key, - ) - return [ - valid_data, - {"key_data": "dict_data"}, - "string_data", - 42, - ] - - -def given_invalid_input_data() -> str: - """Create invalid input data for testing. - - Returns: - Invalid data (string instead of list). - - """ - return "invalid_string_data" - - -# Mock Setup Given Methods -# ------------------------ - - -def given_conversion_error_setup(converter: TemplateConverter) -> List: - """Set up data that will cause conversion errors. - - Args: - converter: The converter instance to mock. - - Returns: - List with valid data and error-causing mock data. - - """ - error_data = Mock(spec=TemplateData) - error_data.key = "error_data" - - valid_data = TemplateData(key="valid_error_test_123") - - original_convert = converter._convert_data_to_oaev - - def mock_convert(data): - if hasattr(data, "key") and data.key == "error_data": - raise Exception("Conversion error") - return original_convert(data) - - converter._convert_data_to_oaev = mock_convert - - return [error_data, valid_data] - - -# ======================================================================== -# SHARED WHEN METHODS - Execute actions being tested -# ======================================================================== - - -# Object Creation When Methods -# ---------------------------- - - -def when_create_collector() -> Collector: - """Create a collector instance. - - Returns: - Collector instance. - - """ - return Collector() - - -def when_initialize_converter() -> TemplateConverter: - """Initialize a converter. - - Returns: - Initialized TemplateConverter instance. - - """ - return TemplateConverter() - - -def when_initialize_expectation_service(): - """Initialize an expectation service. - - Returns: - Initialized TemplateExpectationService instance. - - """ - config = given_test_config() - return TemplateExpectationService(config=config) - - -# Data Processing When Methods -# ---------------------------- - - -def when_convert_data_to_oaev(converter: TemplateConverter, data: List) -> List: - """Convert data to OAEV format. - - Args: - converter: The converter instance. - data: List of data to convert. - - Returns: - List of converted OAEV format data. - - """ - return converter.convert_data_to_oaev(data) - - -def when_call_private_conversion_method( - converter: TemplateConverter, data: TemplateData -) -> Dict: - """Call the private conversion method directly. - - Args: - converter: The converter instance. - data: The data to convert. - - Returns: - Converted OAEV format dictionary. - - """ - return converter._convert_data_to_oaev(data) - - -# Error Handling When Methods -# --------------------------- - - -def when_create_collector_expecting_error(mock_env: Any) -> None: - """Attempt to create collector and expect configuration error. - - Args: - mock_env: Mock environment variable patcher to clean up. - - """ - try: - with pytest.raises((Exception, ValueError)): - when_create_collector() - finally: - mock_env.stop() - - -def when_convert_invalid_data_expecting_validation_error( - converter: TemplateConverter, invalid_data: Any -) -> None: - """Attempt to convert invalid data and expect validation error. - - Args: - converter: The converter instance. - invalid_data: Invalid input data. - - """ - with pytest.raises(TemplateValidationError) as exc_info: - converter.convert_data_to_oaev(invalid_data) - - assert "data must be a list" in str(exc_info.value) # noqa: S101 - - -# ======================================================================== -# SHARED THEN METHODS - Validate results and assert expectations -# ======================================================================== - - -# Object Validation Then Methods -# ------------------------------ - - -def then_collector_created_successfully(collector: Collector) -> None: - """Verify collector was created successfully. - - Args: - collector: The collector instance to verify. - - """ - assert collector is not None # noqa: S101 - assert hasattr(collector, "config_instance") # noqa: S101 - - -def then_collector_has_valid_configuration( - collector: Collector, expected_config: Dict[str, str] -) -> None: - """Verify collector has valid configuration. - - Args: - collector: The collector instance to verify. - expected_config: Expected configuration values. - - """ - daemon_config = collector.config_instance.to_daemon_config() - - assert daemon_config.get("openaev_url") == expected_config.get( # noqa: S101 - "OPENAEV_URL" - ) - assert daemon_config.get("openaev_token") == expected_config.get( # noqa: S101 - "OPENAEV_TOKEN" - ) - assert daemon_config.get("collector_id") == expected_config.get( # noqa: S101 - "COLLECTOR_ID" - ) - assert daemon_config.get("collector_name") == expected_config.get( # noqa: S101 - "COLLECTOR_NAME" - ) - assert daemon_config.get("template_key") == expected_config.get( # noqa: S101 - "TEMPLATE_KEY" - ) - - -def then_converter_initialized_successfully(converter: TemplateConverter) -> None: - """Verify converter was initialized successfully. - - Args: - converter: The converter instance to verify. - - """ - assert converter is not None # noqa: S101 - assert converter.logger is not None # noqa: S101 - - -def then_expectation_service_initialized_successfully( - service: TemplateExpectationService, -) -> None: - """Verify expectation service was initialized successfully. - - Args: - service: The expectation service instance to verify. - - """ - assert service is not None # noqa: S101 - assert service.converter is not None # noqa: S101 - assert service.data_fetcher is not None # noqa: S101 - - -# Data Validation Then Methods -# ---------------------------- - - -def then_empty_list_returned(result: List) -> None: - """Verify an empty list was returned. - - Args: - result: The result to verify. - - """ - assert result == [] # noqa: S101 - - -def then_single_data_converted_completely(result: List, data: TemplateData) -> None: - """Verify single data was converted with all fields. - - Args: - result: The conversion result to verify. - data: The original data object. - - """ - assert len(result) == 1 # noqa: S101 - - converted = result[0] - - assert "key" in converted # noqa: S101 - assert converted["key"] == [data.key] # noqa: S101 - - -def then_multiple_data_converted(result: List, data: List[TemplateData]) -> None: - """Verify multiple data were converted correctly. - - Args: - result: The conversion result to verify. - data: The original data list. - - """ - valid_data = [d for d in data if d.key and d.key.strip()] - assert len(result) == len(valid_data) # noqa: S101 - - keys = [item["key"]["data"][0] for item in result] - for d in valid_data: - assert d.key in keys # noqa: S101 - - -def then_only_valid_data_converted(result: List, expected_valid_count: int = 1) -> None: - """Verify only valid data were converted from mixed data. - - Args: - result: The conversion result to verify. - expected_valid_count: Expected number of valid conversions. - - """ - assert len(result) == expected_valid_count # noqa: S101 - - -def then_large_batch_converted_efficiently(result: List, expected_count: int) -> None: - """Verify large batch was converted efficiently. - - Args: - result: The conversion result to verify. - expected_count: Expected number of converted items. - - """ - assert len(result) == expected_count # noqa: S101 - - converted_keys = {item["key"]["data"][0] for item in result} - assert len(converted_keys) == expected_count # noqa: S101 - - -def then_private_method_converts_properly(result: Dict, data: TemplateData) -> None: - """Verify private method converts data properly. - - Args: - result: The conversion result to verify. - data: The original data object. - - """ - assert isinstance(result, dict) # noqa: S101 - - -# Session and Configuration Validation Then Methods -# ------------------------------------------------- - - -def then_collector_logged_initialization_success( - capfd: Any, daemon_config: Dict[str, str] -) -> None: - """Verify collector initialization was logged appropriately. - - Args: - capfd: Pytest fixture for capturing stdout and stderr output. - daemon_config: Daemon configuration to check log level. - - """ - log_records = capfd.readouterr() - if daemon_config.get("collector_log_level") in ["info", "debug"]: - registered_message = "Template Collector initialized successfully" - assert registered_message in log_records.err # noqa: S101 - - -# Error Validation Then Methods -# ----------------------------- - - -def then_validation_error_raised_with_message(error_message: str) -> None: - """Verify validation error was raised with specific message. - - Args: - error_message: Expected error message substring. - - """ - pass - - -def then_session_error_raised_with_message(error_message: str) -> None: - """Verify session error was raised with specific message. - - Args: - error_message: Expected error message substring. - - """ - pass - - -# Cleanup Then Methods -# ------------------- - - -def then_cleanup_environment_mocks(*mock_envs: Any) -> None: - """Clean up environment variable mocks. - - Args: - mock_envs: Variable number of mock environment objects to stop. - - """ - for mock_env in mock_envs: - if mock_env and hasattr(mock_env, "stop"): - mock_env.stop() diff --git a/template/tests/services/conftest.py b/template/tests/services/conftest.py deleted file mode 100644 index d4425467..00000000 --- a/template/tests/services/conftest.py +++ /dev/null @@ -1,385 +0,0 @@ -"""Conftest for services tests with polyfactory fixtures.""" - -from unittest.mock import Mock, patch - -import pytest -from tests.services.fixtures.factories import ( - ConfigLoaderFactory, - ExpectationResultFactory, - ExpectationTraceFactory, - MockObjectsFactory, - TemplateDataFactory, - TestDataFactory, - create_test_config, -) - - -@pytest.fixture -def mock_config(): - """Provide a mock configuration for tests. - - Returns: - ConfigLoader instance with test configuration values. - - """ - return create_test_config() - - -@pytest.fixture -def mock_detection_helper(): - """Provide a mock detection helper that matches by default. - - Returns: - Mock OpenAEV detection helper that returns True for matches. - - """ - return MockObjectsFactory.create_mock_detection_helper(match_result=True) - - -@pytest.fixture -def mock_detection_helper_no_match(): - """Provide a mock detection helper that doesn't match. - - Returns: - Mock OpenAEV detection helper that returns False for matches. - - """ - return MockObjectsFactory.create_mock_detection_helper(match_result=False) - - -@pytest.fixture -def sample_single_data(): - """Provide a sample Template data. - - Returns: - TemplateData instance for testing. - - """ - return TemplateDataFactory.build() - - -@pytest.fixture -def sample_data(): - """Provide a list of sample TemplateData data. - - Returns: - List of 2 TemplateData instances for testing. - - """ - return [TemplateDataFactory.build() for _ in range(2)] - - -@pytest.fixture -def sample_expectation_result(): - """Provide a sample expectation result. - - Returns: - ExpectationResult instance for testing. - - """ - return ExpectationResultFactory.build() - - -@pytest.fixture -def sample_expectation_trace(): - """Provide a sample expectation trace. - - Returns: - ExpectationTrace instance for testing. - - """ - return ExpectationTraceFactory.build() - - -@pytest.fixture -def detection_signatures(): - """Provide sample detection expectation signatures. - - Returns: - List of signature dictionaries for detection expectations. - - """ - return TestDataFactory.create_expectation_signatures( - signature_type="parent_process_name" - ) - - -@pytest.fixture -def prevention_signatures(): - """Provide sample prevention expectation signatures. - - Returns: - List of signature dictionaries for prevention expectations. - - """ - return TestDataFactory.create_expectation_signatures( - signature_type="parent_process_name" - ) - - -@pytest.fixture -def oaev_detection_data(): - """Provide sample OAEV detection data. - - Returns: - List of OAEV-formatted detection data dictionaries. - - """ - return TestDataFactory.create_oaev_detection_data() - - -@pytest.fixture -def oaev_prevention_data(): - """Provide sample OAEV prevention data. - - Returns: - List of OAEV-formatted prevention data dictionaries. - - """ - return TestDataFactory.create_oaev_prevention_data() - - -@pytest.fixture -def mixed_template_data(): - """Provide mixed Template data (DV data). - - Returns: - List containing both DeepVisibilityEvent and TemplateData instances. - - """ - return TestDataFactory.create_mixed_template_data() - - -@pytest.fixture -def mock_expectation_detection(): - """Provide a mock detection expectation. - - Returns: - Mock DetectionExpectation instance for testing. - - """ - return MockObjectsFactory.create_mock_expectation(expectation_type="detection") - - -@pytest.fixture -def mock_expectation_prevention(): - """Provide a mock prevention expectation. - - Returns: - Mock PreventionExpectation instance for testing. - - """ - return MockObjectsFactory.create_mock_expectation(expectation_type="prevention") - - -@pytest.fixture -def mock_requests_session(): - """Provide a mock requests session. - - Returns: - Mock requests.Session instance for HTTP testing. - - """ - return MockObjectsFactory.create_mock_session() - - -@pytest.fixture(autouse=True) -def mock_logging(): - """Auto-mock logging to reduce noise in tests. - - Auto-applies to all tests to prevent logging output during test execution. - - Yields: - Mock logger instance. - - """ - with patch("logging.getLogger") as mock_logger: - mock_logger.return_value = Mock() - yield mock_logger - - -@pytest.fixture -def disable_sleep(): - """Disable time.sleep in tests for faster execution. - - Patches time.sleep to prevent actual delays during testing. - - Yields: - None (context manager for sleep patching). - - """ - with patch("time.sleep"): - yield - - -@pytest.fixture(params=[1, 3, 5]) -def various_counts(request): - """Provide various counts for testing different data sizes. - - Args: - request: Pytest request object containing parameter values. - - Returns: - Integer count (1, 3, or 5) for parameterized testing. - - """ - return request.param - - -@pytest.fixture(params=[True, False]) -def match_scenarios(request): - """Provide both matching and non-matching scenarios. - - Args: - request: Pytest request object containing parameter values. - - Returns: - Boolean value (True or False) for match testing scenarios. - - """ - return request.param - - -@pytest.fixture(params=["detection", "prevention"]) -def expectation_types(request): - """Provide different expectation types. - - Args: - request: Pytest request object containing parameter values. - - Returns: - String expectation type ("detection" or "prevention"). - - """ - return request.param - - -@pytest.fixture -def config_factory(): - """Provide the ConfigLoaderFactory for creating configs in tests. - - Returns: - ConfigLoaderFactory class for generating test configurations. - - """ - return ConfigLoaderFactory - - -@pytest.fixture -def data_factory(): - """Provide the TemplateDataFactory for creating data. - - Returns: - TemplateDataFactory class for generating test data. - - """ - return TemplateDataFactory - - -@pytest.fixture -def expectation_result_factory(): - """Provide the ExpectationResultFactory for creating results. - - Returns: - ExpectationResultFactory class for generating test results. - - """ - return ExpectationResultFactory - - -@pytest.fixture -def expectation_trace_factory(): - """Provide the ExpectationTraceFactory for creating traces. - - Returns: - ExpectationTraceFactory class for generating test traces. - - """ - return ExpectationTraceFactory - - -@pytest.fixture -def test_data_factory(): - """Provide the TestDataFactory for creating test data combinations. - - Returns: - TestDataFactory class for generating complex test data scenarios. - - """ - return TestDataFactory - - -@pytest.fixture -def mock_objects_factory(): - """Provide the MockObjectsFactory for creating mock objects. - - Returns: - MockObjectsFactory class for generating mock instances. - - """ - return MockObjectsFactory - - -@pytest.fixture(autouse=True) -def cleanup_mocks(): - """Auto-cleanup mocks after each test. - - Auto-applies to all tests to ensure proper mock cleanup. - - Yields: - None (context manager for cleanup operations). - - """ - yield - - -@pytest.fixture -def api_error_responses(): - """Provide various API error responses for testing error handling. - - Returns: - Dictionary mapping HTTP status codes to error response data. - - """ - return { - "400": { - "status_code": 400, - "text": "Bad Request", - "json": {"errors": ["Bad request"]}, - }, - "401": { - "status_code": 401, - "text": "Unauthorized", - "json": {"errors": ["Unauthorized"]}, - }, - "403": { - "status_code": 403, - "text": "Forbidden", - "json": {"errors": ["Forbidden"]}, - }, - "404": { - "status_code": 404, - "text": "Not Found", - "json": {"errors": ["Not found"]}, - }, - "500": { - "status_code": 500, - "text": "Internal Server Error", - "json": {"errors": ["Server error"]}, - }, - } - - -@pytest.fixture -def network_errors(): - """Provide various network errors for testing error handling. - - Returns: - List of different exception types for network error testing. - - """ - return [ - ConnectionError("Connection failed"), - TimeoutError("Request timeout"), - Exception("Generic network error"), - ] diff --git a/template/tests/services/fixtures/factories.py b/template/tests/services/fixtures/factories.py deleted file mode 100644 index 67ac5887..00000000 --- a/template/tests/services/fixtures/factories.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Essential polyfactory factories for Template models and test fixtures.""" - -import os -import uuid -from typing import Any -from unittest.mock import Mock - -from polyfactory import Use -from polyfactory.factories.pydantic_factory import ModelFactory -from src.collector.models import ExpectationResult, ExpectationTrace -from src.models.configs.collector_configs import _ConfigLoaderOAEV -from src.models.configs.config_loader import ConfigLoader, ConfigLoaderCollector -from src.models.configs.template_configs import _ConfigLoaderTemplate -from src.services.model_data import TemplateData - - -class ConfigLoaderOAEVFactory(ModelFactory[_ConfigLoaderOAEV]): - """Factory for OpenAEV configuration. - - Creates test instances of OpenAEV configuration with required - environment variables automatically set. - """ - - __check_model__ = False - - @classmethod - def build(cls, **kwargs): - """Build the model with required environment variables set. - - Args: - **kwargs: Additional keyword arguments for model creation. - - Returns: - _ConfigLoaderOAEV instance with test configuration. - - """ - os.environ["OPENAEV_URL"] = "https://test-openaev.example.com" - os.environ["OPENAEV_TOKEN"] = "test-openaev-token-12345" # noqa: S105 - return super().build(**kwargs) - - -class ConfigLoaderTemplateFactory(ModelFactory[_ConfigLoaderTemplate]): - """Factory for Template configuration. - - Creates test instances of Template configuration with required - environment variables automatically set. - """ - - __check_model__ = False - - @classmethod - def build(cls, **kwargs): - """Build the model with required environment variables set. - - Args: - **kwargs: Additional keyword arguments for model creation. - - Returns: - _ConfigLoaderTemplate instance with test configuration. - - """ - os.environ["TEMPLATE_KEY"] = "test-template-key" - return super().build(**kwargs) - - -class ConfigLoaderCollectorFactory(ModelFactory[ConfigLoaderCollector]): - """Factory for Collector configuration. - - Creates test instances of collector configuration with auto-generated - UUIDs and sensible defaults. - """ - - __check_model__ = False - - id = Use(lambda: f"template--{uuid.uuid4()}") - name = "Template" - - -class ConfigLoaderFactory(ModelFactory[ConfigLoader]): - """Factory for main configuration. - - Creates complete test configuration instances combining OpenAEV, - collector, and Template settings using subfactories. - """ - - __check_model__ = False - - openaev = Use(ConfigLoaderOAEVFactory.build) - collector = Use(ConfigLoaderCollectorFactory.build) - template = Use(ConfigLoaderTemplateFactory.build) - - -class TemplateDataFactory(ModelFactory[TemplateData]): - """Factory for Template data. - - Creates test instances of Template data objects. - """ - - __check_model__ = False - - -class ExpectationResultFactory(ModelFactory[ExpectationResult]): - """Factory for ExpectationResult. - - Creates test instances of expectation processing results with - valid expectation IDs and configurable validation status. - """ - - __check_model__ = False - - expectation_id = Use(lambda: str(uuid.uuid4())) - is_valid = True - error_message = None - matched_alerts = Use(lambda: []) - - -class ExpectationTraceFactory(ModelFactory[ExpectationTrace]): - """Factory for ExpectationTrace. - - Creates test instances of expectation traces for OpenAEV - with properly formatted trace data. - """ - - __check_model__ = False - - -class MockObjectsFactory: - """Factory for creating mock objects. - - Provides static methods for creating various mock objects - used throughout the test suite. - """ - - @staticmethod - def create_mock_detection_helper(match_result: bool = True): - """Create mock detection helper. - - Args: - match_result: Whether the helper should return matches (default True). - - Returns: - Mock OpenAEVDetectionHelper instance. - - """ - mock_helper = Mock() - mock_helper.match_alert_elements.return_value = match_result - return mock_helper - - @staticmethod - def create_mock_expectation( - expectation_type: str = "detection", expectation_id: str = None - ): - """Create mock expectation for testing. - - Args: - expectation_type: Type of expectation ("detection" or "prevention"). - expectation_id: Optional custom expectation ID. - - Returns: - Mock expectation object with required attributes. - - """ - mock_expectation = Mock() - mock_expectation.inject_expectation_id = expectation_id or str(uuid.uuid4()) - mock_expectation.inject_expectation_signatures = [] - mock_expectation.expectation_type = expectation_type - return mock_expectation - - @staticmethod - def create_mock_session(): - """Create mock requests session. - - Returns: - Mock requests.Session instance with headers attribute. - - """ - mock_session = Mock() - mock_session.headers = {} - return mock_session - - -class TestDataFactory: - """Factory for creating essential test data. - - Provides static methods for creating complex test data structures - that simulate real-world scenarios. - """ - - @staticmethod - def create_expectation_signatures( - signature_type: str = "parent_process_name", signature_value: str = None - ) -> list[dict[str, Any]]: - """Create expectation signatures. - - Args: - signature_type: Type of signature to create. - signature_value: Optional custom signature value. - - Returns: - List of signature dictionaries for testing. - - """ - if signature_value is None: - signature_value = f"test-{signature_type}-{uuid.uuid4().hex[:8]}" - - return [{"type": signature_type, "value": signature_value}] - - @staticmethod - def create_oaev_detection_data() -> list[dict[str, Any]]: - """Create OAEV detection data. - - Returns: - List of OAEV-formatted detection data dictionaries. - - """ - return [] - - @staticmethod - def create_oaev_prevention_data() -> list[dict[str, Any]]: - """Create OAEV prevention data. - - Returns: - List of OAEV-formatted prevention data dictionaries. - - """ - return [] - - @staticmethod - def create_mixed_template_data() -> list[Any]: - """Create mixed Template data. - - Returns: - List containing both DV event dicts and TemplateData instances. - - """ - return [] - - -# Helper functions -def create_test_config(**overrides) -> ConfigLoader: - """Create test configuration. - - Args: - **overrides: Configuration values to override defaults. - - Returns: - ConfigLoader instance with test configuration. - - """ - return ConfigLoaderFactory.build(**overrides) - - -def create_test_data(count: int = 1) -> list[TemplateData]: - """Create test TemplateData data. - - Args: - count: Number of data to create (default 1). - - Returns: - List of TemplateData instances for testing. - - """ - return [TemplateDataFactory.build() for _ in range(count)] diff --git a/template/tests/services/test_converter.py b/template/tests/services/test_converter.py deleted file mode 100644 index 8ecdc264..00000000 --- a/template/tests/services/test_converter.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Essential tests for Template Converter services - Gherkin GWT Format.""" - -import pytest -from src.services.converter import TemplateConverter -from src.services.exception import TemplateValidationError -from src.services.model_data import TemplateData - -# -------- -# Scenarios -# -------- - - -# Scenario: Initialize converter successfully -def test_initialize_converter(): - """Scenario: Initialize converter successfully.""" - # Given: All dependencies are available - _given_converter_dependencies_available() - - # When: I initialize the converter - converter = _when_initialize_converter() - - # Then: The converter should be initialized successfully - _then_converter_initialized_successfully(converter) - - -# Scenario: Convert empty data list -def test_convert_empty_data(): - """Scenario: Convert empty data list.""" - # Given: A converter is available - converter = _given_initialized_converter() - - # When: I convert an empty data list - result = _when_convert_data_to_oaev(converter, []) - - # Then: An empty list should be returned - _then_empty_list_returned(result) - - -# Scenario: Convert single data with complete data -def test_convert_single_data_complete_data(): - """Scenario: Convert single data with complete data.""" - # Given: A converter is available - converter = _given_initialized_converter() - # Given: A data with complete data - data = _given_data_with_complete_data() - - # When: I convert the data to OAEV format - result = _when_convert_data_to_oaev(converter, [data]) - - # Then: The data should be converted with all fields - _then_single_data_converted_completely(result, data) - - -# Scenario: Convert invalid data type -def test_convert_invalid_data_type(): - """Scenario: Convert invalid data type.""" - # Given: A converter is available - converter = _given_initialized_converter() - # Given: Invalid input data (not a list) - invalid_data = _given_invalid_input_data() - - # When: I attempt to convert invalid data - # Then: A validation error should be raised - _when_convert_invalid_data_then_validation_error_raised(converter, invalid_data) - - -# -------- -# Given Methods -# -------- - - -# Given: All dependencies are available -def _given_converter_dependencies_available(): - """Ensure all converter dependencies are available.""" - pass - - -# Given: A converter is available -def _given_initialized_converter() -> TemplateConverter: - """Create and return an initialized converter. - - Returns: - Initialized TemplateConverter instance. - - """ - return TemplateConverter() - - -# Given: A data with complete data -def _given_data_with_complete_data() -> TemplateData: - """Create a data with complete data. - - Returns: - TemplateData with key. - - """ - return TemplateData(key="complete_data_123") - - -# Given: Invalid input data (not a list) -def _given_invalid_input_data() -> str: - """Create invalid input data for testing. - - Returns: - Invalid data (string instead of list). - - """ - return "invalid_string_data" - - -# -------- -# When Methods -# -------- - - -# When: I initialize the converter -def _when_initialize_converter() -> TemplateConverter: - """Initialize the converter. - - Returns: - Initialized TemplateConverter instance. - - """ - return TemplateConverter() - - -# When: I convert data to OAEV format -def _when_convert_data_to_oaev(converter: TemplateConverter, data: list) -> list: - """Convert data to OAEV format. - - Args: - converter: The converter instance. - data: List of data to convert. - - Returns: - List of converted OAEV format data. - - """ - return converter.convert_data_to_oaev(data) - - -# When: I attempt to convert invalid data and expect validation error -def _when_convert_invalid_data_then_validation_error_raised( - converter: TemplateConverter, invalid_data: str -) -> None: - """Attempt to convert invalid data and expect validation error. - - Args: - converter: The converter instance. - invalid_data: Invalid input data. - - """ - with pytest.raises(TemplateValidationError) as exc_info: - converter.convert_data_to_oaev(invalid_data) - - assert "data must be a list" in str(exc_info.value) # noqa: S101 - - -# -------- -# Then Methods -# -------- - - -# Then: The converter should be initialized successfully -def _then_converter_initialized_successfully(converter: TemplateConverter) -> None: - """Verify the converter was initialized successfully. - - Args: - converter: The converter instance to verify. - - """ - assert converter is not None # noqa: S101 - assert converter.logger is not None # noqa: S101 - - -# Then: An empty list should be returned -def _then_empty_list_returned(result: list) -> None: - """Verify an empty list was returned. - - Args: - result: The conversion result to verify. - - """ - assert result == [] # noqa: S101 - - -# Then: The data should be converted with all fields -def _then_single_data_converted_completely(result: list, data: TemplateData) -> None: - """Verify single data was converted with all fields. - - Args: - result: The conversion result to verify. - data: The original data object. - - """ - assert len(result) == 1 # noqa: S101 diff --git a/template/tests/services/test_expectation_service.py b/template/tests/services/test_expectation_service.py deleted file mode 100644 index a30b1657..00000000 --- a/template/tests/services/test_expectation_service.py +++ /dev/null @@ -1,481 +0,0 @@ -"""Essential tests for Template Expectation Service - Gherkin GWT Format.""" - -from unittest.mock import Mock -from uuid import uuid4 - -import pytest -from pyoaev.signatures.types import SignatureTypes -from src.services.expectation_service import ( - ExpectationResult, - TemplateExpectationService, -) -from src.services.model_data import TemplateData -from tests.gwt_shared import ( - given_initialized_expectation_service, - given_test_config, - then_expectation_service_initialized_successfully, -) - -# -------- -# Scenarios -# -------- - - -# Scenario: Initialize expectation service with valid configuration -def test_initialize_expectation_service_with_valid_config(): - """Scenario: Initialize expectation service with valid configuration.""" - # Given: A valid configuration is available - config = _given_valid_config_for_expectation_service() - - # When: I initialize the expectation service - service = _when_initialize_expectation_service(config) - - # Then: The expectation service should be initialized successfully - _then_expectation_service_initialized_with_valid_config(service, config) - - -# Scenario: Initialize with invalid configuration raises error -def test_initialize_with_invalid_config(): - """Scenario: Initialize with invalid configuration raises error.""" - # Given: An invalid configuration (None) - invalid_config = _given_invalid_config() - - # When: I attempt to initialize the expectation service - # Then: An AttributeError should be raised - _when_initialize_expectation_service_then_attribute_error_raised(invalid_config) - - -# Scenario: Handle single detection expectation -def test_handle_single_detection_expectation(): - """Scenario: Handle single detection expectation.""" - # Given: An initialized expectation service - service = _given_initialized_expectation_service() - # Given: A detection helper - detection_helper = _given_mock_detection_helper() - # Given: Mock data are available - _given_mock_data_for_service(service) - # Given: A detection expectation - expectation = _given_detection_expectation() - - # When: I handle the detection expectation - result = _when_handle_batch_expectations(service, [expectation], detection_helper) - - # Then: A detection result should be returned - _then_detection_result_returned(result, expectation) - - -# Scenario: Handle prevention expectation -def test_handle_prevention_expectation(): - """Scenario: Handle prevention expectation.""" - # Given: An initialized expectation service - service = _given_initialized_expectation_service() - # Given: A detection helper - detection_helper = _given_mock_detection_helper() - # Given: A prevention expectation - expectation = _given_prevention_expectation() - - # When: I handle the prevention expectation - result = _when_handle_batch_expectations(service, [expectation], detection_helper) - - # Then: A prevention result should be returned - _then_prevention_result_returned(result, expectation) - - -# Scenario: Match data to expectations -def test_match_data_to_expectations(): - """Scenario: Match data to expectations.""" - # Given: An initialized expectation service - service = _given_initialized_expectation_service() - # Given: data and expectations - data, expectations = _given_data_and_expectations() - - # When: I match data to expectations - matches = _when_match_data_to_expectations(service, data, expectations) - - # Then: Proper matches should be found - _then_proper_matches_found(matches, data, expectations) - # Then: The match should succeed without requiring mitigation - _then_match_succeeds_without_mitigation_requirement(matches) - - -# -------- -# Given Methods -# -------- - - -# Given: A valid configuration is available -def _given_valid_config_for_expectation_service(): - """Create a valid configuration for expectation service testing. - - Returns: - Test configuration object. - - """ - return given_test_config() - - -# Given: An invalid configuration (None) -def _given_invalid_config(): - """Create an invalid configuration. - - Returns: - None (invalid configuration). - - """ - return None - - -# Given: An initialized expectation service -def _given_initialized_expectation_service(): - """Create an initialized expectation service. - - Returns: - Initialized TemplateExpectationService instance. - - """ - return given_initialized_expectation_service() - - -# Given: A detection helper -def _given_mock_detection_helper(): - """Create a mock detection helper. - - Returns: - Mock detection helper instance. - - """ - return Mock() - - -# Given: Mock data are available -def _given_mock_data_for_service(service): - """Set up mock data for the service. - - Args: - service: The expectation service instance. - - """ - mock_data = [ - TemplateData( - key="test_data_1", - ) - ] - service.data_fetcher.fetch_data_for_time_window = Mock(return_value=mock_data) - - -# Given: A detection expectation -def _given_detection_expectation(): - """Create a detection expectation. - - Returns: - Mock detection expectation. - - """ - hostname_sig = _create_mock_signature( - SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, "target-host.example.com" - ) - end_date_sig = _create_mock_signature( - Mock(value="end_date"), "2024-01-01T12:00:00Z" - ) - - expectation = _create_mock_expectation( - expectation_id="detection_test_1", signatures=[hostname_sig, end_date_sig] - ) - return expectation - - -# Given: A prevention expectation -def _given_prevention_expectation(): - """Create a prevention expectation. - - Returns: - Mock prevention expectation. - - """ - hostname_sig = _create_mock_signature( - SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, "target-host.example.com" - ) - end_date_sig = _create_mock_signature( - Mock(value="end_date"), "2024-01-01T12:00:00Z" - ) - - expectation = _create_mock_expectation( - expectation_id="prevention_test_1", signatures=[hostname_sig, end_date_sig] - ) - expectation.is_prevention = True - return expectation - - -# Given: data and expectations -def _given_data_and_expectations(): - """Create data and expectations for matching tests. - - Returns: - Tuple of (data, expectations). - - """ - data = [ - TemplateData( - key="match_data_1", - ) - ] - - hostname_sig = _create_mock_signature( - SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, "match-host.example.com" - ) - expectation = _create_mock_expectation(signatures=[hostname_sig]) - expectations = [expectation] - - return data, expectations - - -# Given: A static expectation -def _given_static_expectation(): - """Create a static expectation. - - Returns: - Mock static expectation. - - """ - hostname_sig = _create_mock_signature( - SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS, "static-host.example.com" - ) - end_date_sig = _create_mock_signature( - Mock(value="end_date"), "2024-01-01T12:00:00Z" - ) - - expectation = _create_mock_expectation( - expectation_id="static_test_1", signatures=[hostname_sig, end_date_sig] - ) - return expectation - - -# -------- -# When Methods -# -------- - - -# When: I initialize the expectation service -def _when_initialize_expectation_service(config): - """Initialize expectation service with given configuration. - - Args: - config: Configuration object to use. - - Returns: - Initialized TemplateExpectationService instance. - - """ - return TemplateExpectationService(config=config) - - -# When: I attempt to initialize with invalid config and expect AttributeError -def _when_initialize_expectation_service_then_attribute_error_raised(invalid_config): - """Attempt to initialize with invalid config and expect AttributeError. - - Args: - invalid_config: Invalid configuration to test. - - """ - with pytest.raises(AttributeError): - TemplateExpectationService(config=invalid_config) - - -# When: I handle batch expectations -def _when_handle_batch_expectations(service, expectations, detection_helper): - """Handle batch expectations using the service. - - Args: - service: The expectation service instance. - expectations: List of expectations to handle. - detection_helper: The detection helper to use. - - Returns: - List of expectation results. - - """ - results, _ = service.handle_batch_expectations(expectations, detection_helper) - return results - - -# When: I match data to expectations -def _when_match_data_to_expectations(service, data, expectations): - """Match data to expectations. - - Args: - service: The expectation service instance. - data: List of data. - expectations: List of expectations. - - Returns: - List of matches. - - """ - return service._match_data_to_expectations(expectations, data, "detection") - - -# When: I check if expectation matches data -def _when_check_expectation_matches_data(service, expectation, data): - """Check if expectation matches data. - - Args: - service: The expectation service instance. - expectation: The expectation to check. - data: The data to match against. - - Returns: - Boolean indicating if there's a match. - - """ - expectation_type = ( - "prevention" - if hasattr(expectation, "is_prevention") and expectation.is_prevention - else "detection" - ) - return service._expectation_matches_data(expectation, data, expectation_type) - - -# -------- -# Then Methods -# -------- - - -# Then: The expectation service should be initialized successfully with valid config -def _then_expectation_service_initialized_with_valid_config(service, config): - """Verify expectation service was initialized successfully. - - Args: - service: The service instance to verify. - config: The configuration used for initialization. - - """ - then_expectation_service_initialized_successfully(service) - assert service.batch_size == config.template.expectation_batch_size # noqa: S101 - - -# Then: A detection result should be returned -def _then_detection_result_returned(result, expectation): - """Verify a detection result was returned. - - Args: - result: The result to verify. - expectation: The original expectation. - - """ - assert len(result) == 1 # noqa: S101 - assert isinstance(result[0], ExpectationResult) # noqa: S101 - assert result[0].expectation_id == expectation.inject_expectation_id # noqa: S101 - - -# Then: A prevention result should be returned -def _then_prevention_result_returned(result, expectation): - """Verify a prevention result was returned. - - Args: - result: The result to verify. - expectation: The original expectation. - - """ - assert len(result) == 1 # noqa: S101 - assert isinstance(result[0], ExpectationResult) # noqa: S101 - assert result[0].expectation_id == expectation.inject_expectation_id # noqa: S101 - - -# Then: Proper matches should be found -def _then_proper_matches_found(matches, data, expectations): - """Verify proper matches were found. - - Args: - matches: The found matches. - data: The original data. - expectations: The original expectations. - - """ - assert len(matches) > 0 # noqa: S101 - - -# Then: The match should succeed without requiring mitigation -def _then_match_succeeds_without_mitigation_requirement(matches): - """Verify match succeeds without requiring mitigation. - - Args: - matches: The match results. - - """ - assert matches is not None # noqa: S101 - - -# Then: A static result with Deep Visibility events should be returned -def _then_static_result_with_deep_visibility_returned(result, expectation): - """Verify a static result with Deep Visibility events was returned. - - Args: - result: The result to verify. - expectation: The original expectation. - - """ - assert len(result) == 1 # noqa: S101 - assert isinstance(result[0], ExpectationResult) # noqa: S101 - assert result[0].expectation_id == expectation.inject_expectation_id # noqa: S101 - assert result[0].is_valid # noqa: S101 - assert len(result[0].matched_alerts) > 0 # noqa: S101 - - -# Then: A static result without Deep Visibility events should be returned -def _then_static_result_without_deep_visibility_returned(result, expectation): - """Verify a static result without Deep Visibility events was returned. - - Args: - result: The result to verify. - expectation: The original expectation. - - """ - assert len(result) == 1 # noqa: S101 - assert isinstance(result[0], ExpectationResult) # noqa: S101 - assert result[0].expectation_id == expectation.inject_expectation_id # noqa: S101 - assert result[0].is_valid # noqa: S101 - - -# -------- -# Helper Methods -# -------- - - -def _create_mock_signature(sig_type, value): - """Create a mock signature with proper attributes. - - Args: - sig_type: The signature type. - value: The signature value. - - Returns: - Mock signature object. - - """ - sig = Mock() - sig.type = sig_type - sig.value = value - return sig - - -def _create_mock_expectation(expectation_id=None, signatures=None): - """Create a mock expectation with proper attributes. - - Args: - expectation_id: The expectation ID. - signatures: List of signatures. - - Returns: - Mock expectation object. - - """ - if expectation_id is None: - expectation_id = str(uuid4()) - if signatures is None: - signatures = [] - - expectation = Mock() - expectation.inject_expectation_id = expectation_id - expectation.inject_expectation_signatures = signatures - expectation.id = expectation_id - return expectation diff --git a/template/tests/services/test_fetcher_data.py b/template/tests/services/test_fetcher_data.py deleted file mode 100644 index 1e239369..00000000 --- a/template/tests/services/test_fetcher_data.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Essential tests for Template Data Fetcher service - Gherkin GWT Format.""" - -import pytest -from src.services.exception import TemplateValidationError -from src.services.fetcher_data import FetcherData - -# -------- -# Scenarios -# -------- - - -# Scenario: Fetch data for time window successfully -def test_fetch_data_for_time_window_successfully(): - """Scenario: Fetch data for time window successfully.""" - # Given: A valid data fetcher - fetcher = _given_valid_data_fetcher() - # Given: A valid time window - time_window = _given_valid_time_window() - # When: I fetch data for the time window - data = _when_fetch_data_for_time_window(fetcher, time_window) - - # Then: Data should be returned successfully - _then_data_returned_successfully(data) - - -# Scenario: Handle invalid time window -def test_handle_invalid_time_window(): - """Scenario: Handle invalid time window.""" - # Given: A valid data fetcher - fetcher = _given_valid_data_fetcher() - # Given: An invalid time window - invalid_time_window = _given_invalid_time_window() - - # When: I attempt to fetch data with invalid time window - # Then: A validation error should be raised - _when_fetch_data_then_validation_error_raised(fetcher, invalid_time_window) - - -# -------- -# Given Methods -# -------- - - -# Given: A valid data fetcher -def _given_valid_data_fetcher(): - """Create a valid data fetcher for testing. - - Returns: - Initialized FetcherData instance. - - """ - return FetcherData() - - -# Given: A valid time window -def _given_valid_time_window(): - """Create a valid time window for testing. - - Returns: - Valid timedelta object. - - """ - from datetime import timedelta - - return timedelta(hours=24) - - -# Given: An invalid time window -def _given_invalid_time_window(): - """Create an invalid time window. - - Returns: - Invalid time window (None). - - """ - return None - - -# -------- -# When Methods -# -------- - - -# When: I initialize the data fetcher -def _when_initialize_data_fetcher(): - """Initialize data fetcher. - - Returns: - Initialized FetcherData instance. - - """ - return FetcherData() - - -# When: I fetch data for the time window -def _when_fetch_data_for_time_window(fetcher, time_window): - """Fetch data for given time window. - - Args: - fetcher: The data fetcher instance. - time_window: Time window to fetch data for. - - Returns: - List of fetched data. - - """ - from datetime import datetime, timezone - - end_time = datetime.now(timezone.utc) - start_time = end_time - time_window - return fetcher.fetch_data_for_time_window(start_time, end_time) - - -# When: I attempt to fetch data and expect validation error -def _when_fetch_data_then_validation_error_raised(fetcher, invalid_time_window): - """Attempt to fetch data and expect validation error. - - Args: - fetcher: The data fetcher instance. - invalid_time_window: Invalid time window to test. - - """ - from datetime import datetime, timezone - - if invalid_time_window is not None: - end_time = datetime.now(timezone.utc) - start_time = end_time - invalid_time_window - with pytest.raises(TemplateValidationError): - fetcher.fetch_data_for_time_window(start_time, end_time) - else: - with pytest.raises(TemplateValidationError): - fetcher.fetch_data_for_time_window(None, None) - - -# -------- -# Then Methods -# -------- - - -# Then: The data fetcher should be initialized successfully -def _then_data_fetcher_initialized_successfully(fetcher): - """Verify data fetcher was initialized successfully. - - Args: - fetcher: The data fetcher instance to verify. - - """ - assert fetcher is not None # noqa: S101 - assert fetcher.logger is not None # noqa: S101 - - -# Then: data should be returned successfully -def _then_data_returned_successfully(data): - """Verify data were returned successfully. - - Args: - data: The fetched data to verify. - - """ - assert isinstance(data, list) # noqa: S101 - assert len(data) > 0 # noqa: S101 - - # Basic verification that we got data back - from src.services.model_data import TemplateData - - assert all( # noqa: S101 - isinstance(single_data, TemplateData) for single_data in data - ) diff --git a/template/tests/services/test_trace_service.py b/template/tests/services/test_trace_service.py deleted file mode 100644 index 6ea00452..00000000 --- a/template/tests/services/test_trace_service.py +++ /dev/null @@ -1,235 +0,0 @@ -"""Essential tests for Template Trace Service - Gherkin GWT Format.""" - -import pytest -from src.services.exception import TemplateValidationError -from src.services.trace_service import TemplateTraceService -from tests.gwt_shared import given_test_config - -# -------- -# Scenarios -# -------- - - -# Scenario: Initialize trace service with valid configuration -def test_initialize_trace_service_with_valid_config(): - """Scenario: Initialize trace service with valid configuration.""" - # Given: A valid configuration is available - config = _given_valid_config_for_trace_service() - - # When: I initialize the trace service - service = _when_initialize_trace_service(config) - - # Then: The trace service should be initialized successfully - _then_trace_service_initialized_successfully(service, config) - - -# Scenario: Initialize with invalid configuration raises error -def test_initialize_with_invalid_config(): - """Scenario: Initialize with invalid configuration raises error.""" - # Given: An invalid configuration (None) - invalid_config = _given_invalid_config() - - # When: I attempt to initialize the trace service - # Then: A validation error should be raised - _when_initialize_trace_service_then_validation_error_raised(invalid_config) - - -# Scenario: Create traces from valid expectation results -def test_create_traces_from_valid_expectation_results(): - """Scenario: Create traces from valid expectation results.""" - # Given: A valid trace service - service = _given_valid_trace_service() - # Given: Valid expectation results with matching alerts - results = _given_valid_expectation_results_with_alerts() - - # When: I create traces from the results - traces = _when_create_traces_from_results(service, results) - - # Then: Traces should be created successfully - _then_traces_created_successfully(traces, results) - - -# Scenario: Trace timestamp format is correct for Java backend -def test_trace_timestamp_format_is_correct(): - """Scenario: Trace timestamp format is correct for Java backend.""" - # Given: A valid trace service - service = _given_valid_trace_service() - # Given: Valid expectation results with matching alerts - results = _given_valid_expectation_results_with_alerts() - - # When: I create traces from the results - traces = _when_create_traces_from_results(service, results) - - # Then: Timestamp format should be valid for Java backend - _then_timestamp_format_is_valid(traces) - - -# -------- -# Given Methods -# -------- - - -# Given: A valid configuration is available -def _given_valid_config_for_trace_service(): - """Create a valid configuration for trace service testing. - - Returns: - Test configuration object. - - """ - return given_test_config() - - -# Given: An invalid configuration (None) -def _given_invalid_config(): - """Create an invalid configuration. - - Returns: - None (invalid configuration). - - """ - return None - - -# Given: A valid trace service -def _given_valid_trace_service(): - """Create a valid trace service for testing. - - Returns: - Initialized TemplateTraceService instance. - - """ - config = given_test_config() - return TemplateTraceService(config=config) - - -# Given: Valid expectation results with matching alerts -def _given_valid_expectation_results_with_alerts(): - """Create valid expectation results with alerts for testing. - - Returns: - List of expectation results with alerts. - - """ - from unittest.mock import Mock - - result1 = Mock() - result1.expectation_id = "test_expectation_1" - result1.is_valid = True - result1.matched_alerts = [ - { - "alert_id": "alert_1", - "severity": "high", - "message": "Test alert 1", - "alert_name": "Template Test Alert 1", - "alert_link": "https://foo.bar", - } - ] - result1.expectation = Mock() - result1.expectation.inject_expectation_id = "test_expectation_1" - - return [result1] - - -# -------- -# When Methods -# -------- - - -# When: I initialize the trace service -def _when_initialize_trace_service(config): - """Initialize trace service with given configuration. - - Args: - config: Configuration object to use. - - Returns: - Initialized TemplateTraceService instance. - - """ - return TemplateTraceService(config=config) - - -# When: I attempt to initialize with invalid config and expect validation error -def _when_initialize_trace_service_then_validation_error_raised(invalid_config): - """Attempt to initialize with invalid config and expect validation error. - - Args: - invalid_config: Invalid configuration to test. - - """ - with pytest.raises(TemplateValidationError): - TemplateTraceService(config=invalid_config) - - -# When: I create traces from the results -def _when_create_traces_from_results(service, results): - """Create traces from expectation results. - - Args: - service: The trace service instance. - results: List of expectation results. - - Returns: - List of created traces. - - """ - return service.create_traces_from_results(results, "test_collector_id") - - -# -------- -# Then Methods -# -------- - - -# Then: The trace service should be initialized successfully -def _then_trace_service_initialized_successfully(service, config): - """Verify trace service was initialized successfully. - - Args: - service: The service instance to verify. - config: The configuration used for initialization. - - """ - assert service is not None # noqa: S101 - assert service.config == config # noqa: S101 - assert service.logger is not None # noqa: S101 - - -# Then: Traces should be created successfully -def _then_traces_created_successfully(traces, results): - """Verify traces were created successfully from results. - - Args: - traces: The created traces to verify. - results: The original expectation results. - - """ - assert isinstance(traces, list) # noqa: S101 - assert len(traces) > 0 # noqa: S101 - - from src.collector.models import ExpectationTrace - - for trace in traces: - assert isinstance(trace, ExpectationTrace) # noqa: S101 - assert hasattr(trace, "inject_expectation_trace_expectation") # noqa: S101 - assert hasattr(trace, "inject_expectation_trace_alert_name") # noqa: S101 - assert hasattr(trace, "inject_expectation_trace_alert_link") # noqa: S101 - - -# Then: Timestamp format should be valid for Java backend -def _then_timestamp_format_is_valid(traces): - """Verify timestamp format is valid for Java backend. - - Args: - traces: The created traces to verify. - - """ - import re - - valid_timestamp_pattern = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$" - - for trace in traces: - timestamp = trace.inject_expectation_trace_date - assert re.match(valid_timestamp_pattern, timestamp) # noqa: S101 - assert "+00:00Z" not in timestamp # noqa: S101 diff --git a/template/tests/test_create_collector.py b/template/tests/test_create_collector.py deleted file mode 100644 index 6f5b067f..00000000 --- a/template/tests/test_create_collector.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Test module for the Template Collector initialization - Gherkin GWT Format.""" - -from os import environ as os_environ -from typing import Any - -import pytest -from src.collector import Collector -from src.collector.exception import CollectorConfigError -from tests.conftest import mock_env_vars - -# -------- -# Fixtures -# -------- - - -@pytest.fixture() -def collector_config() -> dict[str, str]: # type: ignore - """Fixture for minimum required configuration. - - Returns: - Dictionary containing all required environment variables - for collector initialization with test values. - - """ - return { - "OPENAEV_URL": "http://fake-url/", - "OPENAEV_TOKEN": "fake-oaev-token", - "COLLECTOR_ID": "fake-collector-id", - "COLLECTOR_NAME": "Template", - "TEMPLATE_KEY": "fake-key", - "COLLECTOR_ICON_FILEPATH": "src/img/template-logo.png", - "COLLECTOR_LOG_LEVEL": "debug", - } - - -# -------- -# Scenarios -# -------- - - -# Scenario: Create a collector with success -def test_create_collector_with_valid_config(capfd, collector_config): # type: ignore - """Scenario: Create a collector with success. - - Args: - capfd: Pytest fixture for capturing stdout and stderr output. - collector_config: Fixture providing valid collector configuration. - - """ - # Given: A valid configuration is available - mock_env = _given_valid_collector_config(collector_config) - - # When: I create the collector - collector = _when_create_collector() - - # Then: The collector should be created successfully - _then_collector_created_successfully(capfd, mock_env, collector, collector_config) - - -# -------- -# Given Methods -# -------- - - -# Given: A valid configuration is available -def _given_valid_collector_config(config_data: dict[str, str]) -> Any: # type: ignore - """Set up valid collector configuration environment. - - Args: - config_data: Dictionary of environment variables to mock. - - Returns: - Mock environment variable patcher object. - - """ - mock_env = mock_env_vars(os_environ, config_data) - return mock_env - - -# -------- -# When Methods -# -------- - - -# When: I create the collector -def _when_create_collector() -> Collector: # type: ignore - """Create the collector instance. - - Returns: - Collector instance for testing. - - """ - collector = Collector() - return collector - - -# When: I attempt to create the collector and expect configuration error -def _when_create_collector_then_config_error_raised(mock_env: Any) -> None: # type: ignore - """Attempt to create collector and expect configuration error. - - Args: - mock_env: Mock environment variable patcher to clean up. - - """ - try: - with pytest.raises((CollectorConfigError, ValueError)): - _when_create_collector() - finally: - mock_env.stop() - - -# When: I attempt to create the collector and expect configuration error (alias) -def _when_create_collector_then_raises_config_error(mock_env: Any) -> None: # type: ignore - """Attempt to create collector and expect configuration error. - - Args: - mock_env: Mock environment variable patcher to clean up. - - """ - _when_create_collector_then_config_error_raised(mock_env) - - -# -------- -# Then Methods -# -------- - - -# Then: The collector should be created successfully -def _then_collector_created_successfully( - capfd: Any, # type: ignore - mock_env: Any, # type: ignore - collector: Collector, # type: ignore - expected_config: dict[str, str], -) -> None: - """Verify the collector was created successfully with correct configuration. - - Args: - capfd: Pytest fixture for capturing stdout and stderr output. - mock_env: Mock environment variable patcher to clean up. - collector: The created collector instance to verify. - expected_config: Expected configuration data to validate against. - - """ - assert collector is not None # noqa: S101 - - daemon_config = collector.config_instance.to_daemon_config() - - assert daemon_config.get("openaev_url") == expected_config.get( - "OPENAEV_URL" - ) # noqa: S101 - assert daemon_config.get("openaev_token") == expected_config.get( - "OPENAEV_TOKEN" - ) # noqa: S101 - assert daemon_config.get("collector_id") == expected_config.get( - "COLLECTOR_ID" - ) # noqa: S101 - assert daemon_config.get("collector_name") == expected_config.get( - "COLLECTOR_NAME" - ) # noqa: S101 - assert daemon_config.get("template_key") == expected_config.get( # noqa: S101 - "TEMPLATE_KEY" - ) - assert daemon_config.get( - "collector_log_level" - ) == expected_config.get( # noqa: S101 - "COLLECTOR_LOG_LEVEL" - ) - - _then_collector_logged_initialization_success(capfd, daemon_config) - mock_env.stop() - - -# Then: The collector initialization should be logged -def _then_collector_logged_initialization_success( - capfd: Any, # type: ignore - daemon_config: dict[str, str], -) -> None: - """Verify that collector initialization was logged appropriately. - - Args: - capfd: Pytest fixture for capturing stdout and stderr output. - daemon_config: Daemon configuration to check log level. - - """ - log_records = capfd.readouterr() - if daemon_config.get("collector_log_level") in ["info", "debug"]: - registered_message = "Template Collector initialized successfully" - assert registered_message in log_records.err # noqa: S101 diff --git a/template/tests/test_template_collector.py b/template/tests/test_template_collector.py new file mode 100644 index 00000000..9079c8d4 --- /dev/null +++ b/template/tests/test_template_collector.py @@ -0,0 +1,32 @@ +import unittest +from unittest.mock import patch + +import src.template_collector as module + + +class TestTemplateCollector(unittest.TestCase): + @patch.object(module, "BaseCollector") + @patch.object(module, "Source") + @patch.object(module, "SUPPORTED_SIGNATURES") + @patch.object(module, "TemplateSourceData") + @patch.object(module, "TemplateDataFetcher") + def test_template_collector_main( + self, + m_templatedatafetcher, + m_templatesourcedata, + m_supportedsignatures, + m_source, + m_basecollector, + ): + module.main() + + m_source.assert_called_once_with( + data_fetcher_model=m_templatedatafetcher, + source_data_model=m_templatesourcedata, + signatures=m_supportedsignatures, + ) + m_basecollector.assert_called_once_with( + name="Template collector", + source=m_source.return_value, + ) + m_basecollector.return_value.start.assert_called_once() From 467ae69df79695f2ab212ea05c936ac4db9b8b1f Mon Sep 17 00:00:00 2001 From: guzmud Date: Thu, 7 May 2026 17:49:44 +0200 Subject: [PATCH 14/25] [template] fix(collector): fixing various issues --- template/src/collector/engines/basic.py | 64 +++++++++---------- template/src/collector/models/expectations.py | 21 +++--- template/src/collector/models/source.py | 7 +- template/src/collector/types/collector.py | 11 +++- .../tests/collector/engines/test_basic.py | 23 ++++--- .../collector/models/test_expectations.py | 4 +- .../tests/collector/models/test_source.py | 5 +- 7 files changed, 72 insertions(+), 63 deletions(-) diff --git a/template/src/collector/engines/basic.py b/template/src/collector/engines/basic.py index 72bed29d..7490c452 100644 --- a/template/src/collector/engines/basic.py +++ b/template/src/collector/engines/basic.py @@ -1,12 +1,13 @@ import logging import os -from pyoaev.apis.inject_expectation.model import ( # type: ignore[import-untyped] +from pyoaev.apis.inject_expectation.model import ( DetectionExpectation, PreventionExpectation, ) -from pyoaev.client import OpenAEV # type: ignore[import_untyped] +from pyoaev.client import OpenAEV from pyoaev.helpers import OpenAEVDetectionHelper +from pyoaev.signatures.types import SignatureTypes from src.collector.internals.oaev_uploaders import ExpectationUploader, TraceUploader from src.collector.models.exception import ( CollectorEngineConfigError, @@ -15,7 +16,9 @@ ) from src.collector.models.expectations import ExpectationResult, ExpectationSummary from src.collector.models.source import Source +from src.collector.protocols.data_fetcher import DataFetcherProtocol from src.collector.protocols.source_handler import SourceHandlerProtocol +from src.collector.types.collector import ExpectationsList from src.collector.utils.retroport_itertools import batched LOG_PREFIX = "[BasicCollectorEngine]" @@ -35,7 +38,7 @@ def __init__( source_handler: SourceHandlerProtocol, oaev_api: OpenAEV, batching: bool = False, - ): + ) -> None: self.name = name self.collector_id = collector_id @@ -62,41 +65,36 @@ def __init__( self.logger = logging.getLogger(__name__) self.current_summary = ExpectationSummary() - self.oaev_detection_helper = None - self.expectation_uploader = None - self.trace_uploader = None + self.oaev_detection_helper = OpenAEVDetectionHelper( + logger=self.logger, + relevant_signatures_types=self.source.signatures, + ) + self.expectation_uploader = ExpectationUploader( + oaev_api=self.oaev_api, + collector_id=self.collector_id, + ) + self.trace_uploader = TraceUploader( + oaev_api=self.oaev_api, + collector_id=self.collector_id, + collector_name=self.name, + ) self.configured = False @property - def data_fetcher_model(self): + def data_fetcher_model(self) -> type[DataFetcherProtocol]: return self.source.data_fetcher_model @property - def signatures(self): + def signatures(self) -> list[SignatureTypes]: return self.source.signatures - def configure_engine(self, config, batching=False): + def configure_engine(self, config, batching=False) -> None: self.logger.info( f"{LOG_PREFIX} Supported signatures: {[sig.value for sig in self.signatures]}" ) self.config = config self.batching = batching - - self.oaev_detection_helper = OpenAEVDetectionHelper( - logger=self.logger, - relevant_signatures_types=self.signatures, - ) - self.expectation_uploader = ExpectationUploader( - oaev_api=self.oaev_api, - collector_id=self.collector_id, - ) - self.trace_uploader = TraceUploader( - oaev_api=self.oaev_api, - collector_id=self.collector_id, - collector_name=self.name, - ) - self._reset_summary() self.configured = True @@ -109,14 +107,14 @@ def _reset_summary(self) -> None: total_processing_time=None, ) - def _filter_supported(self, expectations): + def _filter_supported(self, expectations: ExpectationsList) -> ExpectationsList: return [ exp for exp in expectations if isinstance(exp, (DetectionExpectation, PreventionExpectation)) ] - def _fetch_expectations(self) -> list[DetectionExpectation | PreventionExpectation]: + def _fetch_expectations(self) -> ExpectationsList: """Fetch expectations from OpenAEV. Returns: @@ -141,7 +139,7 @@ def _fetch_expectations(self) -> list[DetectionExpectation | PreventionExpectati self.logger.error(f"{LOG_PREFIX} Error fetching expectations: {e}") return [] - def fetch_and_filter_expectations(self): + def fetch_and_filter_expectations(self) -> ExpectationsList: """fetch expectations and filter out unsupported ones (wrong expectation types)""" self.logger.debug(f"{LOG_PREFIX} Fetching expectations from OpenAEV...") expectations = self._fetch_expectations() @@ -162,7 +160,8 @@ def fetch_and_filter_expectations(self): return expectations def _process_batch( - self, batch: list[DetectionExpectation | PreventionExpectation] + self, + batch: ExpectationsList, ) -> list[ExpectationResult]: """ Processing a single batch of expectations through the following steps: @@ -208,7 +207,7 @@ def _process_batch( if flag: # (5) serialize data as tracedata trace = self.source_handler.serialize_as_tracedata(element) - traces.append(trace) + traces.append(trace.model_dump()) # (6) match expectation (0) with sourcedata (1) matchflag, breakflag = ( @@ -277,13 +276,14 @@ def run_engine(self) -> None: results = [] - batches = [ - expectations, - ] # default: single giant batch of expectations if self.batching: # using a retro-compatible batched # instead of itertools.batched due to python 3.11 support batches = batched(expectations, self.config.expectation_batch_size) + else: + batches = [ + expectations, + ] # default: single giant batch of expectations for batch in batches: self.logger.info( diff --git a/template/src/collector/models/expectations.py b/template/src/collector/models/expectations.py index 800d4360..eee6924f 100644 --- a/template/src/collector/models/expectations.py +++ b/template/src/collector/models/expectations.py @@ -16,8 +16,9 @@ class ExpectationResult(BaseModel): expectation_id: str = Field(..., description="ID of the processed expectation") is_valid: bool = Field(..., description="Whether the expectation was validated") expectation: Any | None = Field(None, description="The original expectation object") - matched_alerts: list[dict[str, Any]] | None = Field( - None, description="List of alerts that matched this expectation" + matched_alerts: list[dict[str, Any]] = Field( + default_factory=list, + description="List of alerts that matched this expectation", ) error_message: str | None = Field( None, description="Error message if processing failed" @@ -29,7 +30,7 @@ class ExpectationResult(BaseModel): @classmethod def from_error( cls, error: Exception, expectation: DetectionExpectation | PreventionExpectation - ): + ) -> "ExpectationResult": """ Produce an ExpectationResult based on an error message and the related expectation @@ -38,9 +39,7 @@ def from_error( expectation_id=str(expectation.inject_expectation_id), is_valid=False, expectation=expectation, - matched_alerts=None, error_message=str(error), - processing_time=None, ) def to_result_text(self) -> str: @@ -197,7 +196,7 @@ def to_api_dict(self) -> dict[str, str]: @classmethod def from_result( cls, result: ExpectationResult, collector_id: str, collector_name: str - ): + ) -> "ExpectationTrace": """ Produce an ExpectationTrace based on the provided ExpectationResult and collector's ID and name @@ -234,26 +233,26 @@ class ExpectationSummary(BaseModel): ) @property - def unsupported(self): + def unsupported(self) -> int: """Number of unsupported expectations received""" return self.received - self.supported @property - def unprocessed(self): + def unprocessed(self) -> int: """Number of expectations skipped during processing""" return self.supported - self.processed @property - def invalid(self): + def invalid(self) -> int: """Number of invalid expectations""" return self.processed - self.valid @property - def total_skipped(self): + def total_skipped(self) -> int: """Number of expectations skipped since receiving (unsupported+unprocessed)""" return self.received - self.processed - def __str__(self): + def __str__(self) -> str: """Return an overview of the summary as a string""" return ( f"{self.received} expectations received, " diff --git a/template/src/collector/models/source.py b/template/src/collector/models/source.py index 83e05a0a..49e52d2f 100644 --- a/template/src/collector/models/source.py +++ b/template/src/collector/models/source.py @@ -59,12 +59,12 @@ def get_expectation_signature_groups( self, signatures: list[SignatureTypes], expectation: DetectionExpectation | PreventionExpectation, - ) -> dict[str, list[dict]]: + ) -> SignatureGroups: """ group the expectation's signatures according to the source provided signatures """ supported_types = {sig_type.value for sig_type in signatures} - signature_groups = {} + signature_groups: SignatureGroups = {} for sig in expectation.inject_expectation_signatures: # ignore unsupported signatures according to source if sig.type.value not in supported_types: @@ -93,7 +93,7 @@ def match_signature_groups_and_oaevdata( for sig_type, signature_data in signature_groups.items(): try: - filtered_data = {sig_type: oaev_data.sig_type} + filtered_data = {sig_type: getattr(oaev_data, sig_type)} except AttributeError: return False match_result = oaev_detection_helper.match_alert_elements( @@ -108,7 +108,6 @@ def serialize_as_tracedata(self, data: SourceDataProtocol) -> TraceData: use pydantic-based TraceData model to serialize then return in dictionary format """ trace = data.to_traces_data() - trace = trace.model_dump() return trace def match_expectation_and_sourcedata( diff --git a/template/src/collector/types/collector.py b/template/src/collector/types/collector.py index 384756f9..5b25d4e7 100644 --- a/template/src/collector/types/collector.py +++ b/template/src/collector/types/collector.py @@ -1,3 +1,10 @@ -from typing import TypeAlias +from typing import Sequence, TypeAlias -SignatureGroups: TypeAlias = list[dict[str, str]] +from pyoaev.apis.inject_expectation.model import ( + DetectionExpectation, + PreventionExpectation, +) + + +ExpectationsList: TypeAlias = Sequence[DetectionExpectation | PreventionExpectation] +SignatureGroups: TypeAlias = dict[str, list[dict[str, str]]] diff --git a/template/tests/collector/engines/test_basic.py b/template/tests/collector/engines/test_basic.py index 77488f03..f3a008b7 100644 --- a/template/tests/collector/engines/test_basic.py +++ b/template/tests/collector/engines/test_basic.py @@ -35,9 +35,9 @@ def test_minimal_init(self): self.assertFalse(collector_engine.configured) self.assertIsNotNone(collector_engine.logger) self.assertIsNotNone(collector_engine.current_summary) - self.assertIsNone(collector_engine.oaev_detection_helper) - self.assertIsNone(collector_engine.expectation_uploader) - self.assertIsNone(collector_engine.trace_uploader) + self.assertIsNotNone(collector_engine.oaev_detection_helper) + self.assertIsNotNone(collector_engine.expectation_uploader) + self.assertIsNotNone(collector_engine.trace_uploader) self.assertEqual(collector_engine.data_fetcher_model, source.data_fetcher_model) self.assertEqual(collector_engine.signatures, source.signatures) @@ -72,9 +72,9 @@ def test_full_init(self): self.assertFalse(collector_engine.configured) self.assertIsNotNone(collector_engine.logger) self.assertIsNotNone(collector_engine.current_summary) - self.assertIsNone(collector_engine.oaev_detection_helper) - self.assertIsNone(collector_engine.expectation_uploader) - self.assertIsNone(collector_engine.trace_uploader) + self.assertIsNotNone(collector_engine.oaev_detection_helper) + self.assertIsNotNone(collector_engine.expectation_uploader) + self.assertIsNotNone(collector_engine.trace_uploader) def test_wrong_source_init(self): """""" @@ -164,6 +164,7 @@ def test_reset_summary(self): name = "my name is" collector_id = "1234abcd" source = MagicMock(spec=module.Source) + source.signatures = [MagicMock()] source_handler = MagicMock(spec_set=SourceHandler) oaev_api = MagicMock(spec_set=module.OpenAEV) @@ -188,6 +189,7 @@ def test_filter_supported(self): name = "my name is" collector_id = "1234abcd" source = MagicMock(spec=module.Source) + source.signatures = [MagicMock()] source_handler = MagicMock(spec_set=SourceHandler) oaev_api = MagicMock(spec_set=module.OpenAEV) @@ -215,6 +217,7 @@ def test_fetch_expectations(self): name = "my name is" collector_id = "1234abcd" source = MagicMock(spec=module.Source) + source.signatures = [MagicMock()] source_handler = MagicMock(spec_set=SourceHandler) oaev_api = MagicMock(spec=module.OpenAEV) @@ -249,6 +252,7 @@ def test_fetch_expectations_api_failure(self): name = "my name is" collector_id = "1234abcd" source = MagicMock(spec=module.Source) + source.signatures = [MagicMock()] source_handler = MagicMock(spec_set=SourceHandler) oaev_api = MagicMock(spec=module.OpenAEV) @@ -280,7 +284,8 @@ def test_fetch_and_filter_expectations( """""" name = "my name is" collector_id = "1234abcd" - source = MagicMock(spec_set=module.Source) + source = MagicMock(spec=module.Source) + source.signatures = [MagicMock()] source_handler = MagicMock(spec_set=SourceHandler) oaev_api = MagicMock(spec_set=module.OpenAEV) fetched_expectations = [ @@ -389,13 +394,13 @@ def test_process_batch( expectation_id=str(expectation1.inject_expectation_id), is_valid=True, expectation=expectation1, - matched_alerts=[source_handler.serialize_as_tracedata.return_value], + matched_alerts=[source_handler.serialize_as_tracedata.return_value.model_dump.return_value], ) m_expectation_result.assert_any_call( expectation_id=str(expectation2.inject_expectation_id), is_valid=False, expectation=expectation2, - matched_alerts=[source_handler.serialize_as_tracedata.return_value], + matched_alerts=[source_handler.serialize_as_tracedata.return_value.model_dump.return_value], ) self.assertEqual(batch_results, [result1, result2]) diff --git a/template/tests/collector/models/test_expectations.py b/template/tests/collector/models/test_expectations.py index f0b6d830..387c5364 100644 --- a/template/tests/collector/models/test_expectations.py +++ b/template/tests/collector/models/test_expectations.py @@ -18,8 +18,8 @@ def test_expectation_result_minimal_init(self): self.assertEqual(expectation_id, expectation_result.expectation_id) self.assertEqual(is_valid, expectation_result.is_valid) + self.assertEqual(expectation_result.matched_alerts, []) self.assertIsNone(expectation_result.expectation) - self.assertIsNone(expectation_result.matched_alerts) self.assertIsNone(expectation_result.error_message) self.assertIsNone(expectation_result.processing_time) @@ -63,7 +63,7 @@ def test_expectation_result_from_error(self): self.assertEqual("id", expectation_result.expectation_id) self.assertFalse(expectation_result.is_valid) self.assertEqual(expectation, expectation_result.expectation) - self.assertIsNone(expectation_result.matched_alerts) + self.assertEqual(expectation_result.matched_alerts, []) self.assertEqual(str(error), expectation_result.error_message) self.assertIsNone(expectation_result.processing_time) diff --git a/template/tests/collector/models/test_source.py b/template/tests/collector/models/test_source.py index 4bf2ca79..4208e8c0 100644 --- a/template/tests/collector/models/test_source.py +++ b/template/tests/collector/models/test_source.py @@ -179,7 +179,7 @@ def test_match_signature_groups_and_oaevdata_success(self): ) oaev_detection_helper.match_alert_elements.assert_called_with( - signature_groups["my_type"], {"my_type": oaev_data.sig_type} + signature_groups["my_type"], {"my_type": oaev_data.my_type} ) self.assertTrue(flag) @@ -198,7 +198,7 @@ def test_match_signature_groups_and_oaevdata_failure(self): ) oaev_detection_helper.match_alert_elements.assert_called_with( - signature_groups["my_type"], {"my_type": oaev_data.sig_type} + signature_groups["my_type"], {"my_type": oaev_data.my_type} ) self.assertFalse(flag) @@ -229,7 +229,6 @@ def test_serialize_as_tracedata(self): module.SourceHandler().serialize_as_tracedata(data) data.to_traces_data.assert_called_once() - data.to_traces_data.return_value.model_dump.assert_called_once() def test_match_expectation_and_sourcedata_prevention_prevented(self): """ From 8307439c0371f84ed9b156328e8c1fa8d273f371 Mon Sep 17 00:00:00 2001 From: guzmud Date: Thu, 7 May 2026 17:51:47 +0200 Subject: [PATCH 15/25] [template] refactor(collector): improving code quality --- template/src/collector/collector.py | 6 +++--- template/src/collector/internals/oaev_uploaders.py | 10 +++++----- .../src/collector/internals/resilient_uploader.py | 6 +++--- template/src/collector/models/data.py | 2 +- template/src/collector/protocols/engine.py | 4 ++-- template/src/collector/types/internals.py | 4 ++-- template/src/collector/utils/retroport_itertools.py | 7 +++++-- template/src/source/template_data_fetcher.py | 2 +- template/src/source/template_source_data.py | 12 ++++++------ 9 files changed, 28 insertions(+), 25 deletions(-) diff --git a/template/src/collector/collector.py b/template/src/collector/collector.py index 34248a15..f3ffbffc 100644 --- a/template/src/collector/collector.py +++ b/template/src/collector/collector.py @@ -18,7 +18,7 @@ LOG_PREFIX = "[Collector]" -class BaseCollector(CollectorDaemon): # type: ignore[misc] +class BaseCollector(CollectorDaemon): """ Generic BaseCollector providing a defined source to a generic collector engine. This collector is use-case agnostic and works with any source provided. @@ -63,7 +63,7 @@ def __init__( collector_type=f"openaev_{self.name}", ) - self.logger.info( # type: ignore[has-type] + self.logger.info( f"{LOG_PREFIX} {self.name} Collector initialized successfully" ) @@ -88,7 +88,7 @@ def __init__( f"Faile to initialize the engine of {self.name} collector: {err}" ) from err - def _setup(self, batching=False) -> None: + def _setup(self, batching: bool = False) -> None: """Set up the collector. Initializes PaloAltoCortexXDR services, expectation handler, expectation manager, diff --git a/template/src/collector/internals/oaev_uploaders.py b/template/src/collector/internals/oaev_uploaders.py index b798c244..caf185c5 100644 --- a/template/src/collector/internals/oaev_uploaders.py +++ b/template/src/collector/internals/oaev_uploaders.py @@ -1,6 +1,6 @@ """Expectation uploader and expectation trace uploader based on the ResilientUploader object""" -from typing import Any, Iterable +from typing import Any, Iterable, Mapping from pyoaev.client import OpenAEV from src.collector.internals.resilient_uploader import ResilientUploader @@ -25,7 +25,7 @@ def __init__(self, oaev_api: OpenAEV, collector_id: str): def expectation_prepare_bulk_data( self, results: list[Any] - ) -> tuple[Iterable[Any], int]: + ) -> tuple[dict[str, Any], int]: """ convert a list of results into the required format for API's bulk_update later down the road @@ -65,11 +65,11 @@ def expectation_prepare_bulk_data( def expectation_bulk_upload(self, bulk_data: Iterable[Any]) -> None: """expectation bulk update using the OpenAEV API""" self.oaev_api.inject_expectation.bulk_update( - inject_expectation_input_by=bulk_data + inject_expectation_input_by_id=bulk_data ) def expectation_unpack_bulk_data( - self, bulk_data: Iterable[Any] + self, bulk_data: Mapping[str, Any] ) -> Iterable[tuple[str, Any]]: """unpack the default expectation bulk data format into a (index,data) iterable""" return bulk_data.items() @@ -144,6 +144,6 @@ def trace_unpack_bulk_data( """unpack the default expectation trace bulk data format into a (index,data) iterable""" return enumerate(traces, 1) - def trace_individual_upload(self, _, trace: Any) -> None: + def trace_individual_upload(self, _: Any, trace: Any) -> None: """expectation trace single upload using the OpenAEV API""" self.oaev_api.inject_expectation_trace.create(trace.to_api_dict()) diff --git a/template/src/collector/internals/resilient_uploader.py b/template/src/collector/internals/resilient_uploader.py index 5b8d27df..b8cf9044 100644 --- a/template/src/collector/internals/resilient_uploader.py +++ b/template/src/collector/internals/resilient_uploader.py @@ -6,7 +6,7 @@ """ import logging -from typing import Any +from typing import Any, Sequence from src.collector.models.exception import ( APIError, @@ -43,7 +43,7 @@ def __init__( self._unpack_bulk_data = _unpack_bulk_data self._individual_upload = _individual_upload - def prepare_bulk_data(self, data: list[Any]) -> list[Any]: + def prepare_bulk_data(self, data: list[Any]) -> Sequence[Any]: """ Using the provided function, prepare the data for bulk upload """ @@ -65,7 +65,7 @@ def prepare_bulk_data(self, data: list[Any]) -> list[Any]: return bulk_data - def bulk_upload_data(self, bulk_data: list[Any]) -> None: + def bulk_upload_data(self, bulk_data: Sequence[Any]) -> None: """ Using the provided functions, attempt to bulk upload and on failure, unpack the bulk data and attempt to diff --git a/template/src/collector/models/data.py b/template/src/collector/models/data.py index d270df5b..35df9bff 100644 --- a/template/src/collector/models/data.py +++ b/template/src/collector/models/data.py @@ -43,7 +43,7 @@ class TraceData(BaseModel): alert_name: str = Field(..., description="Alert name") alert_link: AnyUrl = Field(..., description="Alert link") alert_date: datetime = Field( - ..., description="Alert date", default_factory=lambda: datetime.now(UTC) + default_factory=lambda: datetime.now(UTC), description="Alert date" ) def __str__(self) -> str: diff --git a/template/src/collector/protocols/engine.py b/template/src/collector/protocols/engine.py index 9e3d5704..2cfa537e 100644 --- a/template/src/collector/protocols/engine.py +++ b/template/src/collector/protocols/engine.py @@ -15,8 +15,8 @@ def __init__( source_handler: SourceHandlerProtocol, oaev_api: OpenAEV, batching: bool = False, - ): ... + ) -> None: ... - def configure_engine(self, config, batching=False): ... + def configure_engine(self, config, batching=False) -> None: ... def run_engine(self) -> None: ... diff --git a/template/src/collector/types/internals.py b/template/src/collector/types/internals.py index ba2045f7..0fc633c1 100644 --- a/template/src/collector/types/internals.py +++ b/template/src/collector/types/internals.py @@ -1,6 +1,6 @@ -from typing import Any, Callable, Iterable, TypeAlias +from typing import Any, Callable, Iterable, Mapping, TypeAlias PrepareBulkFunction: TypeAlias = Callable[[list[Any]], tuple[Iterable[Any], int]] BulkUploadFunction: TypeAlias = Callable[[Iterable[Any]], None] -UnpackBulkFunction: TypeAlias = Callable[[Iterable[Any]], Iterable[tuple[Any, Any]]] +UnpackBulkFunction: TypeAlias = Callable[[Iterable[Any] | Mapping[str, Any]], Iterable[tuple[Any, Any]]] IndividualUploadFunction: TypeAlias = Callable[[Any, Any], None] diff --git a/template/src/collector/utils/retroport_itertools.py b/template/src/collector/utils/retroport_itertools.py index 2e3c152a..3c6ea92c 100644 --- a/template/src/collector/utils/retroport_itertools.py +++ b/template/src/collector/utils/retroport_itertools.py @@ -1,8 +1,11 @@ import itertools import sys +from typing import Iterable, Sequence, TypeVar +T = TypeVar("T") -def _batched(iterable, size): + +def _batched(iterable: Sequence[T], size: int) -> Iterable[Sequence[T]]: """ pseudo-itertools.batched for python 3.11 based on https://docs.python.org/3/library/itertools.html#itertools.batched @@ -14,7 +17,7 @@ def _batched(iterable, size): yield batch -def batched(iterable, size): +def batched(iterable: Sequence[T], size: int) -> Iterable[Sequence[T]]: """providing support for itertools.batched in 3.11""" if not (sys.version_info.major >= 3 and sys.version_info.minor >= 12): return _batched(iterable, size) diff --git a/template/src/source/template_data_fetcher.py b/template/src/source/template_data_fetcher.py index 47dad26f..2e81ec04 100644 --- a/template/src/source/template_data_fetcher.py +++ b/template/src/source/template_data_fetcher.py @@ -6,6 +6,6 @@ class TemplateDataFetcher: Placeholder data fetcher class, meant to follow the data fetcher protocol """ - def fetch_data(self): + def fetch_data(self) -> list[TemplateSourceData]: """return placeholder data in the source data format""" return [TemplateSourceData(), TemplateSourceData(), TemplateSourceData()] diff --git a/template/src/source/template_source_data.py b/template/src/source/template_source_data.py index f3f9a68c..7d13fd97 100644 --- a/template/src/source/template_source_data.py +++ b/template/src/source/template_source_data.py @@ -8,28 +8,28 @@ class TemplateSourceData: Placeholder source data, meant to follow the source data protocol """ - def __init__(self): + def __init__(self) -> None: """Generate random placeholder data""" self.value = secrets.token_hex(8) - def to_oaev_data(self): + def to_oaev_data(self) -> OAEVData: """Serialize source data into OAEVData""" return OAEVData(parent_process_name=f"{self.value}") - def to_traces_data(self): + def to_traces_data(self) -> TraceData: """Serialize traces data into TraceData""" return TraceData( alert_name=f"Alert {self.value}", alert_link=f"http://fake.url/{self.value}" ) - def is_prevented(self): + def is_prevented(self) -> bool: """Placeholder analysis of the data to determine if the threat is prevented""" return bool(secrets.randbits(1)) - def is_detected(self): + def is_detected(self) -> bool: """Placeholder analysis of the data to determine if the threat is detected""" return bool(secrets.randbits(1)) - def __str__(self): + def __str__(self) -> str: """Str output of the source data for logging purposes""" return f"{self.value}" From 012d176555afccdae888e12535d78682578de645 Mon Sep 17 00:00:00 2001 From: guzmud Date: Mon, 11 May 2026 09:16:33 +0200 Subject: [PATCH 16/25] [template] feat(types): adding BulkData generic type (+fix test) --- template/src/collector/collector.py | 6 +++--- .../src/collector/internals/oaev_uploaders.py | 19 ++++++++----------- .../collector/internals/resilient_uploader.py | 7 ++++--- template/src/collector/types/collector.py | 1 - template/src/collector/types/internals.py | 9 +++++---- template/src/template_collector.py | 2 +- .../tests/collector/engines/test_basic.py | 8 ++++++-- .../internals/test_oaev_uploaders.py | 2 +- 8 files changed, 28 insertions(+), 26 deletions(-) diff --git a/template/src/collector/collector.py b/template/src/collector/collector.py index f3ffbffc..b1aab93d 100644 --- a/template/src/collector/collector.py +++ b/template/src/collector/collector.py @@ -85,14 +85,14 @@ def __init__( self.set_callback(self.engine.run_engine) except Exception as err: raise CollectorEngineConfigError( - f"Faile to initialize the engine of {self.name} collector: {err}" + f"Failed to initialize the engine of {self.name} collector: {err}" ) from err def _setup(self, batching: bool = False) -> None: """Set up the collector. - Initializes PaloAltoCortexXDR services, expectation handler, expectation manager, - and OpenAEV detection helper. Sets up the collector for processing expectations. + Setup the collector daemon and configure the engine. + Set up the collector for processing expectations. Raises: CollectorSetupError: If collector setup fails. diff --git a/template/src/collector/internals/oaev_uploaders.py b/template/src/collector/internals/oaev_uploaders.py index caf185c5..a96cf932 100644 --- a/template/src/collector/internals/oaev_uploaders.py +++ b/template/src/collector/internals/oaev_uploaders.py @@ -1,10 +1,11 @@ """Expectation uploader and expectation trace uploader based on the ResilientUploader object""" -from typing import Any, Iterable, Mapping +from typing import Any, Iterable from pyoaev.client import OpenAEV from src.collector.internals.resilient_uploader import ResilientUploader from src.collector.models.expectations import ExpectationTrace +from src.collector.types.internals import BulkData LOG_PREFIX = "[Uploader]" @@ -23,9 +24,7 @@ def __init__(self, oaev_api: OpenAEV, collector_id: str): _individual_upload=self.expectation_individual_upload, ) - def expectation_prepare_bulk_data( - self, results: list[Any] - ) -> tuple[dict[str, Any], int]: + def expectation_prepare_bulk_data(self, results: list[Any]) -> tuple[BulkData, int]: """ convert a list of results into the required format for API's bulk_update later down the road @@ -62,14 +61,14 @@ def expectation_prepare_bulk_data( return bulk_data, skipped_count - def expectation_bulk_upload(self, bulk_data: Iterable[Any]) -> None: + def expectation_bulk_upload(self, bulk_data: BulkData) -> None: """expectation bulk update using the OpenAEV API""" self.oaev_api.inject_expectation.bulk_update( inject_expectation_input_by_id=bulk_data ) def expectation_unpack_bulk_data( - self, bulk_data: Mapping[str, Any] + self, bulk_data: BulkData ) -> Iterable[tuple[str, Any]]: """unpack the default expectation bulk data format into a (index,data) iterable""" return bulk_data.items() @@ -99,7 +98,7 @@ def __init__(self, oaev_api: OpenAEV, collector_id: str, collector_name: str): _individual_upload=self.trace_individual_upload, ) - def trace_prepare_bulk_data(self, results: list[Any]) -> tuple[Iterable[Any], int]: + def trace_prepare_bulk_data(self, results: list[Any]) -> tuple[BulkData, int]: """ convert a list of results into the required format for API's bulk_create later down the road @@ -132,15 +131,13 @@ def trace_prepare_bulk_data(self, results: list[Any]) -> tuple[Iterable[Any], in return traces, skipped_count - def trace_bulk_upload(self, traces: Iterable[Any]) -> None: + def trace_bulk_upload(self, traces: BulkData) -> None: """expectation trace bulk upload using the OpenAEV API""" self.oaev_api.inject_expectation_trace.bulk_create( payload={"expectation_traces": [trace.to_api_dict() for trace in traces]} ) - def trace_unpack_bulk_data( - self, traces: Iterable[Any] - ) -> Iterable[tuple[int, Any]]: + def trace_unpack_bulk_data(self, traces: BulkData) -> Iterable[tuple[int, Any]]: """unpack the default expectation trace bulk data format into a (index,data) iterable""" return enumerate(traces, 1) diff --git a/template/src/collector/internals/resilient_uploader.py b/template/src/collector/internals/resilient_uploader.py index b8cf9044..31fe3564 100644 --- a/template/src/collector/internals/resilient_uploader.py +++ b/template/src/collector/internals/resilient_uploader.py @@ -6,7 +6,7 @@ """ import logging -from typing import Any, Sequence +from typing import Any from src.collector.models.exception import ( APIError, @@ -14,6 +14,7 @@ BulkUploadError, ) from src.collector.types.internals import ( + BulkData, BulkUploadFunction, IndividualUploadFunction, PrepareBulkFunction, @@ -43,7 +44,7 @@ def __init__( self._unpack_bulk_data = _unpack_bulk_data self._individual_upload = _individual_upload - def prepare_bulk_data(self, data: list[Any]) -> Sequence[Any]: + def prepare_bulk_data(self, data: list[Any]) -> BulkData: """ Using the provided function, prepare the data for bulk upload """ @@ -65,7 +66,7 @@ def prepare_bulk_data(self, data: list[Any]) -> Sequence[Any]: return bulk_data - def bulk_upload_data(self, bulk_data: Sequence[Any]) -> None: + def bulk_upload_data(self, bulk_data: BulkData) -> None: """ Using the provided functions, attempt to bulk upload and on failure, unpack the bulk data and attempt to diff --git a/template/src/collector/types/collector.py b/template/src/collector/types/collector.py index 5b25d4e7..073c8d0f 100644 --- a/template/src/collector/types/collector.py +++ b/template/src/collector/types/collector.py @@ -5,6 +5,5 @@ PreventionExpectation, ) - ExpectationsList: TypeAlias = Sequence[DetectionExpectation | PreventionExpectation] SignatureGroups: TypeAlias = dict[str, list[dict[str, str]]] diff --git a/template/src/collector/types/internals.py b/template/src/collector/types/internals.py index 0fc633c1..c91c0cc4 100644 --- a/template/src/collector/types/internals.py +++ b/template/src/collector/types/internals.py @@ -1,6 +1,7 @@ -from typing import Any, Callable, Iterable, Mapping, TypeAlias +from typing import Any, Callable, Iterable, Mapping, Sequence, TypeAlias -PrepareBulkFunction: TypeAlias = Callable[[list[Any]], tuple[Iterable[Any], int]] -BulkUploadFunction: TypeAlias = Callable[[Iterable[Any]], None] -UnpackBulkFunction: TypeAlias = Callable[[Iterable[Any] | Mapping[str, Any]], Iterable[tuple[Any, Any]]] +BulkData: TypeAlias = Mapping[str, Any] | Sequence[Any] +PrepareBulkFunction: TypeAlias = Callable[[list[Any]], tuple[BulkData, int]] +BulkUploadFunction: TypeAlias = Callable[[BulkData], None] +UnpackBulkFunction: TypeAlias = Callable[[BulkData], Iterable[tuple[Any, Any]]] IndividualUploadFunction: TypeAlias = Callable[[Any, Any], None] diff --git a/template/src/template_collector.py b/template/src/template_collector.py index 6843bed5..6eff53ec 100644 --- a/template/src/template_collector.py +++ b/template/src/template_collector.py @@ -1,5 +1,5 @@ """ -Template collector meant to easier collector development, +Template collector meant to ease collector development, based on the distinction between the normalized collector engine and the custom source related to the implemented tool/service """ diff --git a/template/tests/collector/engines/test_basic.py b/template/tests/collector/engines/test_basic.py index f3a008b7..268b3815 100644 --- a/template/tests/collector/engines/test_basic.py +++ b/template/tests/collector/engines/test_basic.py @@ -394,13 +394,17 @@ def test_process_batch( expectation_id=str(expectation1.inject_expectation_id), is_valid=True, expectation=expectation1, - matched_alerts=[source_handler.serialize_as_tracedata.return_value.model_dump.return_value], + matched_alerts=[ + source_handler.serialize_as_tracedata.return_value.model_dump.return_value + ], ) m_expectation_result.assert_any_call( expectation_id=str(expectation2.inject_expectation_id), is_valid=False, expectation=expectation2, - matched_alerts=[source_handler.serialize_as_tracedata.return_value.model_dump.return_value], + matched_alerts=[ + source_handler.serialize_as_tracedata.return_value.model_dump.return_value + ], ) self.assertEqual(batch_results, [result1, result2]) diff --git a/template/tests/collector/internals/test_oaev_uploaders.py b/template/tests/collector/internals/test_oaev_uploaders.py index d41ff8b0..1a7024ec 100644 --- a/template/tests/collector/internals/test_oaev_uploaders.py +++ b/template/tests/collector/internals/test_oaev_uploaders.py @@ -79,7 +79,7 @@ def test_expectation_uploader_expectation_bulk_upload(self): self.expectation_uploader.expectation_bulk_upload(bulk_data) self.oaev_api.inject_expectation.bulk_update.assert_called_once_with( - inject_expectation_input_by=bulk_data + inject_expectation_input_by_id=bulk_data ) def test_expectation_uploader_expectation_unpack_bulk_data(self): From c6eb06adb082bcd1c425fab2a15794bd5b763134 Mon Sep 17 00:00:00 2001 From: guzmud Date: Mon, 11 May 2026 09:39:25 +0200 Subject: [PATCH 17/25] [template] fix(collector): add slugify for collector type --- template/pyproject.toml | 1 + template/src/collector/collector.py | 6 +++++- template/src/collector/engines/basic.py | 11 +++-------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/template/pyproject.toml b/template/pyproject.toml index 432c36d9..f307d9ca 100644 --- a/template/pyproject.toml +++ b/template/pyproject.toml @@ -22,6 +22,7 @@ pyoaev = [ pydantic = "^2.11.7" pydantic-settings = "^2.11.0" requests = "^2.32.5" +python-slugify = {extras = ["unidecode"], version = "^8.0.4"} [tool.poetry.extras] prod = ["pyoaev"] diff --git a/template/src/collector/collector.py b/template/src/collector/collector.py index b1aab93d..f3c44107 100644 --- a/template/src/collector/collector.py +++ b/template/src/collector/collector.py @@ -1,6 +1,7 @@ import logging from pyoaev.daemons import CollectorDaemon +from slugify import slugify from src.collector.engines.basic import BasicCollectorEngine from src.collector.models.exception import ( CollectorConfigError, @@ -60,7 +61,7 @@ def __init__( super().__init__( configuration=self.config.to_daemon_config(), - collector_type=f"openaev_{self.name}", + collector_type=f"openaev_{slugify(self.name)}", ) self.logger.info( @@ -84,6 +85,9 @@ def __init__( ) self.set_callback(self.engine.run_engine) except Exception as err: + self.logger.info( + f"{LOG_PREFIX} {self.name} Failure during collector engine configuration: {err}" + ) raise CollectorEngineConfigError( f"Failed to initialize the engine of {self.name} collector: {err}" ) from err diff --git a/template/src/collector/engines/basic.py b/template/src/collector/engines/basic.py index 7490c452..e4294144 100644 --- a/template/src/collector/engines/basic.py +++ b/template/src/collector/engines/basic.py @@ -43,22 +43,17 @@ def __init__( self.collector_id = collector_id if source and not isinstance(source, Source): - # TODO custom exception + logging - raise TypeError("Source provided is not of type Source") # TODO + raise TypeError("Source provided is not of type Source") self.source = source if source_handler and not isinstance(source_handler, SourceHandlerProtocol): - # TODO custom exception + logging raise TypeError( "Source handler provided does not follow source handler protocol" - ) # TODO + ) self.source_handler = source_handler if oaev_api and not isinstance(oaev_api, OpenAEV): - # TODO custom exception + logging - raise TypeError( - "Source handler provided does not follow source handler protocol" - ) # TODO + raise TypeError("OAEV API must be of OpenAEV type") self.oaev_api = oaev_api self.batching = batching From 50443be2e737cbb5c913903a855c9e2db6757bfc Mon Sep 17 00:00:00 2001 From: guzmud Date: Mon, 11 May 2026 10:44:11 +0200 Subject: [PATCH 18/25] [template] fix(uploaders): error between results and valid_results --- template/src/collector/internals/oaev_uploaders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/src/collector/internals/oaev_uploaders.py b/template/src/collector/internals/oaev_uploaders.py index a96cf932..a300002c 100644 --- a/template/src/collector/internals/oaev_uploaders.py +++ b/template/src/collector/internals/oaev_uploaders.py @@ -111,7 +111,7 @@ def trace_prepare_bulk_data(self, results: list[Any]) -> tuple[BulkData, int]: traces = [] skipped_count = 0 - for result in results: + for result in valid_results: try: # skipping result without expectation_id if not result.expectation_id: From 337a7a6db95816302a42810f05c66ae4f5db99a6 Mon Sep 17 00:00:00 2001 From: guzmud Date: Mon, 11 May 2026 10:55:59 +0200 Subject: [PATCH 19/25] [template] fix(config): wrong aliases in config_loader --- template/src/models/settings/config_loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/template/src/models/settings/config_loader.py b/template/src/models/settings/config_loader.py index dbba6622..93525619 100644 --- a/template/src/models/settings/config_loader.py +++ b/template/src/models/settings/config_loader.py @@ -27,12 +27,12 @@ class ConfigLoaderCollector(_ConfigLoaderCollector): """ id: str = Field( - alias="Collector_ID", + alias="COLLECTOR_ID", default="template--0413e3c7-5c9e-46f5-adc4-33832e9b49a1", description="A unique UUIDv4 identifier for this collector instance.", ) name: str = Field( - alias="Collector_NAME", + alias="COLLECTOR_NAME", default="Template", description="Name of the collector.", ) From b7732aee19eb233cb4a7e911d88ce74f56a338cf Mon Sep 17 00:00:00 2001 From: guzmud Date: Wed, 13 May 2026 14:07:49 +0200 Subject: [PATCH 20/25] [template] fix(engine): moving try/except to inside the batch loop --- template/src/collector/engines/basic.py | 40 +++++++++++++------ .../tests/collector/engines/test_basic.py | 37 +++++++++++++---- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/template/src/collector/engines/basic.py b/template/src/collector/engines/basic.py index e4294144..e4f6a975 100644 --- a/template/src/collector/engines/basic.py +++ b/template/src/collector/engines/basic.py @@ -171,6 +171,7 @@ def _process_batch( then return a list of all the results produced from the batch """ batch_results = [] + try: # (1) fetch data self.logger.info( @@ -178,8 +179,24 @@ def _process_batch( f"data fetcher {self.data_fetcher_model} to source handler" ) data = self.source_handler.get_source_data(self.data_fetcher_model()) + except Exception as err: # per batch + batch_results = [ + ExpectationResult.from_error( + ExpectationProcessingError( + f"Batch processing error during data fetching: {err}" + ), + expectation, + ) + for expectation in batch + ] + self.logger.error( + f"{LOG_PREFIX} Error processing batch during data fetching: {err}" + ) + return batch_results - for expectation in batch: + error_count = 0 + for expectation in batch: + try: matched = False traces = [] for element in data: @@ -222,19 +239,18 @@ def _process_batch( expectation=expectation, matched_alerts=traces, ) - batch_results.append(result) - self.logger.info( - f"{LOG_PREFIX} Batch processed: {len(batch_results)} results" - ) - except Exception as err: # per batch - batch_results = [ - ExpectationResult.from_error( - ExpectationProcessingError(f"Batch processing error: {err}"), + except Exception as err: + result = ExpectationResult.from_error( + ExpectationProcessingError(f"Processing error: {err}"), expectation, ) - for expectation in batch - ] - self.logger.error(f"{LOG_PREFIX} Error processing batch: {err}") + self.logger.error(f"{LOG_PREFIX} Processing batch: {err}") + error_count += 1 + batch_results.append(result) + + self.logger.info( + f"{LOG_PREFIX} Batch processed: {len(batch_results)} results (including {error_count} errors))" + ) return batch_results diff --git a/template/tests/collector/engines/test_basic.py b/template/tests/collector/engines/test_basic.py index 268b3815..9cd578ac 100644 --- a/template/tests/collector/engines/test_basic.py +++ b/template/tests/collector/engines/test_basic.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, patch, sentinel import src.collector.engines.basic as module from src.collector.models.source import SourceHandler @@ -313,12 +313,14 @@ def test_fetch_and_filter_expectations( self.assertEqual(collector_engine.current_summary.received, 1) self.assertEqual(collector_engine.current_summary.supported, 1) + @patch.object(module, "ExpectationProcessingError") @patch.object(module, "ExpectationResult") @patch.object(module.BasicCollectorEngine, "_reset_summary") def test_process_batch( self, m_reset_summary, m_expectation_result, + m_expectation_processing_error, ): """""" name = "my name is" @@ -338,15 +340,20 @@ def test_process_batch( source_handler.match_expectation_and_sourcedata.side_effect = [ [True, True], [False, False], + Exception("Nope"), ] oaev_api = MagicMock(spec_set=module.OpenAEV) expectation1 = MagicMock() expectation2 = MagicMock() - batch = [expectation1, expectation2] + expectation3 = MagicMock() + batch = [expectation1, expectation2, expectation3] result1 = MagicMock() result2 = MagicMock() + result3 = MagicMock() m_expectation_result.side_effect = [result1, result2] + m_expectation_result.from_error.return_value = result3 + m_expectation_processing_error.return_value = sentinel.exp_pro_error collector_engine = module.BasicCollectorEngine( name=name, @@ -363,32 +370,42 @@ def test_process_batch( batch_results = collector_engine._process_batch(batch) source_handler.get_source_data.assert_called_with(source.data_fetcher_model()) - self.assertEqual(source_handler.serialize_as_oaevdata._mock_call_count, 2) + + self.assertEqual(source_handler.serialize_as_oaevdata._mock_call_count, 3) source_handler.serialize_as_oaevdata.assert_called_with(data_element) + self.assertEqual( - source_handler.get_expectation_signature_groups._mock_call_count, 2 + source_handler.get_expectation_signature_groups._mock_call_count, 3 ) source_handler.get_expectation_signature_groups.assert_any_call( source.signatures, expectation1 ) - source_handler.get_expectation_signature_groups.assert_called_with( + source_handler.get_expectation_signature_groups.assert_any_call( source.signatures, expectation2 ) + source_handler.get_expectation_signature_groups.assert_called_with( + source.signatures, expectation3 + ) + source_handler.match_signature_groups_and_oaevdata.assert_any_call( source_handler.get_expectation_signature_groups.return_value, source_handler.serialize_as_oaevdata.return_value, collector_engine.oaev_detection_helper, ) source_handler.serialize_as_tracedata.assert_called_with(data_element) + self.assertEqual( - source_handler.match_expectation_and_sourcedata._mock_call_count, 2 + source_handler.match_expectation_and_sourcedata._mock_call_count, 3 ) source_handler.match_expectation_and_sourcedata.assert_any_call( expectation1, data_element ) - source_handler.match_expectation_and_sourcedata.assert_called_with( + source_handler.match_expectation_and_sourcedata.assert_any_call( expectation2, data_element ) + source_handler.match_expectation_and_sourcedata.assert_called_with( + expectation3, data_element + ) m_expectation_result.assert_any_call( expectation_id=str(expectation1.inject_expectation_id), @@ -406,7 +423,11 @@ def test_process_batch( source_handler.serialize_as_tracedata.return_value.model_dump.return_value ], ) - self.assertEqual(batch_results, [result1, result2]) + m_expectation_processing_error.assert_called_with("Processing error: Nope") + m_expectation_result.from_error.assert_called_with( + sentinel.exp_pro_error, expectation3 + ) + self.assertEqual(batch_results, [result1, result2, result3]) @patch.object(module, "TraceUploader") @patch.object(module, "ExpectationUploader") From 576621bb1fa0819288bf3f673e88cce72e91e420 Mon Sep 17 00:00:00 2001 From: guzmud Date: Wed, 13 May 2026 14:19:15 +0200 Subject: [PATCH 21/25] [template] fix(models): fixing missing proper default factory in data model --- template/src/collector/models/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/src/collector/models/data.py b/template/src/collector/models/data.py index 35df9bff..d15d3643 100644 --- a/template/src/collector/models/data.py +++ b/template/src/collector/models/data.py @@ -11,7 +11,7 @@ class OAEVData(BaseModel, extra="allow"): Apart from context, the allowed fields are signature types (e.g. parent_process_name) """ - __pydantic_extra__: dict[str, str] = {} # or should it be dict[str, Any] + __pydantic_extra__: dict[str, str] = Field(default_factory=dict) _allowed_values: ClassVar[frozenset[str]] = frozenset( [sig.value for sig in SignatureTypes] ) From 49e2234b8d45910fafdf69544278ce318737d1fe Mon Sep 17 00:00:00 2001 From: guzmud Date: Wed, 13 May 2026 16:47:31 +0200 Subject: [PATCH 22/25] [template] feat(settings): moving template config to a generic custom config --- template/src/collector/collector.py | 42 ++++++++++++------- template/src/models/settings/__init__.py | 4 +- template/src/models/settings/config_loader.py | 22 +++++----- ...{template_configs.py => custom_configs.py} | 14 +++---- 4 files changed, 46 insertions(+), 36 deletions(-) rename template/src/models/settings/{template_configs.py => custom_configs.py} (70%) diff --git a/template/src/collector/collector.py b/template/src/collector/collector.py index f3c44107..55fab7a9 100644 --- a/template/src/collector/collector.py +++ b/template/src/collector/collector.py @@ -29,7 +29,7 @@ def __init__( self, name: str, source: Source, - source_handler: SourceHandlerProtocol | None = None, + source_handler_model: type[SourceHandlerProtocol] | None = None, engine_model: type[CollectorEngineProtocol] | None = None, ) -> None: """Initialize the collector. @@ -41,33 +41,43 @@ def __init__( self.name = name try: + self.config = ConfigLoader() + + super().__init__( + configuration=self.config.to_daemon_config(), + collector_type=f"openaev_{slugify(self.name)}", + ) + + self.logger.info( + f"{LOG_PREFIX} {self.name} Collector initialized successfully" + ) + if source and not isinstance(source, Source): + self.logger.error( + f"{LOG_PREFIX} Source provided is not of type Source" + ) raise TypeError("Source provided is not of type Source") self.source = source - if source_handler and not isinstance(source_handler, SourceHandlerProtocol): + if source_handler_model and not issubclass(source_handler_model, SourceHandlerProtocol): + self.logger.error( + f"{LOG_PREFIX} Source handler model provided does not follow source handler protocol" + ) raise TypeError( - "Source handler provided does not follow source handler protocol" + "Source handler model provided does not follow source handler protocol" ) - self.source_handler = source_handler or SourceHandler() + source_handler_model = source_handler_model or SourceHandler + self.source_handler = source_handler_model(self.config.custom) if engine_model and not issubclass(engine_model, CollectorEngineProtocol): + self.logger.error( + f"{LOG_PREFIX} Engine model provided does not follow collector engine protocol" + ) raise TypeError( "Engine model provided does not follow collector engine protocol" ) self.engine_model = engine_model or BasicCollectorEngine - self.config = ConfigLoader() - - super().__init__( - configuration=self.config.to_daemon_config(), - collector_type=f"openaev_{slugify(self.name)}", - ) - - self.logger.info( - f"{LOG_PREFIX} {self.name} Collector initialized successfully" - ) - except Exception as err: logging.basicConfig(level=logging.ERROR) self.logger = logging.getLogger(__name__) @@ -108,7 +118,7 @@ def _setup(self, batching: bool = False) -> None: super()._setup() self.logger.debug(f"{LOG_PREFIX} Configuring the collector engine...") - self.engine.configure_engine(self.config.template, batching=batching) + self.engine.configure_engine(self.config.custom, batching=batching) self.logger.info(f"{LOG_PREFIX} Collector setup completed successfully") diff --git a/template/src/models/settings/__init__.py b/template/src/models/settings/__init__.py index c5c9ad18..21bf9f17 100644 --- a/template/src/models/settings/__init__.py +++ b/template/src/models/settings/__init__.py @@ -3,11 +3,11 @@ _ConfigLoaderCollector, _ConfigLoaderOAEV, ) -from src.models.settings.template_configs import _ConfigLoaderTemplate +from src.models.settings.custom_configs import _ConfigLoaderCustom __all__ = [ "ConfigBaseSettings", "_ConfigLoaderCollector", "_ConfigLoaderOAEV", - "_ConfigLoaderTemplate", + "_ConfigLoaderCustom", ] diff --git a/template/src/models/settings/config_loader.py b/template/src/models/settings/config_loader.py index 93525619..3dbaf793 100644 --- a/template/src/models/settings/config_loader.py +++ b/template/src/models/settings/config_loader.py @@ -15,7 +15,7 @@ ConfigBaseSettings, _ConfigLoaderCollector, _ConfigLoaderOAEV, - _ConfigLoaderTemplate, + _ConfigLoaderCustom, ) @@ -33,7 +33,7 @@ class ConfigLoaderCollector(_ConfigLoaderCollector): ) name: str = Field( alias="COLLECTOR_NAME", - default="Template", + default="Custom", description="Name of the collector.", ) @@ -41,7 +41,7 @@ class ConfigLoaderCollector(_ConfigLoaderCollector): class ConfigLoader(ConfigBaseSettings): """Configuration loader for the collector. - Main configuration class that combines OpenAEV, collector, and Template + Main configuration class that combines OpenAEV, collector, and custom settings. Supports loading from YAML files, environment variables, and provides methods for converting to daemon-compatible format. """ @@ -54,9 +54,9 @@ class ConfigLoader(ConfigBaseSettings): default_factory=ConfigLoaderCollector, # type: ignore[unused-ignore] description="Collector configurations.", ) - template: _ConfigLoaderTemplate = Field( - default_factory=_ConfigLoaderTemplate, - description="Template configurations.", + custom: _ConfigLoaderCustom = Field( + default_factory=_ConfigLoaderCustom, + description="Custom configurations.", ) @classmethod @@ -140,11 +140,11 @@ def to_daemon_config(self) -> Configuration: "data": int(self.collector.period.total_seconds()) }, # type: ignore[union-attr] "collector_icon_filepath": {"data": self.collector.icon_filepath}, - # Template configuration (flattened) - "template_key": {"data": self.template.key}, - "template_time_window": {"data": self.template.time_window}, - "template_expectation_batch_size": { - "data": self.template.expectation_batch_size + # Custom configuration (flattened) + "custom_key": {"data": self.custom.key}, + "custom_time_window": {"data": self.custom.time_window}, + "custom_expectation_batch_size": { + "data": self.custom.expectation_batch_size }, }, config_base_model=self, diff --git a/template/src/models/settings/template_configs.py b/template/src/models/settings/custom_configs.py similarity index 70% rename from template/src/models/settings/template_configs.py rename to template/src/models/settings/custom_configs.py index 142e0e6a..e6cf2a44 100644 --- a/template/src/models/settings/template_configs.py +++ b/template/src/models/settings/custom_configs.py @@ -1,4 +1,4 @@ -"""Configuration for Template integration.""" +"""Configuration for custom integration.""" from datetime import timedelta @@ -6,25 +6,25 @@ from src.models.settings import ConfigBaseSettings -class _ConfigLoaderTemplate(ConfigBaseSettings): - """Template API configuration settings. +class _ConfigLoaderCustom(ConfigBaseSettings): + """Custom API configuration settings. Contains connection details, timing parameters, and retry settings - for Template API integration. + for custom API integration. """ key: str | None = Field( - alias="TEMPLATE_KEY", + alias="CUSTOM_KEY", default="value", description="key value example for configuration.", ) time_window: timedelta = Field( - alias="TEMPLATE_TIME_WINDOW", + alias="CUSTOM_TIME_WINDOW", default=timedelta(hours=1), description="Time window for Template threat searches when no date signatures are provided (ISO 8601 format).", ) expectation_batch_size: int = Field( - alias="TEMPLATE_EXPECTATION_BATCH_SIZE", + alias="CUSTOM_EXPECTATION_BATCH_SIZE", default=50, description="Number of expectations to process in each batch for batch-based processing.", ) From fb2a085a3a31c4caf98ed71a43e1ab71dc471e28 Mon Sep 17 00:00:00 2001 From: guzmud Date: Wed, 13 May 2026 16:51:41 +0200 Subject: [PATCH 23/25] [template] feat(collector): propagating custom config to sourcehandler and datafetcher --- template/src/collector/collector.py | 8 +-- template/src/collector/engines/basic.py | 8 ++- template/src/collector/models/source.py | 8 ++- .../src/collector/protocols/data_fetcher.py | 3 + template/src/collector/protocols/engine.py | 5 +- .../src/collector/protocols/source_handler.py | 3 +- template/src/collector/types/collector.py | 2 + template/src/models/settings/config_loader.py | 2 +- template/src/source/template_data_fetcher.py | 5 ++ .../tests/collector/engines/test_basic.py | 5 +- .../tests/collector/models/test_source.py | 65 +++++++++++++++---- template/tests/collector/test_collector.py | 31 ++++++--- 12 files changed, 112 insertions(+), 33 deletions(-) diff --git a/template/src/collector/collector.py b/template/src/collector/collector.py index 55fab7a9..8649684f 100644 --- a/template/src/collector/collector.py +++ b/template/src/collector/collector.py @@ -53,13 +53,13 @@ def __init__( ) if source and not isinstance(source, Source): - self.logger.error( - f"{LOG_PREFIX} Source provided is not of type Source" - ) + self.logger.error(f"{LOG_PREFIX} Source provided is not of type Source") raise TypeError("Source provided is not of type Source") self.source = source - if source_handler_model and not issubclass(source_handler_model, SourceHandlerProtocol): + if source_handler_model and not issubclass( + source_handler_model, SourceHandlerProtocol + ): self.logger.error( f"{LOG_PREFIX} Source handler model provided does not follow source handler protocol" ) diff --git a/template/src/collector/engines/basic.py b/template/src/collector/engines/basic.py index e4f6a975..9b2daba5 100644 --- a/template/src/collector/engines/basic.py +++ b/template/src/collector/engines/basic.py @@ -18,7 +18,7 @@ from src.collector.models.source import Source from src.collector.protocols.data_fetcher import DataFetcherProtocol from src.collector.protocols.source_handler import SourceHandlerProtocol -from src.collector.types.collector import ExpectationsList +from src.collector.types.collector import CustomConfig, ExpectationsList from src.collector.utils.retroport_itertools import batched LOG_PREFIX = "[BasicCollectorEngine]" @@ -84,7 +84,7 @@ def data_fetcher_model(self) -> type[DataFetcherProtocol]: def signatures(self) -> list[SignatureTypes]: return self.source.signatures - def configure_engine(self, config, batching=False) -> None: + def configure_engine(self, config: CustomConfig, batching: bool = False) -> None: self.logger.info( f"{LOG_PREFIX} Supported signatures: {[sig.value for sig in self.signatures]}" ) @@ -178,7 +178,9 @@ def _process_batch( f"{LOG_PREFIX} Fetching data providing " f"data fetcher {self.data_fetcher_model} to source handler" ) - data = self.source_handler.get_source_data(self.data_fetcher_model()) + data = self.source_handler.get_source_data( + self.data_fetcher_model(self.source_handler.config) + ) except Exception as err: # per batch batch_results = [ ExpectationResult.from_error( diff --git a/template/src/collector/models/source.py b/template/src/collector/models/source.py index 49e52d2f..ce60cfa2 100644 --- a/template/src/collector/models/source.py +++ b/template/src/collector/models/source.py @@ -9,7 +9,7 @@ from src.collector.protocols.data_fetcher import DataFetcherProtocol from src.collector.protocols.source_data import SourceDataProtocol from src.collector.protocols.source_handler import SourceHandlerProtocol -from src.collector.types.collector import SignatureGroups +from src.collector.types.collector import CustomConfig, SignatureGroups class Source(BaseModel): @@ -38,6 +38,12 @@ class SourceHandler(SourceHandlerProtocol): - how to match an expectation and the source data to check for detection/prevention """ + def __init__(self, config: CustomConfig) -> None: + """ + attach the source handler object the custom config provided through the base collector + """ + self.config = config + def get_source_data( self, data_fetcher: DataFetcherProtocol ) -> list[SourceDataProtocol]: diff --git a/template/src/collector/protocols/data_fetcher.py b/template/src/collector/protocols/data_fetcher.py index 2603487c..dbbe6a6b 100644 --- a/template/src/collector/protocols/data_fetcher.py +++ b/template/src/collector/protocols/data_fetcher.py @@ -1,8 +1,11 @@ from typing import Protocol, runtime_checkable from src.collector.protocols.source_data import SourceDataProtocol +from src.collector.types.collector import CustomConfig @runtime_checkable class DataFetcherProtocol(Protocol): + def __init__(self, config: CustomConfig) -> None: ... + def fetch_data(self) -> list[SourceDataProtocol]: ... diff --git a/template/src/collector/protocols/engine.py b/template/src/collector/protocols/engine.py index 2cfa537e..b3aede02 100644 --- a/template/src/collector/protocols/engine.py +++ b/template/src/collector/protocols/engine.py @@ -3,6 +3,7 @@ from pyoaev.client import OpenAEV from src.collector.models.source import Source from src.collector.protocols.source_handler import SourceHandlerProtocol +from src.collector.types.collector import CustomConfig @runtime_checkable @@ -17,6 +18,8 @@ def __init__( batching: bool = False, ) -> None: ... - def configure_engine(self, config, batching=False) -> None: ... + def configure_engine( + self, config: CustomConfig, batching: bool = False + ) -> None: ... def run_engine(self) -> None: ... diff --git a/template/src/collector/protocols/source_handler.py b/template/src/collector/protocols/source_handler.py index 3cc7a4df..71dec2e0 100644 --- a/template/src/collector/protocols/source_handler.py +++ b/template/src/collector/protocols/source_handler.py @@ -9,11 +9,12 @@ from src.collector.models.data import OAEVData, TraceData from src.collector.protocols.data_fetcher import DataFetcherProtocol from src.collector.protocols.source_data import SourceDataProtocol -from src.collector.types.collector import SignatureGroups +from src.collector.types.collector import CustomConfig, SignatureGroups @runtime_checkable class SourceHandlerProtocol(Protocol): + def __init__(self, config: CustomConfig) -> None: ... def get_source_data( self, data_fetcher: DataFetcherProtocol diff --git a/template/src/collector/types/collector.py b/template/src/collector/types/collector.py index 073c8d0f..99836e07 100644 --- a/template/src/collector/types/collector.py +++ b/template/src/collector/types/collector.py @@ -4,6 +4,8 @@ DetectionExpectation, PreventionExpectation, ) +from src.models.settings.custom_configs import _ConfigLoaderCustom +CustomConfig: TypeAlias = _ConfigLoaderCustom ExpectationsList: TypeAlias = Sequence[DetectionExpectation | PreventionExpectation] SignatureGroups: TypeAlias = dict[str, list[dict[str, str]]] diff --git a/template/src/models/settings/config_loader.py b/template/src/models/settings/config_loader.py index 3dbaf793..019ed23a 100644 --- a/template/src/models/settings/config_loader.py +++ b/template/src/models/settings/config_loader.py @@ -14,8 +14,8 @@ from src.models.settings import ( ConfigBaseSettings, _ConfigLoaderCollector, - _ConfigLoaderOAEV, _ConfigLoaderCustom, + _ConfigLoaderOAEV, ) diff --git a/template/src/source/template_data_fetcher.py b/template/src/source/template_data_fetcher.py index 2e81ec04..652d672b 100644 --- a/template/src/source/template_data_fetcher.py +++ b/template/src/source/template_data_fetcher.py @@ -1,3 +1,4 @@ +from src.collector.types.collector import CustomConfig from src.source.template_source_data import TemplateSourceData @@ -6,6 +7,10 @@ class TemplateDataFetcher: Placeholder data fetcher class, meant to follow the data fetcher protocol """ + def __init__(self, custom_config: CustomConfig) -> None: + """attaching the custom configuration to the data fetcher object""" + self.config = custom_config + def fetch_data(self) -> list[TemplateSourceData]: """return placeholder data in the source data format""" return [TemplateSourceData(), TemplateSourceData(), TemplateSourceData()] diff --git a/template/tests/collector/engines/test_basic.py b/template/tests/collector/engines/test_basic.py index 9cd578ac..b24427ba 100644 --- a/template/tests/collector/engines/test_basic.py +++ b/template/tests/collector/engines/test_basic.py @@ -333,6 +333,7 @@ def test_process_batch( ] source.data_fetcher_model = data_fetcher_model source_handler = MagicMock(spec=SourceHandler) + source_handler.config = MagicMock() data_element = MagicMock() source_handler.get_source_data.return_value = [ data_element, @@ -369,7 +370,9 @@ def test_process_batch( batch_results = collector_engine._process_batch(batch) - source_handler.get_source_data.assert_called_with(source.data_fetcher_model()) + source_handler.get_source_data.assert_called_with( + source.data_fetcher_model(source_handler.config) + ) self.assertEqual(source_handler.serialize_as_oaevdata._mock_call_count, 3) source_handler.serialize_as_oaevdata.assert_called_with(data_element) diff --git a/template/tests/collector/models/test_source.py b/template/tests/collector/models/test_source.py index 4208e8c0..43023855 100644 --- a/template/tests/collector/models/test_source.py +++ b/template/tests/collector/models/test_source.py @@ -106,14 +106,27 @@ def test_source_wrong_signatures_init(self): class SourceHandlerTest(unittest.TestCase): + def test_source_handler_init(self): + """ + test the proper init of the SourceHandler object + """ + config = MagicMock() + + source_handler = module.SourceHandler(config=config) + + self.assertEqual(config, source_handler.config) + def test_get_source_data(self): """ assert the calls made to data fetcher by source handler for the get_source_data function """ + config = MagicMock() data_fetcher = MagicMock() - module.SourceHandler().get_source_data(data_fetcher) + source_handler = module.SourceHandler(config=config) + + source_handler.get_source_data(data_fetcher) data_fetcher.fetch_data.assert_called_once() @@ -123,8 +136,11 @@ def test_serialize_as_oaevdata(self): for the serialize_as_oaevdata function """ data = MagicMock() + config = MagicMock() - module.SourceHandler().serialize_as_oaevdata(data) + source_handler = module.SourceHandler(config=config) + + source_handler.serialize_as_oaevdata(data) data.to_oaev_data.assert_called_once() @@ -149,8 +165,11 @@ def test_get_expectation_signature_groups(self): end_date_ies, ] ) + config = MagicMock() + + source_handler = module.SourceHandler(config=config) - signature_groups = module.SourceHandler().get_expectation_signature_groups( + signature_groups = source_handler.get_expectation_signature_groups( signatures, expectation ) @@ -173,8 +192,11 @@ def test_match_signature_groups_and_oaevdata_success(self): oaev_data = MagicMock() oaev_detection_helper = MagicMock() oaev_detection_helper.match_alert_elements.return_value = True + config = MagicMock() - flag = module.SourceHandler().match_signature_groups_and_oaevdata( + source_handler = module.SourceHandler(config=config) + + flag = source_handler.match_signature_groups_and_oaevdata( signature_groups, oaev_data, oaev_detection_helper ) @@ -192,8 +214,11 @@ def test_match_signature_groups_and_oaevdata_failure(self): oaev_data = MagicMock() oaev_detection_helper = MagicMock() oaev_detection_helper.match_alert_elements.return_value = False + config = MagicMock() + + source_handler = module.SourceHandler(config=config) - flag = module.SourceHandler().match_signature_groups_and_oaevdata( + flag = source_handler.match_signature_groups_and_oaevdata( signature_groups, oaev_data, oaev_detection_helper ) @@ -211,8 +236,11 @@ def test_match_signature_groups_and_oaevdata_empty(self): oaev_data = None oaev_detection_helper = MagicMock() oaev_detection_helper.match_alert_elements.return_value = True + config = MagicMock() - flag = module.SourceHandler().match_signature_groups_and_oaevdata( + source_handler = module.SourceHandler(config=config) + + flag = source_handler.match_signature_groups_and_oaevdata( signature_groups, oaev_data, oaev_detection_helper ) @@ -225,8 +253,11 @@ def test_serialize_as_tracedata(self): for the serialize_as_tracedata function """ data = MagicMock() + config = MagicMock() + + source_handler = module.SourceHandler(config=config) - module.SourceHandler().serialize_as_tracedata(data) + source_handler.serialize_as_tracedata(data) data.to_traces_data.assert_called_once() @@ -238,8 +269,11 @@ def test_match_expectation_and_sourcedata_prevention_prevented(self): data = MagicMock() data.is_prevented.return_value = True data.is_detected.return_value = True + config = MagicMock() - matchflag, breakflag = module.SourceHandler().match_expectation_and_sourcedata( + source_handler = module.SourceHandler(config=config) + + matchflag, breakflag = source_handler.match_expectation_and_sourcedata( expectation, data ) @@ -256,8 +290,11 @@ def test_match_expectation_and_sourcedata_prevention_not_prevented(self): data = MagicMock() data.is_prevented.return_value = False data.is_detected.return_value = True + config = MagicMock() + + source_handler = module.SourceHandler(config=config) - matchflag, breakflag = module.SourceHandler().match_expectation_and_sourcedata( + matchflag, breakflag = source_handler.match_expectation_and_sourcedata( expectation, data ) @@ -274,8 +311,11 @@ def test_match_expectation_and_sourcedata_detection_detected(self): data = MagicMock() data.is_prevented.return_value = True data.is_detected.return_value = True + config = MagicMock() - matchflag, breakflag = module.SourceHandler().match_expectation_and_sourcedata( + source_handler = module.SourceHandler(config=config) + + matchflag, breakflag = source_handler.match_expectation_and_sourcedata( expectation, data ) @@ -292,8 +332,11 @@ def test_match_expectation_and_sourcedata_detection_not_detected(self): data = MagicMock() data.is_prevented.return_value = True data.is_detected.return_value = False + config = MagicMock() + + source_handler = module.SourceHandler(config=config) - matchflag, breakflag = module.SourceHandler().match_expectation_and_sourcedata( + matchflag, breakflag = source_handler.match_expectation_and_sourcedata( expectation, data ) diff --git a/template/tests/collector/test_collector.py b/template/tests/collector/test_collector.py index 4b489269..564d0123 100644 --- a/template/tests/collector/test_collector.py +++ b/template/tests/collector/test_collector.py @@ -12,7 +12,10 @@ @patch.object(module, "BasicCollectorEngine", spec_set=module.BasicCollectorEngine) @patch.object(module, "ConfigLoader") class TestBaseCollector(unittest.TestCase): - def test_minimal_init(self, m_configloader, m_basiccollectorengine): + @patch.object(module, "SourceHandler", spec_set=module.SourceHandler) + def test_minimal_init( + self, m_source_handler, m_configloader, m_basiccollectorengine + ): """""" m_configloader.return_value.to_daemon_config.return_value = daemon_config_data name = "my collector" @@ -26,6 +29,7 @@ def test_minimal_init(self, m_configloader, m_basiccollectorengine): self.assertIsNotNone(collector.engine) self.assertIsNotNone(collector.api) m_configloader.return_value.to_daemon_config.assert_called_once() + m_source_handler.assert_called_with(collector.config.custom) m_basiccollectorengine.assert_called_with( name=name, collector_id=collector.get_id(), @@ -34,20 +38,27 @@ def test_minimal_init(self, m_configloader, m_basiccollectorengine): oaev_api=collector.api, ) - def test_init_with_custom_handler(self, m_configloader, m_basiccollectorengine): + def test_init_with_alternative_handler( + self, m_configloader, m_basiccollectorengine + ): """""" + + class NoProcessHandler(module.SourceHandler): + def match_signature_groups_and_oaevdata(self, **kwargs): + return False + m_configloader.return_value.to_daemon_config.return_value = daemon_config_data name = "my collector" source = MagicMock(spec_set=module.Source) - source_handler = MagicMock(spec_set=module.SourceHandler) + source_handler_model = NoProcessHandler collector = module.BaseCollector( - name=name, source=source, source_handler=source_handler + name=name, source=source, source_handler_model=source_handler_model ) self.assertEqual(collector.name, name) self.assertEqual(collector.source, source) - self.assertEqual(source_handler, collector.source_handler) + self.assertIsInstance(collector.source_handler, source_handler_model) self.assertIsNotNone(collector.engine) self.assertIsNotNone(collector.api) m_configloader.return_value.to_daemon_config.assert_called_once() @@ -64,11 +75,11 @@ def test_init_with_wrong_handler(self, m_configloader, m_basiccollectorengine): m_configloader.return_value.to_daemon_config.return_value = daemon_config_data name = "my collector" source = MagicMock(spec_set=module.Source) - source_handler = MagicMock() + source_handler_model = MagicMock() with self.assertRaises(module.CollectorConfigError): module.BaseCollector( - name=name, source=source, source_handler=source_handler + name=name, source=source, source_handler_model=source_handler_model ) def test_init_with_alternative_engine(self, m_configloader, m_basiccollectorengine): @@ -118,15 +129,15 @@ def test_setup( m_configloader.return_value.to_daemon_config.return_value = daemon_config_data name = "my collector" source = MagicMock(spec_set=module.Source) - source_handler = MagicMock(spec_set=module.SourceHandler) batching = True collector = module.BaseCollector( - name=name, source=source, source_handler=source_handler + name=name, + source=source, ) collector._setup(batching=batching) m_collectordaemon_setup.assert_called_once() m_basiccollectorengine.return_value.configure_engine.assert_called_with( - collector.config.template, batching=batching + collector.config.custom, batching=batching ) From 781e4ed8a37cc64aeab33639ce5a8f247517e0fb Mon Sep 17 00:00:00 2001 From: guzmud Date: Mon, 18 May 2026 11:47:59 +0200 Subject: [PATCH 24/25] [template] fix(config): propagating the renaming from template to custom --- template/docker-compose.yml | 2 +- template/src/config.yml.sample | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/template/docker-compose.yml b/template/docker-compose.yml index 1d99b138..0897e127 100644 --- a/template/docker-compose.yml +++ b/template/docker-compose.yml @@ -6,5 +6,5 @@ services: - OPENAEV_URL=http://localhost - OPENAEV_TOKEN=ChangeMe - COLLECTOR_ID=ChangeMe - - TEMPLATE_KEY=ChangeMe + - CUSTOM_KEY=ChangeMe restart: always diff --git a/template/src/config.yml.sample b/template/src/config.yml.sample index c9a0b9bd..731694ff 100644 --- a/template/src/config.yml.sample +++ b/template/src/config.yml.sample @@ -5,5 +5,5 @@ openaev: collector: id: "ChangeMe" -template: +custom: key: "value" From 3bf40f2cc6cd1ad2890c15e89c61de8df203b2dc Mon Sep 17 00:00:00 2001 From: guzmud Date: Mon, 18 May 2026 11:48:36 +0200 Subject: [PATCH 25/25] [template] chore(README): updating the README to match the new architecture --- template/README.md | 81 +++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/template/README.md b/template/README.md index 6ce84d06..ea87dc45 100644 --- a/template/README.md +++ b/template/README.md @@ -1,27 +1,43 @@ # OpenAEV Template Collector -A template for OpenAEV collector built from late-2025/early-2026 collectors (e.g. SentinelOne). Provides a modular approach to collector development based on a service-provider architecture with `Protocol` based interfaces for advanced customisation. +A template for OpenAEV collectors based on a split between a generic base collector (under `src/collector`) and a custom source (under `src/source`). The premade elements provided by the generic base collector should be enough for basic collector development. Yet, thanks to various models, protocols and alternative implementations, the generic elements can be customized, expanded and/or overwritten for more complex collectors. ## Overview -This collector is not meant to be used directly in OpenAEV but as a first support for collector development. Please update this README.md with the relevant elements to describe your collector. +This collector is not meant to be used directly in OpenAEV: it's a starting point for collector development. Please, remember to update this README.md with the relevant elements to describe your collector. -The codebase to adapt to your specific needs can be found under `src/services/`, by replacing reference to the abstract TemplateData with your specific objects, updating the DataFetcher and the various services according to this custom object and your specific needs (keywords such as data and template can be used to help parsing the generic code that should be customized). +For a basic collector, the work required should be limited to : +1. rewriting files under `src/source` to fit your needs +2. replacing `src/template_collector.py` with your own, feeding your source into a generic collector +3. updating the import made in `src/__main__.py` (according to your file's name from point number 2) -Once `src/services/` updated, the imports must be updated in `src/collector/collector.py`. Finally, new configuration parameters for your collector should be integrated under the `src/models/configs/` folder, replacing the `template_configs.py` file with yours. +Under `src/source` the minimal expectations are the following: +- a data fetcher object following the `DataFetcherProtocol` (e.g. `src/source/template_data_fetcher.py`) +- a source data object following the `SourceDataProtocol` (e.g. `src/source/template_source_data.py`) +- a list of supported signature types (e.g. `src/source/template_signatures.py`) +A source (from `src/collector/models/source.py`) will be built from those three elements in the `src/templateçcollector.py` (from point number 2 in the list earlier). Protocols can be found under `src/collector/protocols/` for more details. + +As of now, outside of `src/collector` (generic) and `src/source` (custom) there is still a mixed codebase of generic and custom elements under `src/models/settings`. In rder to forward custom parameters to your source elements, the `src/models/custom_configs.py` is available. Note that elements added to `custom_configs.py` must be reflected in `config_loader.py` too. + +Your custom configuration will be propagated through the source handler to the `__init__.py` of you data fetcher object as a `custom_config` parameter. From there, it can be used at your convenience. + +*Nota bene*: for now, please keep the `CUSTOM_EXPECTATION_BATCH_SIZE` available in the custom parameters. Do not hesitate to check the `CONTRIBUTING.md` for more details regarding the collector design and help regarding development setup. ## Features -- **Batch Processing**: Processes expectations in configurable batches for improved performance +- **Clean Split**: Clear distinction between the generic collector and the custom source +- **Highly Customizable**: Alternative engines, source handler injection, base models and protocols for source +- **Opt-in Batch Processing**: Processes expectations in configurable batches for improved performance - **Trace Generation**: Creates detailed traces with links back if available +- **Resilient Uploader**: Provides a resilient uploader for results and traces upload into OpenAEV - **Flexible Configuration**: Support for YAML, environment variables, and multiple deployment scenarios ## Requirements - OpenAEV Platform -- Python 3.12+ (for manual deployment) +- Python 3.11+ ## Configuration @@ -63,9 +79,9 @@ Below are the parameters you'll need to set for the collector: | Parameter | config.yml | Docker environment variable | Default | Mandatory | Description | |--------------------------|--------------------------------------|----------------------------------------|-----------------------------|-----------|----------------------------------------------------------------------------------------------------| -| Base URL | template.key | `TEMPLATE_KEY` |value| No | Template example key value | -| Time Window | template.time_window | `TEMPLATE_TIME_WINDOW` | PT1H | No | Default search time window when no date signatures are provided (ISO 8601 format) | -| Expectation Batch Size | template.expectation_batch_size | `TEMPLATE_EXPECTATION_BATCH_SIZE` | 50 | No | Number of expectations to process in each batch for batch-based processing | +| Key | custom.key | `CUSTOM_KEY` |value | No | Template example key value | +| Time Window | custom.time_window | `CUSTOM_TIME_WINDOW` | PT1H | No | Default search time window when no date signatures are provided (ISO 8601 format) | +| Expectation Batch Size | custom.expectation_batch_size | `CUSTOM_EXPECTATION_BATCH_SIZE` | 50 | No | Number of expectations to process in each batch for batch-based processing | ### Example Configuration Files @@ -81,7 +97,7 @@ collector: period: "PT10M" log_level: "info" -template: +custom: key: "your-value" time_window: "PT1H" expectation_batch_size: 50 @@ -92,7 +108,7 @@ template: export OPENAEV_URL="https://your-openaev-instance.com" export OPENAEV_TOKEN="your-openaev-token" export COLLECTOR_ID="template--your-unique-uuid" -export TEMPLATE_KEY="value" +export CUSTOM_KEY="value" ``` ## Deployment @@ -130,7 +146,7 @@ docker run -d \ -e OPENAEV_URL="https://your-openaev-instance.com" \ -e OPENAEV_TOKEN="your-token" \ -e COLLECTOR_ID="template--your-uuid" \ - -e TEMPLATE_KEY="your-value" \ + -e CUSTOM_KEY="your-value" \ openaev-template-collector # Or run with configuration file mounted @@ -146,23 +162,23 @@ docker run -d \ The collector supports the following OpenAEV signature types: - **change_me**: detail of the supported signature -### Processing Flow +### Link between collector, engine and source handler -1. **Expectation Retrieval**: Fetches pending expectations from OpenAEV -2. **Batch Creation**: Groups expectations into configurable batches for processing -3. **Time Window Determination**: Extracts time windows from expectations or uses default configuration -4. **Data Fetching**: Fetch data for the determined time window -6. **Expectation Matching**: Matches data against expectation criteria using detection helper -7. **Result Reporting**: Updates expectation status in OpenAEV -8. **Trace Creation**: Creates detailed traces +1. **Start from the daemon**: The `BaseCollector` is the foundation, inheriting from `CollectorDaemon` +2. **Provide an engine to the collector**: A `CollectorEngine` is attached to the `BaseCollector` (by default the `BasicCollectorEngine`) +3. **Provide a source handler to the engine**: Through the `BaseCollector` a `SourceHandler` is provided to the `CollectorEngine` +4. **Setup and start the engine**: The `BaseCollector` will setup and start the `CollectorEngine` +5. **Use the source handler through the engine**: While processing, the `CollectorEngine` will rely on the `SourceHandler` to operate elements from the `Source` -### Batch Processing - -The collector implements efficient batch processing to handle large volumes of expectations: +### Processing Flow -1. **Configurable Batch Size**: Processes expectations in batches based on `expectation_batch_size` configuration -2. **Time Window Optimization**: Extracts and consolidates time windows across batch expectations -3. **Bulk Data Fetching**: Fetches data for the entire time window rather than individual queries +1. **Fetch and filter the expectations**: Fetches pending expectations from OpenAEV and filter them according to handled expectation types +2. **If enabled, split expectations in batches**: Groups expectations into configurable batches for processing +3. **Fetch data**: Fetch data using the source handler to call the data fetcher +4. **Match data and signature types**: Match fetched data with supported signature types +5. **Match data and expectations**: Match fetched data with expectation using the OAEV detection helper +6. **Create and upload results**: Update expectation status in OpenAEV +7. **Create and upload traces**: Creates detailed traces OpenAEV-side ## Troubleshooting @@ -201,7 +217,7 @@ collector: #### For High-Volume Environments - Reduce `collector.period` for more frequent processing -- Increase `template.expectation_batch_size` for better throughput +- Increase `custom.expectation_batch_size` for better throughput #### For Low-Latency Requirements - Use shorter time windows in expectations for faster queries @@ -209,12 +225,11 @@ collector: ## Architecture -The collector uses a modular, service-provider architecture: - -- **Collector Core**: Main daemon handling scheduling and coordination -- **Expectation Service**: Batch processing and data correlation logic -- **Data Fetcher**: Dedicated service for fetching data -- **Trace Service**: Trace creation and submission +The collector is based on a split between a generic base collector, seen as a data processing unit, and a custom source of data. The main architectural elements are the following: +- **BaseCollector**: Main daemon handling scheduling and engine management +- **CollectorEngine**: Generic expectation processing engine dispatching and matching the various relevant data +- **Source**: Container made of a data fetcher, a source data format and the associated signatures +- **SourceHandler**: Wrapper provided to the collector engine to interact with the custom service - **Configuration System**: Hierarchical configuration management This architecture allows for easy extension and customization while maintaining clean separation of concerns.