feat(netwitness): add NetWitness collector (#428)

.circleci/config.yml

@@ -227,6 +227,19 @@ jobs:
             fi
             docker save -o ~/openaev/images/collector-splunk-es openaev/collector-splunk-es:${CIRCLE_SHA1}
             docker save -o ~/openaev/images/collector-splunk-es-ubi9 openaev/collector-splunk-es:${CIRCLE_SHA1}-ubi9
+      - run:
+          working_directory: ~/openaev/netwitness
+          name: Build Docker image openaev/collector-netwitness
+          command: |
+            if [[ "${CIRCLE_BRANCH}" == "main" ]]; then
+              docker build --pull --progress=plain -t openaev/collector-netwitness:${CIRCLE_SHA1} --build-arg PYOAEV_GIT_BRANCH_OVERRIDE="${CIRCLE_BRANCH}" .
+              docker build --pull --progress=plain -t openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9 -f Dockerfile_ubi9 --build-arg PYOAEV_GIT_BRANCH_OVERRIDE="${CIRCLE_BRANCH}" .
+            else 
+              docker build --pull --progress=plain -t openaev/collector-netwitness:${CIRCLE_SHA1} .
+              docker build --pull --progress=plain -t openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9 -f Dockerfile_ubi9 .
+            fi
+            docker save -o ~/openaev/images/collector-netwitness openaev/collector-netwitness:${CIRCLE_SHA1}
+            docker save -o ~/openaev/images/collector-netwitness-ubi9 openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9
       - run:
           working_directory: ~/openaev/openaev
           name: Build Docker image openaev/collector-openaev
@@ -414,6 +427,13 @@ jobs:
             docker tag openaev/collector-splunk-es:${CIRCLE_SHA1}-ubi9 openaev/collector-splunk-es:${IMAGETAG}-ubi9
             docker tag openaev/collector-splunk-es:${CIRCLE_SHA1}-ubi9 openbas/collector-splunk-es:${IMAGETAG}-ubi9
 
+            docker image load < collector-netwitness
+            docker tag openaev/collector-netwitness:${CIRCLE_SHA1} openaev/collector-netwitness:${IMAGETAG}
+            docker tag openaev/collector-netwitness:${CIRCLE_SHA1} openbas/collector-netwitness:${IMAGETAG}
+            docker image load < collector-netwitness-ubi9
+            docker tag openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9 openaev/collector-netwitness:${IMAGETAG}-ubi9
+            docker tag openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9 openbas/collector-netwitness:${IMAGETAG}-ubi9
+
             docker image load < collector-openaev
             docker tag openaev/collector-openaev:${CIRCLE_SHA1} openaev/collector-openaev:${IMAGETAG}
             docker tag openaev/collector-openaev:${CIRCLE_SHA1} openbas/collector-openbas:${IMAGETAG}
@@ -497,6 +517,10 @@ jobs:
             docker push openaev/collector-splunk-es:${IMAGETAG}-ubi9
             docker push openbas/collector-splunk-es:${IMAGETAG}
             docker push openbas/collector-splunk-es:${IMAGETAG}-ubi9
+            docker push openaev/collector-netwitness:${IMAGETAG}
+            docker push openaev/collector-netwitness:${IMAGETAG}-ubi9
+            docker push openbas/collector-netwitness:${IMAGETAG}
+            docker push openbas/collector-netwitness:${IMAGETAG}-ubi9
             docker push openaev/collector-openaev:${IMAGETAG}
             docker push openaev/collector-openaev:${IMAGETAG}-ubi9
             docker push openbas/collector-openbas:${IMAGETAG}
@@ -544,6 +568,8 @@ jobs:
               docker tag openaev/collector-sentinelone:${IMAGETAG} openbas/collector-sentinelone:latest
               docker tag openaev/collector-splunk-es:${IMAGETAG} openaev/collector-splunk-es:latest
               docker tag openaev/collector-splunk-es:${IMAGETAG} openbas/collector-splunk-es:latest
+              docker tag openaev/collector-netwitness:${IMAGETAG} openaev/collector-netwitness:latest
+              docker tag openaev/collector-netwitness:${IMAGETAG} openbas/collector-netwitness:latest
               docker tag openaev/collector-openaev:${IMAGETAG} openaev/collector-openaev:latest
               docker tag openaev/collector-openaev:${IMAGETAG} openbas/collector-openbas:latest
               docker tag openaev/collector-microsoft-azure:${IMAGETAG} openaev/collector-microsoft-azure:latest
@@ -577,6 +603,8 @@ jobs:
               docker push openbas/collector-sentinelone:latest
               docker push openaev/collector-splunk-es:latest
               docker push openbas/collector-splunk-es:latest
+              docker push openaev/collector-netwitness:latest
+              docker push openbas/collector-netwitness:latest
               docker push openaev/collector-openaev:latest
               docker push openbas/collector-openbas:latest
               docker push openaev/collector-microsoft-azure:latest

netwitness/.build.env

@@ -0,0 +1,2 @@
+COLLECTOR_CMD=src
+

netwitness/.dockerignore

@@ -0,0 +1,11 @@
+# Configuration files
+config.yml
+
+# Build artifacts
+dist
+
+# Cache directories
+__pycache__
+.ruff_cache
+.mypy_cache
+.pytest_cache

netwitness/.gitignore

@@ -0,0 +1,10 @@
+config.yml
+
+# Build artifacts
+dist
+
+# Cache directories
+__pycache__
+.ruff_cache
+.mypy_cache
+.pytest_cache

netwitness/Dockerfile

@@ -0,0 +1,33 @@
+
+FROM python:3.13-alpine AS builder
+
+# poetry version available on Ubuntu 24.04
+RUN pip3 install poetry==2.1.3
+
+RUN apk update && apk upgrade
+
+ARG installdir=/collector
+ADD . ${installdir}
+RUN cd ${installdir} && poetry build
+
+FROM python:3.13-alpine AS runner
+
+# Declare the build argument
+ARG PYOAEV_GIT_BRANCH_OVERRIDE
+
+ARG installdir=/collector
+COPY --from=builder ${installdir} ${installdir}
+RUN cd ${installdir}/dist && pip3 install --no-cache-dir "$(ls *.whl)[prod]"
+
+RUN if [[ ${PYOAEV_GIT_BRANCH_OVERRIDE} ]] ; then \
+    echo "Forcing specific version of client-python" && \
+    apk add --no-cache git && \
+    pip install pip3-autoremove && \
+    pip-autoremove pyoaev -y && \
+    pip install git+https://github.com/OpenAEV-Platform/client-python@${PYOAEV_GIT_BRANCH_OVERRIDE} ; \
+    fi
+
+# necessary for icon location
+WORKDIR ${installdir}
+
+CMD ["NetWitnessCollector"]

netwitness/Dockerfile_ubi9

@@ -0,0 +1,43 @@
+FROM registry.access.redhat.com/ubi9/ubi-minimal AS base
+
+RUN set -eux; \
+    microdnf -y --setopt=install_weak_deps=0 install python3.12; \
+    microdnf clean all;
+
+
+FROM base AS builder
+
+RUN set -eux; \
+    microdnf -y --setopt=install_weak_deps=0 install python3.12-pip; \
+    pip3.12 install poetry==2.1.3; \
+    microdnf -y remove python3.12-pip; \
+    microdnf clean all;
+
+WORKDIR /collector
+COPY ./ ./
+RUN set -eux; \
+    poetry build
+
+
+FROM base AS runner
+
+ARG PYOAEV_GIT_BRANCH_OVERRIDE=""
+
+WORKDIR /collector
+COPY --from=builder /collector/ ./
+
+RUN set -eux; \
+    microdnf -y --setopt=install_weak_deps=0 install python3.12-pip; \
+    (cd dist && pip3.12 install --no-cache-dir "$(ls *.whl)[prod]"); \
+    if [ -n "${PYOAEV_GIT_BRANCH_OVERRIDE}" ] ; then \
+        echo "Forcing specific version of client-python"; \
+        microdnf -y --setopt=install_weak_deps=0 install git-core; \
+        pip3.12 install pip3-autoremove; \
+        pip-autoremove pyoaev -y; \
+        pip3.12 install git+https://github.com/OpenAEV-Platform/client-python@${PYOAEV_GIT_BRANCH_OVERRIDE}; \
+        microdnf -y remove git-core; \
+    fi; \
+    microdnf -y remove python3.12-pip; \
+    microdnf clean all;
+
+CMD ["NetWitnessCollector"]

netwitness/README.md

@@ -0,0 +1,135 @@
+# OpenAEV NetWitness Collector
+
+A NetWitness integration for OpenAEV that validates detection expectations by querying the NetWitness Core SDK and matching the returned sessions against expected outcomes.
+
+**Note**: Requires network access to a NetWitness Core service (Broker or Concentrator) with the RESTful API enabled.
+
+## Overview
+
+This collector validates OpenAEV expectations by querying your NetWitness environment for matching sessions via the Core SDK query API (NWQL). When OpenAEV runs security exercises, this collector automatically checks whether the expected activity was observed by NetWitness, providing visibility into your detection capabilities.
+
+The collector builds an NWQL query from the attack signatures, executes it against the Core SDK, and parses the returned session metadata, with support for IP-based matching and parent process tracking through the URL meta.
+
+## Features
+
+- **Detection Validation**: Runs NetWitness Core SDK queries to verify detections
+- **IP-based Matching**: Supports both source and destination IPv4 / IPv6 address matching (`ip.src` / `ip.dst`)
+- **Parent Process Tracking**: Extracts inject/agent identifiers from parent process names and matches them against the `url` meta
+- **Flexible Authentication**: HTTP basic authentication (Core SDK) or a bearer token (NetWitness Platform API)
+- **Retry Mechanism**: Built-in retry logic with a configurable offset to handle ingestion latency
+- **Trace Generation**: Creates traces with links back to NetWitness Investigate
+- **Flexible Configuration**: Support for YAML, environment variables, and multiple deployment scenarios
+
+## Required permissions
+
+The NetWitness collector requires credentials (a Core service user or token) with:
+- Permission to run queries against the Core SDK (`/sdk?msg=query`)
+
+See the NetWitness documentation on the [Core RESTful API](https://community.netwitness.com/s/article/SDKCommands).
+
+## Requirements
+
+- OpenAEV Platform
+- A NetWitness Core service (Broker on port 50103 or Concentrator on port 50105) with the RESTful API reachable
+- Python 3.11+ (for manual deployment)
+- A user account or token with permission to query the Core SDK
+
+## Configuration
+
+There are a number of configuration options, which are set either in `docker-compose.yml` (for Docker) or in `config.yml` (for manual deployment).
+
+The collector supports multiple configuration sources in order of precedence:
+1. Environment variables
+2. YAML configuration file (`src/config.yml`)
+3. Default values
+
+### OpenAEV environment variables
+
+Below are the parameters you'll need to set for OpenAEV:
+
+| Parameter         | config.yml        | Docker environment variable | Mandatory | Description                                           |
+|-------------------|-------------------|-----------------------------|-----------|-------------------------------------------------------|
+| OpenAEV URL       | openaev.url       | `OPENAEV_URL`               | Yes       | The URL of the OpenAEV platform.                      |
+| OpenAEV Token     | openaev.token     | `OPENAEV_TOKEN`             | Yes       | The default admin token set in the OpenAEV platform.  |
+| OpenAEV Tenant ID | openaev.tenant_id | `OPENAEV_TENANT_ID`         | No        | Identifier of the tenant within the OpenAEV platform. |
+
+> Warning
+>
+> The `tenant_id` parameter is a new configuration option. A period of backward compatibility is ensured: if this key is not defined,
+> existing configurations will not be affected, and the default value will be `None`. However, if a value is provided, it will be
+> validated by Pydantic and must conform to a valid UUID format, otherwise a validation error will be returned.
+
+### Base collector environment variables
+
+Below are the parameters you'll need to set for running the collector properly:
+
+| Parameter        | config.yml          | Docker environment variable | Default                                         | Mandatory | Description                                                                                   |
+|------------------|---------------------|-----------------------------|-------------------------------------------------|-----------|-----------------------------------------------------------------------------------------------|
+| Collector ID     | collector.id        | `COLLECTOR_ID`              | netwitness--0b13e3f7-5c9e-46f5-acc4-33032e9b... | Yes       | A unique `UUIDv4` identifier for this collector instance.                                     |
+| Collector Name   | collector.name      | `COLLECTOR_NAME`            | NetWitness                                      | No        | Name of the collector.                                                                        |
+| Collector Period | collector.period    | `COLLECTOR_PERIOD`          | PT1M                                            | No        | Collection interval (ISO 8601 format).                                                        |
+| Log Level        | collector.log_level | `COLLECTOR_LOG_LEVEL`       | error                                           | No        | Determines the verbosity of the logs. Options are `debug`, `info`, `warn`, or `error`.        |
+| Platform         | collector.platform  | `COLLECTOR_PLATFORM`        | SIEM                                            | No        | Type of security platform this collector works for. One of: `EDR, XDR, SIEM, SOAR, NDR, ISPM` |
+
+### Collector extra parameters environment variables
+
+Below are the parameters you'll need to set for the collector:
+
+| Parameter    | config.yml              | Docker environment variable | Default                              | Mandatory | Description                                                                              |
+|--------------|-------------------------|-----------------------------|--------------------------------------|-----------|-----------------------------------------------------------------------------------------|
+| Base URL     | netwitness.base_url     | `NETWITNESS_BASE_URL`       | https://netwitness.company.com:50103 | Yes       | Base URL of a NetWitness Core service (Broker/Concentrator).                             |
+| Username     | netwitness.username     | `NETWITNESS_USERNAME`       |                                      | No*       | Username for HTTP basic authentication to the Core SDK.                                  |
+| Password     | netwitness.password     | `NETWITNESS_PASSWORD`       |                                      | No*       | Password for HTTP basic authentication.                                                  |
+| Token        | netwitness.token        | `NETWITNESS_TOKEN`          |                                      | No*       | Bearer token for the NetWitness Platform API (optional).                                 |
+| Max Results  | netwitness.max_results  | `NETWITNESS_MAX_RESULTS`    | 100                                  | No        | Maximum number of sessions to return per query.                                         |
+| Console URL  | netwitness.console_url  | `NETWITNESS_CONSOLE_URL`    |                                      | No        | NetWitness console URL used to build trace links (defaults to base_url).                 |
+| Verify SSL   | netwitness.verify_ssl   | `NETWITNESS_VERIFY_SSL`     | true                                 | No        | Whether to verify the NetWitness TLS certificate.                                       |
+| Time Window  | netwitness.time_window  | `NETWITNESS_TIME_WINDOW`    | PT1H                                 | No        | Default search window when no date signatures are provided (ISO 8601 format).            |
+| Offset       | netwitness.offset       | `NETWITNESS_OFFSET`         | PT30S                                | No        | Delay between retry attempts to account for ingestion latency (ISO 8601 format).         |
+| Max Retry    | netwitness.max_retry    | `NETWITNESS_MAX_RETRY`      | 3                                    | No        | Maximum number of retry attempts after the initial query fails or returns no results.    |
+
+> \* Authentication is required: provide either `NETWITNESS_TOKEN` or both `NETWITNESS_USERNAME` and `NETWITNESS_PASSWORD`.
+
+### Example Configuration Files
+
+#### YAML Configuration (`src/config.yml`)
+```yaml
+openaev:
+  url: "https://your-openaev-instance.com"
+  token: "your-openaev-token"
+# tenant_id: "your-openaev-tenant-id"
+collector:
+  id: "netwitness--your-unique-uuid"
+  name: "NetWitness Production"
+  period: "PT10M"
+  log_level: "info"
+
+netwitness:
+  base_url: "https://your-netwitness.company.com:50103"
+  username: "api"
+  password: "your-password"
+  max_results: 100
+  verify_ssl: true
+  offset: "PT45S"
+  max_retry: 5
+```
+
+#### Environment Variables
+```bash
+export OPENAEV_URL="https://your-openaev-instance.com"
+export OPENAEV_TOKEN="your-openaev-token"
+export OPENAEV_TENANT_ID="your-openaev-tenant-id"
+export COLLECTOR_ID="netwitness--your-unique-uuid"
+export NETWITNESS_BASE_URL="https://your-netwitness.company.com:50103"
+export NETWITNESS_USERNAME="api"
+export NETWITNESS_PASSWORD="your-password"
+```
+
+## API endpoints used
+
+- **Authentication**: HTTP basic (Core SDK) or bearer token
+- **Query**: `GET /sdk?msg=query&query=<NWQL>&force-content-type=application/json`
+- **NWQL meta used for matching**: `ip.src`, `ip.dst`, `url`, `time`
+- **Reference**: [NetWitness Core SDK commands](https://community.netwitness.com/s/article/SDKCommands)
+
+> **Note**: The required permissions and endpoints listed above are based on the current code and documentation. NetWitness may change API requirements at any time. Always check the official NetWitness documentation for the latest requirements before deploying.

netwitness/docker-compose.yml

@@ -0,0 +1,16 @@
+version: "3"
+services:
+  collector-netwitness:
+    image: openaev/collector-netwitness:rolling
+    environment:
+      - OPENAEV_URL=http://localhost
+      - OPENAEV_TOKEN=ChangeMe
+      - OPENAEV_TENANT_ID=ChangeMe
+      - COLLECTOR_ID=ChangeMe
+      - NETWITNESS_BASE_URL=https://change.me:50103
+      # Authentication: provide username/password (Core SDK) or NETWITNESS_TOKEN (bearer)
+      - NETWITNESS_USERNAME=ChangeMe
+      - NETWITNESS_PASSWORD=ChangeMe
+      # - NETWITNESS_TOKEN=ChangeMe
+      - NETWITNESS_MAX_RESULTS=100
+    restart: always

netwitness/manifest-metadata.json

@@ -0,0 +1,18 @@
+{
+  "title": "NetWitness",
+  "slug": "openaev_netwitness",
+  "description": "Collect responses from NetWitness",
+  "short_description": "Collect responses from NetWitness",
+  "use_cases": ["Security response"],
+  "verified": false,
+  "last_verified_date": "",
+  "playbook_supported": false,
+  "max_confidence_level": 80,
+  "support_version": "",
+  "subscription_link": "https://www.netwitness.com/",
+  "source_code": "",
+  "manager_supported": true,
+  "container_version": "rolling",
+  "container_image": "openaev/collector-netwitness",
+  "container_type": "COLLECTOR"
+}