Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,19 @@ jobs:
fi
docker save -o ~/openaev/images/collector-splunk-es openaev/collector-splunk-es:${CIRCLE_SHA1}
docker save -o ~/openaev/images/collector-splunk-es-ubi9 openaev/collector-splunk-es:${CIRCLE_SHA1}-ubi9
- run:
working_directory: ~/openaev/netwitness
name: Build Docker image openaev/collector-netwitness
command: |
if [[ "${CIRCLE_BRANCH}" == "main" ]]; then
docker build --pull --progress=plain -t openaev/collector-netwitness:${CIRCLE_SHA1} --build-arg PYOAEV_GIT_BRANCH_OVERRIDE="${CIRCLE_BRANCH}" .
docker build --pull --progress=plain -t openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9 -f Dockerfile_ubi9 --build-arg PYOAEV_GIT_BRANCH_OVERRIDE="${CIRCLE_BRANCH}" .
else
docker build --pull --progress=plain -t openaev/collector-netwitness:${CIRCLE_SHA1} .
docker build --pull --progress=plain -t openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9 -f Dockerfile_ubi9 .
fi
docker save -o ~/openaev/images/collector-netwitness openaev/collector-netwitness:${CIRCLE_SHA1}
docker save -o ~/openaev/images/collector-netwitness-ubi9 openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9
- run:
working_directory: ~/openaev/openaev
name: Build Docker image openaev/collector-openaev
Expand Down Expand Up @@ -414,6 +427,13 @@ jobs:
docker tag openaev/collector-splunk-es:${CIRCLE_SHA1}-ubi9 openaev/collector-splunk-es:${IMAGETAG}-ubi9
docker tag openaev/collector-splunk-es:${CIRCLE_SHA1}-ubi9 openbas/collector-splunk-es:${IMAGETAG}-ubi9

docker image load < collector-netwitness
docker tag openaev/collector-netwitness:${CIRCLE_SHA1} openaev/collector-netwitness:${IMAGETAG}
docker tag openaev/collector-netwitness:${CIRCLE_SHA1} openbas/collector-netwitness:${IMAGETAG}
docker image load < collector-netwitness-ubi9
docker tag openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9 openaev/collector-netwitness:${IMAGETAG}-ubi9
docker tag openaev/collector-netwitness:${CIRCLE_SHA1}-ubi9 openbas/collector-netwitness:${IMAGETAG}-ubi9

docker image load < collector-openaev
docker tag openaev/collector-openaev:${CIRCLE_SHA1} openaev/collector-openaev:${IMAGETAG}
docker tag openaev/collector-openaev:${CIRCLE_SHA1} openbas/collector-openbas:${IMAGETAG}
Expand Down Expand Up @@ -497,6 +517,10 @@ jobs:
docker push openaev/collector-splunk-es:${IMAGETAG}-ubi9
docker push openbas/collector-splunk-es:${IMAGETAG}
docker push openbas/collector-splunk-es:${IMAGETAG}-ubi9
docker push openaev/collector-netwitness:${IMAGETAG}
docker push openaev/collector-netwitness:${IMAGETAG}-ubi9
docker push openbas/collector-netwitness:${IMAGETAG}
docker push openbas/collector-netwitness:${IMAGETAG}-ubi9
docker push openaev/collector-openaev:${IMAGETAG}
docker push openaev/collector-openaev:${IMAGETAG}-ubi9
docker push openbas/collector-openbas:${IMAGETAG}
Expand Down Expand Up @@ -544,6 +568,8 @@ jobs:
docker tag openaev/collector-sentinelone:${IMAGETAG} openbas/collector-sentinelone:latest
docker tag openaev/collector-splunk-es:${IMAGETAG} openaev/collector-splunk-es:latest
docker tag openaev/collector-splunk-es:${IMAGETAG} openbas/collector-splunk-es:latest
docker tag openaev/collector-netwitness:${IMAGETAG} openaev/collector-netwitness:latest
docker tag openaev/collector-netwitness:${IMAGETAG} openbas/collector-netwitness:latest
docker tag openaev/collector-openaev:${IMAGETAG} openaev/collector-openaev:latest
docker tag openaev/collector-openaev:${IMAGETAG} openbas/collector-openbas:latest
docker tag openaev/collector-microsoft-azure:${IMAGETAG} openaev/collector-microsoft-azure:latest
Expand Down Expand Up @@ -577,6 +603,8 @@ jobs:
docker push openbas/collector-sentinelone:latest
docker push openaev/collector-splunk-es:latest
docker push openbas/collector-splunk-es:latest
docker push openaev/collector-netwitness:latest
docker push openbas/collector-netwitness:latest
docker push openaev/collector-openaev:latest
docker push openbas/collector-openbas:latest
docker push openaev/collector-microsoft-azure:latest
Expand Down
2 changes: 2 additions & 0 deletions netwitness/.build.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
COLLECTOR_CMD=src

11 changes: 11 additions & 0 deletions netwitness/.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Configuration files
config.yml

# Build artifacts
dist

# Cache directories
__pycache__
.ruff_cache
.mypy_cache
.pytest_cache
10 changes: 10 additions & 0 deletions netwitness/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
config.yml

# Build artifacts
dist

# Cache directories
__pycache__
.ruff_cache
.mypy_cache
.pytest_cache
33 changes: 33 additions & 0 deletions netwitness/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@

FROM python:3.13-alpine AS builder

# poetry version available on Ubuntu 24.04
RUN pip3 install poetry==2.1.3

RUN apk update && apk upgrade

ARG installdir=/collector
ADD . ${installdir}
RUN cd ${installdir} && poetry build

FROM python:3.13-alpine AS runner

# Declare the build argument
ARG PYOAEV_GIT_BRANCH_OVERRIDE

ARG installdir=/collector
COPY --from=builder ${installdir} ${installdir}
RUN cd ${installdir}/dist && pip3 install --no-cache-dir "$(ls *.whl)[prod]"

RUN if [[ ${PYOAEV_GIT_BRANCH_OVERRIDE} ]] ; then \
echo "Forcing specific version of client-python" && \
apk add --no-cache git && \
pip install pip3-autoremove && \
pip-autoremove pyoaev -y && \
pip install git+https://github.com/OpenAEV-Platform/client-python@${PYOAEV_GIT_BRANCH_OVERRIDE} ; \
fi

# necessary for icon location
WORKDIR ${installdir}

CMD ["NetWitnessCollector"]
43 changes: 43 additions & 0 deletions netwitness/Dockerfile_ubi9
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
FROM registry.access.redhat.com/ubi9/ubi-minimal AS base

RUN set -eux; \
microdnf -y --setopt=install_weak_deps=0 install python3.12; \
microdnf clean all;


FROM base AS builder

RUN set -eux; \
microdnf -y --setopt=install_weak_deps=0 install python3.12-pip; \
pip3.12 install poetry==2.1.3; \
microdnf -y remove python3.12-pip; \
microdnf clean all;

WORKDIR /collector
COPY ./ ./
RUN set -eux; \
poetry build


FROM base AS runner

ARG PYOAEV_GIT_BRANCH_OVERRIDE=""

WORKDIR /collector
COPY --from=builder /collector/ ./

RUN set -eux; \
microdnf -y --setopt=install_weak_deps=0 install python3.12-pip; \
(cd dist && pip3.12 install --no-cache-dir "$(ls *.whl)[prod]"); \
if [ -n "${PYOAEV_GIT_BRANCH_OVERRIDE}" ] ; then \
echo "Forcing specific version of client-python"; \
microdnf -y --setopt=install_weak_deps=0 install git-core; \
pip3.12 install pip3-autoremove; \
pip-autoremove pyoaev -y; \
pip3.12 install git+https://github.com/OpenAEV-Platform/client-python@${PYOAEV_GIT_BRANCH_OVERRIDE}; \
microdnf -y remove git-core; \
fi; \
microdnf -y remove python3.12-pip; \
microdnf clean all;

CMD ["NetWitnessCollector"]
137 changes: 137 additions & 0 deletions netwitness/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
# OpenAEV NetWitness Collector

A NetWitness integration for OpenAEV that validates detection expectations by querying the NetWitness Core SDK and matching the returned sessions against expected outcomes.

**Note**: Requires network access to a NetWitness Core service (Broker or Concentrator) with the RESTful API enabled.

## Overview

This collector validates OpenAEV expectations by querying your NetWitness environment for matching sessions via the Core SDK query API (NWQL). When OpenAEV runs security exercises, this collector automatically checks whether the expected activity was observed by NetWitness, providing visibility into your detection capabilities.

The collector builds an NWQL query from the attack signatures, executes it against the Core SDK, and parses the returned session metadata, with support for IP-based matching and parent process tracking through the URL meta.

## Features

- **Detection Validation**: Runs NetWitness Core SDK queries to verify detections
- **IP-based Matching**: Supports both source and destination IPv4 / IPv6 address matching (`ip.src` / `ip.dst`)
- **Parent Process Tracking**: Extracts inject/agent identifiers from parent process names and matches them against the `url` meta
- **Flexible Authentication**: HTTP basic authentication (Core SDK) or a bearer token (NetWitness Platform API)
- **Retry Mechanism**: Built-in retry logic with a configurable offset to handle ingestion latency
- **Trace Generation**: Creates traces with links back to NetWitness Investigate
- **Flexible Configuration**: Support for YAML, environment variables, and multiple deployment scenarios

## Required permissions

The NetWitness collector requires credentials (a Core service user or token) with:
- Permission to run queries against the Core SDK (`/sdk?msg=query`)

See the NetWitness documentation on the [Core RESTful API](https://community.netwitness.com/s/article/SDKCommands).

## Requirements

- OpenAEV Platform
- A NetWitness Core service (Broker on port 50103 or Concentrator on port 50105) with the RESTful API reachable
- Python 3.11+ (for manual deployment)
- A user account or token with permission to query the Core SDK

## Configuration

There are a number of configuration options, which are set either in `docker-compose.yml` (for Docker) or in `config.yml` (for manual deployment).

The collector loads configuration from a single source, selected in this order (the first one found wins; sources are not merged):
1. `.env` file (`src/.env`), if present
2. YAML configuration file (`src/config.yml`), if present
3. Environment variables

Any value not provided by the selected source falls back to its default.

### OpenAEV environment variables

Below are the parameters you'll need to set for OpenAEV:

| Parameter | config.yml | Docker environment variable | Mandatory | Description |
|-------------------|-------------------|-----------------------------|-----------|-------------------------------------------------------|
| OpenAEV URL | openaev.url | `OPENAEV_URL` | Yes | The URL of the OpenAEV platform. |
| OpenAEV Token | openaev.token | `OPENAEV_TOKEN` | Yes | The default admin token set in the OpenAEV platform. |
| OpenAEV Tenant ID | openaev.tenant_id | `OPENAEV_TENANT_ID` | No | Identifier of the tenant within the OpenAEV platform. |

> Warning
>
> The `tenant_id` parameter is a new configuration option. A period of backward compatibility is ensured: if this key is not defined,
> existing configurations will not be affected, and the default value will be `None`. However, if a value is provided, it will be
> validated by Pydantic and must conform to a valid UUID format, otherwise a validation error will be returned.

### Base collector environment variables

Below are the parameters you'll need to set for running the collector properly:

| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description |
|------------------|---------------------|-----------------------------|-------------------------------------------------|-----------|-----------------------------------------------------------------------------------------------|
| Collector ID | collector.id | `COLLECTOR_ID` | netwitness--0b13e3f7-5c9e-46f5-acc4-33032e9b... | Yes | A unique `UUIDv4` identifier for this collector instance. |
| Collector Name | collector.name | `COLLECTOR_NAME` | NetWitness | No | Name of the collector. |
| Collector Period | collector.period | `COLLECTOR_PERIOD` | PT1M | No | Collection interval (ISO 8601 format). |
| Log Level | collector.log_level | `COLLECTOR_LOG_LEVEL` | error | No | Determines the verbosity of the logs. Options are `debug`, `info`, `warn`, or `error`. |
| Platform | collector.platform | `COLLECTOR_PLATFORM` | SIEM | No | Type of security platform this collector works for. One of: `EDR, XDR, SIEM, SOAR, NDR, ISPM` |

### Collector extra parameters environment variables

Below are the parameters you'll need to set for the collector:

| Parameter | config.yml | Docker environment variable | Default | Mandatory | Description |
|--------------|-------------------------|-----------------------------|--------------------------------------|-----------|-----------------------------------------------------------------------------------------|
| Base URL | netwitness.base_url | `NETWITNESS_BASE_URL` | https://netwitness.company.com:50103 | Yes | Base URL of a NetWitness Core service (Broker/Concentrator). |
| Username | netwitness.username | `NETWITNESS_USERNAME` | | No* | Username for HTTP basic authentication to the Core SDK. |
| Password | netwitness.password | `NETWITNESS_PASSWORD` | | No* | Password for HTTP basic authentication. |
| Token | netwitness.token | `NETWITNESS_TOKEN` | | No* | Bearer token for the NetWitness Platform API (optional). |
| Max Results | netwitness.max_results | `NETWITNESS_MAX_RESULTS` | 100 | No | Maximum number of sessions to return per query. |
| Console URL | netwitness.console_url | `NETWITNESS_CONSOLE_URL` | | No | NetWitness console URL used to build trace links (defaults to base_url). |
| Verify SSL | netwitness.verify_ssl | `NETWITNESS_VERIFY_SSL` | true | No | Whether to verify the NetWitness TLS certificate. |
| Time Window | netwitness.time_window | `NETWITNESS_TIME_WINDOW` | PT1H | No | Default search window when no date signatures are provided (ISO 8601 format). |
| Offset | netwitness.offset | `NETWITNESS_OFFSET` | PT30S | No | Delay between retry attempts to account for ingestion latency (ISO 8601 format). |
| Max Retry | netwitness.max_retry | `NETWITNESS_MAX_RETRY` | 3 | No | Maximum number of retry attempts after the initial query fails or returns no results. |

> \* Authentication is required: provide either `NETWITNESS_TOKEN` or both `NETWITNESS_USERNAME` and `NETWITNESS_PASSWORD`.

### Example Configuration Files

#### YAML Configuration (`src/config.yml`)
```yaml
openaev:
url: "https://your-openaev-instance.com"
token: "your-openaev-token"
# tenant_id: "your-openaev-tenant-id"
collector:
id: "netwitness--your-unique-uuid"
name: "NetWitness Production"
period: "PT10M"
log_level: "info"

netwitness:
base_url: "https://your-netwitness.company.com:50103"
username: "api"
password: "your-password"
max_results: 100
verify_ssl: true
offset: "PT45S"
max_retry: 5
```

#### Environment Variables
```bash
export OPENAEV_URL="https://your-openaev-instance.com"
export OPENAEV_TOKEN="your-openaev-token"
export OPENAEV_TENANT_ID="your-openaev-tenant-id"
export COLLECTOR_ID="netwitness--your-unique-uuid"
export NETWITNESS_BASE_URL="https://your-netwitness.company.com:50103"
export NETWITNESS_USERNAME="api"
export NETWITNESS_PASSWORD="your-password"
```

## API endpoints used

- **Authentication**: HTTP basic (Core SDK) or bearer token
- **Query**: `GET /sdk?msg=query&query=<NWQL>&force-content-type=application/json`
- **NWQL meta used for matching**: `ip.src`, `ip.dst`, `url`, `time`
- **Reference**: [NetWitness Core SDK commands](https://community.netwitness.com/s/article/SDKCommands)

> **Note**: The required permissions and endpoints listed above are based on the current code and documentation. NetWitness may change API requirements at any time. Always check the official NetWitness documentation for the latest requirements before deploying.
16 changes: 16 additions & 0 deletions netwitness/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
version: "3"
services:
collector-netwitness:
image: openaev/collector-netwitness:rolling
environment:
- OPENAEV_URL=http://localhost
- OPENAEV_TOKEN=ChangeMe
- OPENAEV_TENANT_ID=ChangeMe
- COLLECTOR_ID=ChangeMe
- NETWITNESS_BASE_URL=https://change.me:50103
# Authentication: provide username/password (Core SDK) or NETWITNESS_TOKEN (bearer)
- NETWITNESS_USERNAME=ChangeMe
- NETWITNESS_PASSWORD=ChangeMe
# - NETWITNESS_TOKEN=ChangeMe
- NETWITNESS_MAX_RESULTS=100
restart: always
18 changes: 18 additions & 0 deletions netwitness/manifest-metadata.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"title": "NetWitness",
"slug": "openaev_netwitness",
"description": "Collect responses from NetWitness",
"short_description": "Collect responses from NetWitness",
"use_cases": ["Security response"],
"verified": false,
"last_verified_date": "",
"playbook_supported": false,
"max_confidence_level": 80,
"support_version": "",
"subscription_link": "https://www.netwitness.com/",
"source_code": "",
"manager_supported": true,
"container_version": "rolling",
"container_image": "openaev/collector-netwitness",
"container_type": "COLLECTOR"
}
Loading
Loading