diff --git a/packages/sentinel_one/changelog.yml b/packages/sentinel_one/changelog.yml index 3491d558354..90ea9a764c1 100644 --- a/packages/sentinel_one/changelog.yml +++ b/packages/sentinel_one/changelog.yml @@ -1,4 +1,15 @@ # newer versions go on top +- version: "2.11.3" + changes: + - description: Fix error handling for first phase of agent data collection. + type: bugfix + link: https://github.com/elastic/integrations/pull/20484 + - description: Fix handling of deleted threats. + type: bugfix + link: https://github.com/elastic/integrations/pull/20484 + - description: Fix global error handler message formatting. + type: bugfix + link: https://github.com/elastic/integrations/pull/20484 - version: "2.11.2" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.expected b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.expected index af6885d33e8..5fcc9b8e19f 100644 --- a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.expected +++ b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-all.expected @@ -36,14 +36,14 @@ inputs: "sortBy": ["updatedAt"], "sortOrder": ["asc"], "updatedAt__gte": [ - state.?cursor.?list_updated_at_gte.orValue( - state.?cursor.?last_timestamp.orValue( + state.?cursor.list_updated_at_gte.orValue( + state.?cursor.last_timestamp.orValue( (now - duration(state.initial_interval)).format(time_layout.RFC3339) ) ) ], ?"siteIds": state.?site_ids.optMap(v, [string(v)]), - ?"cursor": state.?cursor.?next_page.token.optMap(v, [v]), + ?"cursor": state.?cursor.next_page.token.optMap(v, [v]), }.format_query() ).with( { @@ -60,8 +60,8 @@ inputs: ?"token": (body.?pagination.nextCursor.orValue(null) != null) ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": body.?pagination.nextCursor.orValue(null) != null, - "list_updated_at_gte": state.?cursor.?list_updated_at_gte.orValue( - state.?cursor.?last_timestamp.orValue( + "list_updated_at_gte": state.?cursor.list_updated_at_gte.orValue( + state.?cursor.last_timestamp.orValue( (now - duration(state.initial_interval)).format(time_layout.RFC3339) ) ), @@ -87,7 +87,9 @@ inputs: } ) ) - ).as(state, + ).as(state, has(state.?events.error) ? + state + : state.with( !has(state.?cursor.worklist) ? state @@ -105,14 +107,20 @@ inputs: "Authorization": ["ApiToken " + state.api_token], }, } - ).do_request().as(resp, (resp.StatusCode == 200) ? - resp.Body.decode_json().as(body, + ).do_request().as(resp, (resp.StatusCode == 200 || resp.StatusCode == 404) ? + ( + (resp.StatusCode == 200) ? + resp.Body.decode_json() + : + // Do not bother getting 404 body since it's not used. + {} + ).as(body, (body.?pagination.nextCursor.orValue(null) != null).as(has_more_events, { "data": has_more_events ? state.cursor.worklist.data : tail(state.cursor.worklist.data), }.as(new_worklist, { - "events": (has(body.data) && body.data.size() > 0) ? + "events": (resp.StatusCode == 200 && has(body.data) && body.data.size() > 0) ? body.data.map(e, { "message": e.encode_json(), @@ -133,7 +141,16 @@ inputs: ?"token": has_more_events ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": state.?cursor.fetch_more.orValue(false), - "list_updated_at_gte": state.cursor.list_updated_at_gte, + ?"list_updated_at_gte": (body.?pagination.nextCursor.orValue(null) != null) ? + optional.of( + state.?cursor.list_updated_at_gte.orValue( + state.?cursor.last_timestamp.orValue( + (now - duration(state.initial_interval)).format(time_layout.RFC3339) + ) + ) + ) + : + optional.none(), ?"last_timestamp": has_more_events ? state.?cursor.last_timestamp : diff --git a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.expected b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.expected index 6dd2892f687..ef78db5f2c7 100644 --- a/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.expected +++ b/packages/sentinel_one/data_stream/threat_event/_dev/test/policy/test-default.expected @@ -25,14 +25,14 @@ inputs: "sortBy": ["updatedAt"], "sortOrder": ["asc"], "updatedAt__gte": [ - state.?cursor.?list_updated_at_gte.orValue( - state.?cursor.?last_timestamp.orValue( + state.?cursor.list_updated_at_gte.orValue( + state.?cursor.last_timestamp.orValue( (now - duration(state.initial_interval)).format(time_layout.RFC3339) ) ) ], ?"siteIds": state.?site_ids.optMap(v, [string(v)]), - ?"cursor": state.?cursor.?next_page.token.optMap(v, [v]), + ?"cursor": state.?cursor.next_page.token.optMap(v, [v]), }.format_query() ).with( { @@ -49,8 +49,8 @@ inputs: ?"token": (body.?pagination.nextCursor.orValue(null) != null) ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": body.?pagination.nextCursor.orValue(null) != null, - "list_updated_at_gte": state.?cursor.?list_updated_at_gte.orValue( - state.?cursor.?last_timestamp.orValue( + "list_updated_at_gte": state.?cursor.list_updated_at_gte.orValue( + state.?cursor.last_timestamp.orValue( (now - duration(state.initial_interval)).format(time_layout.RFC3339) ) ), @@ -76,7 +76,9 @@ inputs: } ) ) - ).as(state, + ).as(state, has(state.?events.error) ? + state + : state.with( !has(state.?cursor.worklist) ? state @@ -94,14 +96,20 @@ inputs: "Authorization": ["ApiToken " + state.api_token], }, } - ).do_request().as(resp, (resp.StatusCode == 200) ? - resp.Body.decode_json().as(body, + ).do_request().as(resp, (resp.StatusCode == 200 || resp.StatusCode == 404) ? + ( + (resp.StatusCode == 200) ? + resp.Body.decode_json() + : + // Do not bother getting 404 body since it's not used. + {} + ).as(body, (body.?pagination.nextCursor.orValue(null) != null).as(has_more_events, { "data": has_more_events ? state.cursor.worklist.data : tail(state.cursor.worklist.data), }.as(new_worklist, { - "events": (has(body.data) && body.data.size() > 0) ? + "events": (resp.StatusCode == 200 && has(body.data) && body.data.size() > 0) ? body.data.map(e, { "message": e.encode_json(), @@ -122,7 +130,16 @@ inputs: ?"token": has_more_events ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": state.?cursor.fetch_more.orValue(false), - "list_updated_at_gte": state.cursor.list_updated_at_gte, + ?"list_updated_at_gte": (body.?pagination.nextCursor.orValue(null) != null) ? + optional.of( + state.?cursor.list_updated_at_gte.orValue( + state.?cursor.last_timestamp.orValue( + (now - duration(state.initial_interval)).format(time_layout.RFC3339) + ) + ) + ) + : + optional.none(), ?"last_timestamp": has_more_events ? state.?cursor.last_timestamp : diff --git a/packages/sentinel_one/data_stream/threat_event/_dev/test/scripts/deleted_threat_404.txt b/packages/sentinel_one/data_stream/threat_event/_dev/test/scripts/deleted_threat_404.txt new file mode 100644 index 00000000000..1ad044e57e8 --- /dev/null +++ b/packages/sentinel_one/data_stream/threat_event/_dev/test/scripts/deleted_threat_404.txt @@ -0,0 +1,104 @@ +# Test that a 404 on a deleted threat's explore/events endpoint does not +# stall the input. The mock returns two threats: a deleted one (404 on +# explore/events) followed by a valid one (200 with events). The CEL +# program should skip the deleted threat and collect events from the +# valid one. + +[!external_stack] skip 'Skipping external stack test.' +[!exec:jq] skip 'Skipping test requiring absent jq command' + +use_stack -profile ${CONFIG_PROFILES}/${PROFILE} +install_agent -profile ${CONFIG_PROFILES}/${PROFILE} -network_name NETWORK_NAME +docker_up -profile ${CONFIG_PROFILES}/${PROFILE} -network ${NETWORK_NAME} s1-mock +add_package -profile ${CONFIG_PROFILES}/${PROFILE} +add_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} test_config.yaml DATA_STREAM_NAME + +# Wait for 2 threat event documents from the valid threat. The deleted +# threat's 404 produces a "retry" message that the pipeline drops. +get_docs -profile ${CONFIG_PROFILES}/${PROFILE} -want 2 -confirm 15s -timeout 5m ${DATA_STREAM_NAME} +cp stdout got_docs.json + +# Verify both documents are real threat events, not error events. +exec jq '[.hits.hits[]._source | select(.sentinel_one.threat_event != null)] | length' got_docs.json +stdout '^2$' + +# Verify no error events were indexed. +exec jq '[.hits.hits[]._source | select(.error.message != null)] | length' got_docs.json +stdout '^0$' + +remove_package_policy -profile ${CONFIG_PROFILES}/${PROFILE} ${DATA_STREAM_NAME} +uninstall_agent -profile ${CONFIG_PROFILES}/${PROFILE} -timeout 1m +docker_down s1-mock + +-- test_config.yaml -- +input: cel +vars: + url: http://s1-mock:8080 + api_token: test-api-token +data_stream: + vars: + interval: 30s + batch_size: 10 + preserve_original_event: true + enable_request_tracer: false +-- s1-mock/docker-compose.yml -- +version: '2.3' +services: + s1-mock: + image: docker.elastic.co/observability/stream:v0.20.0 + hostname: s1-mock + ports: + - 8080 + environment: + PORT: "8080" + volumes: + - ./config.yml:/config.yml + command: + - http-server + - --addr=:8080 + - --config=/config.yml +-- s1-mock/config.yml -- +rules: + # Threats list: returns two threats. deleted-threat-001 has been removed + # from SentinelOne; threat-002 is still valid. The deleted threat is + # first so the CEL program hits the 404 before reaching the valid one. + - path: /web/api/v2.1/threats + methods: [GET] + request_headers: + Authorization: + - "ApiToken test-api-token" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"data":[{"id":"deleted-threat-001","threatInfo":{"threatName":"DeletedThreat","classification":"Malware","createdAt":"2024-01-15T10:00:00.000Z","updatedAt":"2024-01-15T10:01:00.000Z"}},{"id":"threat-002","threatInfo":{"threatName":"ValidThreat","classification":"Trojan","createdAt":"2024-01-15T11:00:00.000Z","updatedAt":"2024-01-15T11:01:00.000Z"}}],"pagination":{"nextCursor":null,"totalItems":2}} + + # Explore events for deleted threat: 404 with SentinelOne error format. + - path: /web/api/v2.1/threats/deleted-threat-001/explore/events + methods: [GET] + request_headers: + Authorization: + - "ApiToken test-api-token" + responses: + - status_code: 404 + headers: + Content-Type: + - "application/json" + body: |- + {"code":4040010,"detail":"Threat ID deleted-threat-001 not found","title":"Requested resource was not found"} + + # Explore events for valid threat: 200 with 2 events. + - path: /web/api/v2.1/threats/threat-002/explore/events + methods: [GET] + request_headers: + Authorization: + - "ApiToken test-api-token" + responses: + - status_code: 200 + headers: + Content-Type: + - "application/json" + body: |- + {"data":[{"id":"event-001","eventType":"DNS","createdAt":"2024-01-15T11:00:00.000Z","agentId":"agent-001","processName":"explorer.exe","pid":"1234","srcIp":"89.160.20.128","dstIp":"127.0.0.1","dstPort":53},{"id":"event-002","eventType":"NETWORK","createdAt":"2024-01-15T11:01:00.000Z","agentId":"agent-001","processName":"chrome.exe","pid":"5678","srcIp":"89.160.20.128","dstIp":"127.0.0.1","dstPort":443}],"pagination":{"nextCursor":null,"totalItems":2}} diff --git a/packages/sentinel_one/data_stream/threat_event/agent/stream/cel.yml.hbs b/packages/sentinel_one/data_stream/threat_event/agent/stream/cel.yml.hbs index d3792581b6c..6c5b5020c9c 100644 --- a/packages/sentinel_one/data_stream/threat_event/agent/stream/cel.yml.hbs +++ b/packages/sentinel_one/data_stream/threat_event/agent/stream/cel.yml.hbs @@ -39,14 +39,14 @@ program: |- "sortBy": ["updatedAt"], "sortOrder": ["asc"], "updatedAt__gte": [ - state.?cursor.?list_updated_at_gte.orValue( - state.?cursor.?last_timestamp.orValue( + state.?cursor.list_updated_at_gte.orValue( + state.?cursor.last_timestamp.orValue( (now - duration(state.initial_interval)).format(time_layout.RFC3339) ) ) ], ?"siteIds": state.?site_ids.optMap(v, [string(v)]), - ?"cursor": state.?cursor.?next_page.token.optMap(v, [v]), + ?"cursor": state.?cursor.next_page.token.optMap(v, [v]), }.format_query() ).with( { @@ -63,8 +63,8 @@ program: |- ?"token": (body.?pagination.nextCursor.orValue(null) != null) ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": body.?pagination.nextCursor.orValue(null) != null, - "list_updated_at_gte": state.?cursor.?list_updated_at_gte.orValue( - state.?cursor.?last_timestamp.orValue( + "list_updated_at_gte": state.?cursor.list_updated_at_gte.orValue( + state.?cursor.last_timestamp.orValue( (now - duration(state.initial_interval)).format(time_layout.RFC3339) ) ), @@ -90,7 +90,9 @@ program: |- } ) ) - ).as(state, + ).as(state, has(state.?events.error) ? + state + : state.with( !has(state.?cursor.worklist) ? state @@ -108,14 +110,20 @@ program: |- "Authorization": ["ApiToken " + state.api_token], }, } - ).do_request().as(resp, (resp.StatusCode == 200) ? - resp.Body.decode_json().as(body, + ).do_request().as(resp, (resp.StatusCode == 200 || resp.StatusCode == 404) ? + ( + (resp.StatusCode == 200) ? + resp.Body.decode_json() + : + // Do not bother getting 404 body since it's not used. + {} + ).as(body, (body.?pagination.nextCursor.orValue(null) != null).as(has_more_events, { "data": has_more_events ? state.cursor.worklist.data : tail(state.cursor.worklist.data), }.as(new_worklist, { - "events": (has(body.data) && body.data.size() > 0) ? + "events": (resp.StatusCode == 200 && has(body.data) && body.data.size() > 0) ? body.data.map(e, { "message": e.encode_json(), @@ -136,7 +144,16 @@ program: |- ?"token": has_more_events ? optional.of(body.pagination.nextCursor) : optional.none(), }, "fetch_more": state.?cursor.fetch_more.orValue(false), - "list_updated_at_gte": state.cursor.list_updated_at_gte, + ?"list_updated_at_gte": (body.?pagination.nextCursor.orValue(null) != null) ? + optional.of( + state.?cursor.list_updated_at_gte.orValue( + state.?cursor.last_timestamp.orValue( + (now - duration(state.initial_interval)).format(time_layout.RFC3339) + ) + ) + ) + : + optional.none(), ?"last_timestamp": has_more_events ? state.?cursor.last_timestamp : diff --git a/packages/sentinel_one/data_stream/threat_event/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/threat_event/elasticsearch/ingest_pipeline/default.yml index e9df1d16e9e..63ab0febdc4 100644 --- a/packages/sentinel_one/data_stream/threat_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one/data_stream/threat_event/elasticsearch/ingest_pipeline/default.yml @@ -819,7 +819,7 @@ processors: tag: convert_file_size_to_long target_field: file.size type: long - if: ctx.sentinel_one?.threat_event?.file.size != '' + if: ctx.sentinel_one?.threat_event?.file?.size != '' ignore_missing: true on_failure: - append: @@ -907,7 +907,7 @@ processors: on_failure: - append: field: error.message - value: |- + value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/sentinel_one/manifest.yml b/packages/sentinel_one/manifest.yml index f270bd1b9ea..9006b00db81 100644 --- a/packages/sentinel_one/manifest.yml +++ b/packages/sentinel_one/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: sentinel_one title: SentinelOne -version: "2.11.2" +version: "2.11.3" description: Collect logs from SentinelOne with Elastic Agent. type: integration categories: