feat(wsgi): Apply data_collection filtering to URL query strings

sentry_sdk/_types.py

@@ -183,7 +183,7 @@ class DataCollectionUserOptions(TypedDict, total=False):
         cookies: "KeyValueCollectionBehaviour"
         http_headers: "HttpHeadersCollectionUserOptions"
         http_bodies: "List[str]"
-        query_params: "KeyValueCollectionBehaviour"
+        url_query_params: "KeyValueCollectionBehaviour"
         graphql: "GraphQLCollectionUserOptions"
         gen_ai: "GenAICollectionUserOptions"
         database_query_data: bool
@@ -197,7 +197,7 @@ class DataCollection(TypedDict):
         cookies: "KeyValueCollectionBehaviour"
         http_headers: "HttpHeadersCollectionBehaviour"
         http_bodies: "List[str]"
-        query_params: "KeyValueCollectionBehaviour"
+        url_query_params: "KeyValueCollectionBehaviour"
         graphql: "GraphQLCollectionBehaviour"
         gen_ai: "GenAICollectionBehaviour"
         database_query_data: bool

sentry_sdk/data_collection.py

@@ -7,7 +7,7 @@
 
 ``data_collection`` supersedes the single ``send_default_pii`` boolean with a
 structured configuration that lets users enable or restrict automatically
-collected data by category (user identity, cookies, HTTP headers, query params,
+collected data by category (user identity, cookies, HTTP headers, URL query params,
 HTTP bodies, generative AI inputs/outputs, stack frame variables, source
 context).
 
@@ -23,10 +23,13 @@
 """
 
 import warnings
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, List, Mapping, Optional, Union, cast
+from urllib.parse import parse_qs
+
+from sentry_sdk._types import SENSITIVE_DATA_SUBSTITUTE
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, Literal
+    from typing import Any, Dict
 
     from sentry_sdk._types import (
         DataCollection,
@@ -48,7 +51,7 @@
 # Default number of source lines captured above and below a stack frame.
 _DEFAULT_FRAME_CONTEXT_LINES = 5
 
-# Collection modes for key-value data (cookies, headers, query params).
+# Collection modes for key-value data (cookies, headers, URL query params).
 # snake_case (Python-only deviation from the spec's camelCase); never
 # serialized to Sentry.
 _VALID_KEY_VALUE_COLLECTION_BEHAVIOUR_MODES = ("off", "denylist", "allowlist")
@@ -77,6 +80,82 @@
 ]
 
 
+def _is_sensitive_key(key: str, extra_terms: "Optional[List[str]]" = None) -> bool:
+    """
+    Return whether ``key`` matches the sensitive denylist using a partial,
+    case-insensitive substring match.
+
+    :param extra_terms: additional deny terms (e.g. user-provided) to consider
+        alongside the built-in `_SENSITIVE_DENYLIST`.
+    """
+    lowered = key.lower()
+    for term in _SENSITIVE_DENYLIST:
+        if term in lowered:
+            return True
+    if extra_terms:
+        for term in extra_terms:
+            if term and term.lower() in lowered:
+                return True
+    return False
+
+
+def _apply_data_collection_filtering_to_query_string(
+    query_string: str,
+    behaviour: "KeyValueCollectionBehaviour",
+) -> "Union[str, None]":
+    parsed_qs = parse_qs(query_string, keep_blank_values=True)
+    filtered_qs = _apply_key_value_collection_filtering(
+        items=parsed_qs, behaviour=behaviour
+    )
+
+    if filtered_qs:
+        parts = []
+        for key, value in filtered_qs.items():
+            values = value if isinstance(value, list) else [value]
+            for item in values:
+                parts.append(f"{key}={item}")
+        return "&".join(parts)
+
+    return None
+
+
+def _apply_key_value_collection_filtering(
+    items: "Mapping[str, Any]",
+    behaviour: "KeyValueCollectionBehaviour",
+    substitute: "Any" = SENSITIVE_DATA_SUBSTITUTE,
+) -> "Dict[str, Any]":
+
+    if behaviour["mode"] == "off":
+        return {}
+
+    result: "Dict[str, Any]" = {}
+
+    if behaviour["mode"] == "allowlist":
+        for key, value in items.items():
+            is_allowed = False
+            if isinstance(key, str):
+                lowered = key.lower()
+                is_allowed = any(
+                    term and term.lower() in lowered
+                    for term in behaviour.get("terms", [])
+                )
+            if is_allowed and not _is_sensitive_key(key):
+                result[key] = value
+            else:
+                result[key] = substitute
+        return result
+
+    # denylist behaviour
+    for key, value in items.items():
+        if isinstance(key, str) and _is_sensitive_key(
+            key=key, extra_terms=behaviour.get("terms", [])
+        ):
+            result[key] = substitute
+        else:
+            result[key] = value
+    return result
+
+
 def _map_from_send_default_pii(
     *,
     send_default_pii: bool,
@@ -88,13 +167,12 @@
     ``send_default_pii`` collects today. Used when ``data_collection`` is not
     provided explicitly.
     """
-    kv_mode: "Literal['denylist', 'off']" = "denylist" if send_default_pii else "off"
     terms = [] if send_default_pii else ["forwarded", "-ip", "remote-", "via", "-user"]
 
     return {
         "provided_by_user": False,
         "user_info": send_default_pii,
-        "cookies": {"mode": kv_mode, "terms": terms},
+        "cookies": {"mode": "denylist", "terms": terms},
         # Headers are collected in both PII modes today (sensitive ones filtered
         # when PII is off), so this never maps to "off".
         "http_headers": {
@@ -103,7 +181,7 @@
         # Bodies are collected regardless of PII today, bounded by
         # ``max_request_body_size``.
         "http_bodies": list(_ALL_HTTP_BODY_TYPES),
-        "query_params": {"mode": kv_mode, "terms": terms},
+        "url_query_params": {"mode": "denylist", "terms": terms},
         "graphql": {"document": send_default_pii, "variables": send_default_pii},
         "gen_ai": {"inputs": send_default_pii, "outputs": send_default_pii},
         "database_query_data": send_default_pii,
@@ -153,7 +231,7 @@
         "cookies": _kvcb_from_value(d.get("cookies") or {}),
         "http_headers": _http_headers_from_value(d.get("http_headers") or {}),
         "http_bodies": http_bodies,
-        "query_params": _kvcb_from_value(d.get("query_params") or {}),
+        "url_query_params": _kvcb_from_value(d.get("url_query_params") or {}),
         "graphql": _graphql_from_value(d.get("graphql") or {}),
         "gen_ai": _gen_ai_from_value(d.get("gen_ai") or {}),
         "database_query_data": d.get("database_query_data", True),

sentry_sdk/integrations/_asgi_common.py

@@ -115,9 +115,13 @@ def _get_request_data(
     if ty in ("http", "websocket"):
         request_data["method"] = asgi_scope.get("method")
 
-        request_data["headers"] = headers = _filter_headers(
-            _get_headers(asgi_scope),
+        headers = _get_headers(asgi_scope)
+
+        request_data["headers"] = _filter_headers(
+            headers,
+            use_annotated_value=False,
         )
+
         request_data["query_string"] = _get_query(asgi_scope)
 
         request_data["url"] = _get_url(
@@ -148,8 +152,10 @@ def _get_request_attributes(
         if asgi_scope.get("method"):
             attributes["http.request.method"] = asgi_scope["method"].upper()
 
-        headers = _filter_headers(_get_headers(asgi_scope), use_annotated_value=False)
-        for header, value in headers.items():
+        headers = _get_headers(asgi_scope)
+
+        filtered_headers = _filter_headers(headers, use_annotated_value=False)
+        for header, value in filtered_headers.items():
             attributes[f"http.request.header.{header.lower()}"] = value
 
         if should_send_default_pii():

sentry_sdk/integrations/_wsgi_common.py

@@ -3,8 +3,9 @@
 
 import sentry_sdk
 from sentry_sdk._types import SENSITIVE_DATA_SUBSTITUTE
+from sentry_sdk.data_collection import _apply_key_value_collection_filtering
 from sentry_sdk.scope import should_send_default_pii
-from sentry_sdk.utils import AnnotatedValue, logger
+from sentry_sdk.utils import AnnotatedValue, has_data_collection_enabled, logger
 
 try:
     from django.http.request import RawPostDataException
@@ -88,7 +89,14 @@ def extract_into_event(self, event: "Event") -> None:
         content_length = self.content_length()
         request_info = event.get("request", {})
 
-        if should_send_default_pii():
+        if has_data_collection_enabled(client.options):
+            cookies = _apply_key_value_collection_filtering(
+                items=dict(self.cookies()),
+                behaviour=client.options["data_collection"]["cookies"],
+            )
+            if cookies:
+                request_info["cookies"] = cookies
+        elif should_send_default_pii():
             request_info["cookies"] = dict(self.cookies())
 
         if not request_body_within_bounds(client, content_length):
@@ -206,19 +214,39 @@ def _filter_headers(
     headers: "Mapping[str, str]",
     use_annotated_value: bool = True,
 ) -> "Mapping[str, Union[AnnotatedValue, str]]":
-    if should_send_default_pii():
-        return headers
+    client_options = sentry_sdk.get_client().options
 
-    substitute: "Union[AnnotatedValue, str]" = (
-        SENSITIVE_DATA_SUBSTITUTE
-        if not use_annotated_value
-        else AnnotatedValue.removed_because_over_size_limit()
-    )
+    if has_data_collection_enabled(client_options):
+        data_collection_configuration = client_options["data_collection"]
+
+        filtered = _apply_key_value_collection_filtering(
+            items=headers,
+            behaviour=data_collection_configuration["http_headers"]["request"],
+        )
 
-    return {
-        k: (v if k.upper().replace("-", "_") not in SENSITIVE_HEADERS else substitute)
-        for k, v in headers.items()
-    }
+        for key in filtered:
+            if isinstance(key, str) and key.lower() in ("cookie", "set-cookie"):
+                filtered[key] = SENSITIVE_DATA_SUBSTITUTE
+
+        return filtered
+    else:
+        if should_send_default_pii():
+            return headers
+
+        substitute: "Union[AnnotatedValue, str]" = (
+            SENSITIVE_DATA_SUBSTITUTE
+            if not use_annotated_value
+            else AnnotatedValue.removed_because_over_size_limit()
+        )
+
+        return {
+            k: (
+                v
+                if k.upper().replace("-", "_") not in SENSITIVE_HEADERS
+                else substitute
+            )
+            for k, v in headers.items()
+        }
 
 
 def _in_http_status_code_range(

sentry_sdk/integrations/aiohttp.py

@@ -150,7 +150,8 @@ async def sentry_app_handle(
 
                         header_attributes: "dict[str, Any]" = {}
                         for header, header_value in _filter_headers(
-                            headers, use_annotated_value=False
+                            headers,
+                            use_annotated_value=False,
                         ).items():
                             header_attributes[
                                 f"http.request.header.{header.lower()}"

sentry_sdk/integrations/fastapi.py

@@ -6,7 +6,6 @@
 import sentry_sdk
 from sentry_sdk.consts import SPANDATA
 from sentry_sdk.integrations import DidNotEnable
-from sentry_sdk.scope import should_send_default_pii
 from sentry_sdk.traces import StreamedSpan, get_current_span
 from sentry_sdk.tracing import SOURCE_FOR_STYLE, TransactionSource
 from sentry_sdk.tracing_utils import has_span_streaming_enabled
@@ -118,7 +117,7 @@ def event_processor(event: "Event", hint: "Dict[str, Any]") -> "Event":
             # Extract information from request
             request_info = event.get("request", {})
             if info:
-                if "cookies" in info and should_send_default_pii():
+                if "cookies" in info:
                     request_info["cookies"] = info["cookies"]
                 if "data" in info:
                     request_info["data"] = info["data"]

sentry_sdk/integrations/litestar.py

@@ -3,6 +3,7 @@
 
 import sentry_sdk
 from sentry_sdk.consts import OP, SPANDATA
+from sentry_sdk.data_collection import _apply_key_value_collection_filtering
 from sentry_sdk.integrations import (
     _DEFAULT_FAILED_REQUEST_STATUS_CODES,
     DidNotEnable,
@@ -16,6 +17,7 @@
 from sentry_sdk.utils import (
     ensure_integration_enabled,
     event_from_exception,
+    has_data_collection_enabled,
     transaction_from_function,
 )
 
@@ -279,7 +281,8 @@ def patch_http_route_handle() -> None:
     async def handle_wrapper(
         self: "HTTPRoute", scope: "HTTPScope", receive: "Receive", send: "Send"
     ) -> None:
-        if sentry_sdk.get_client().get_integration(LitestarIntegration) is None:
+        client = sentry_sdk.get_client()
+        if client.get_integration(LitestarIntegration) is None:
             return await old_handle(self, scope, receive, send)
 
         sentry_scope = sentry_sdk.get_isolation_scope()
@@ -318,7 +321,14 @@ async def handle_wrapper(
         def event_processor(event: "Event", _: "Hint") -> "Event":
             request_info = event.get("request", {})
             request_info["content_length"] = len(scope.get("_body", b""))
-            if should_send_default_pii():
+            if has_data_collection_enabled(client.options):
+                cookies = _apply_key_value_collection_filtering(
+                    items=extracted_request_data["cookies"],
+                    behaviour=client.options["data_collection"]["cookies"],
+                )
+                if cookies:
+                    request_info["cookies"] = cookies
+            elif should_send_default_pii():
                 request_info["cookies"] = extracted_request_data["cookies"]
             if request_data is not None:
                 request_info["data"] = request_data

sentry_sdk/integrations/starlette.py

@@ -10,6 +10,7 @@
 import sentry_sdk
 from sentry_sdk._types import OVER_SIZE_LIMIT_SUBSTITUTE
 from sentry_sdk.consts import OP, SPANDATA
+from sentry_sdk.data_collection import _apply_key_value_collection_filtering
 from sentry_sdk.integrations import (
     _DEFAULT_FAILED_REQUEST_STATUS_CODES,
     DidNotEnable,
@@ -35,6 +36,7 @@
     capture_internal_exceptions,
     ensure_integration_enabled,
     event_from_exception,
+    has_data_collection_enabled,
     nullcontext,
     parse_version,
     transaction_from_function,
@@ -719,8 +721,15 @@ def __init__(self: "StarletteRequestExtractor", request: "Request") -> None:
     def extract_cookies_from_request(
         self: "StarletteRequestExtractor",
     ) -> "Optional[Dict[str, Any]]":
+        client_options = sentry_sdk.get_client().options
         cookies: "Optional[Dict[str, Any]]" = None
-        if should_send_default_pii():
+
+        if has_data_collection_enabled(client_options):
+            cookies = _apply_key_value_collection_filtering(
+                items=self.cookies(),
+                behaviour=client_options["data_collection"]["cookies"],
+            )
+        elif should_send_default_pii():
             cookies = self.cookies()
 
         return cookies
@@ -734,7 +743,14 @@ async def extract_request_info(
 
         with capture_internal_exceptions():
             # Add cookies
-            if should_send_default_pii():
+            if has_data_collection_enabled(client.options):
+                cookies = _apply_key_value_collection_filtering(
+                    items=self.cookies(),
+                    behaviour=client.options["data_collection"]["cookies"],
+                )
+                if cookies:
+                    request_info["cookies"] = cookies
+            elif should_send_default_pii():
                 request_info["cookies"] = self.cookies()
 
             # If there is no body, just return the cookies