feat(aiohttp): Apply data_collection filtering to URL query strings

sentry_sdk/_types.py

@@ -183,7 +183,7 @@ class DataCollectionUserOptions(TypedDict, total=False):
         cookies: "KeyValueCollectionBehaviour"
         http_headers: "HttpHeadersCollectionUserOptions"
         http_bodies: "List[str]"
-        query_params: "KeyValueCollectionBehaviour"
+        url_query_params: "KeyValueCollectionBehaviour"
         graphql: "GraphQLCollectionUserOptions"
         gen_ai: "GenAICollectionUserOptions"
         database_query_data: bool
@@ -197,7 +197,7 @@ class DataCollection(TypedDict):
         cookies: "KeyValueCollectionBehaviour"
         http_headers: "HttpHeadersCollectionBehaviour"
         http_bodies: "List[str]"
-        query_params: "KeyValueCollectionBehaviour"
+        url_query_params: "KeyValueCollectionBehaviour"
         graphql: "GraphQLCollectionBehaviour"
         gen_ai: "GenAICollectionBehaviour"
         database_query_data: bool

sentry_sdk/data_collection.py

@@ -7,7 +7,7 @@
 
 ``data_collection`` supersedes the single ``send_default_pii`` boolean with a
 structured configuration that lets users enable or restrict automatically
-collected data by category (user identity, cookies, HTTP headers, query params,
+collected data by category (user identity, cookies, HTTP headers, URL query params,
 HTTP bodies, generative AI inputs/outputs, stack frame variables, source
 context).
 
@@ -23,10 +23,13 @@
 """
 
 import warnings
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, List, Mapping, Optional, Union, cast
+from urllib.parse import parse_qs
+
+from sentry_sdk._types import SENSITIVE_DATA_SUBSTITUTE
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, Literal
+    from typing import Any, Dict
 
     from sentry_sdk._types import (
         DataCollection,
@@ -48,7 +51,7 @@
 # Default number of source lines captured above and below a stack frame.
 _DEFAULT_FRAME_CONTEXT_LINES = 5
 
-# Collection modes for key-value data (cookies, headers, query params).
+# Collection modes for key-value data (cookies, headers, URL query params).
 # snake_case (Python-only deviation from the spec's camelCase); never
 # serialized to Sentry.
 _VALID_KEY_VALUE_COLLECTION_BEHAVIOUR_MODES = ("off", "denylist", "allowlist")
@@ -77,6 +80,82 @@
 ]
 
 
+def _is_sensitive_key(key: str, extra_terms: "Optional[List[str]]" = None) -> bool:
+    """
+    Return whether ``key`` matches the sensitive denylist using a partial,
+    case-insensitive substring match.
+
+    :param extra_terms: additional deny terms (e.g. user-provided) to consider
+        alongside the built-in `_SENSITIVE_DENYLIST`.
+    """
+    lowered = key.lower()
+    for term in _SENSITIVE_DENYLIST:
+        if term in lowered:
+            return True
+    if extra_terms:
+        for term in extra_terms:
+            if term and term.lower() in lowered:
+                return True
+    return False
+
+
+def _apply_data_collection_filtering_to_query_string(
+    query_string: str,
+    behaviour: "KeyValueCollectionBehaviour",
+) -> "Union[str, None]":
+    parsed_qs = parse_qs(query_string, keep_blank_values=True)
+    filtered_qs = _apply_key_value_collection_filtering(
+        items=parsed_qs, behaviour=behaviour
+    )
+
+    if filtered_qs:
+        parts = []
+        for key, value in filtered_qs.items():
+            values = value if isinstance(value, list) else [value]
+            for item in values:
+                parts.append(f"{key}={item}")
+        return "&".join(parts)
+
+    return None
+
+
+def _apply_key_value_collection_filtering(
+    items: "Mapping[str, Any]",
+    behaviour: "KeyValueCollectionBehaviour",
+    substitute: "Any" = SENSITIVE_DATA_SUBSTITUTE,
+) -> "Dict[str, Any]":
+
+    if behaviour["mode"] == "off":
+        return {}
+
+    result: "Dict[str, Any]" = {}
+
+    if behaviour["mode"] == "allowlist":
+        for key, value in items.items():
+            is_allowed = False
+            if isinstance(key, str):
+                lowered = key.lower()
+                is_allowed = any(
+                    term and term.lower() in lowered
+                    for term in behaviour.get("terms", [])
+                )
+            if is_allowed and not _is_sensitive_key(key):
+                result[key] = value
+            else:
+                result[key] = substitute
+        return result
+
+    # denylist behaviour
+    for key, value in items.items():
+        if isinstance(key, str) and _is_sensitive_key(
+            key=key, extra_terms=behaviour.get("terms", [])
+        ):
+            result[key] = substitute
+        else:
+            result[key] = value
+    return result
+
+
 def _map_from_send_default_pii(
     *,
     send_default_pii: bool,
@@ -88,13 +167,12 @@ def _map_from_send_default_pii(
     ``send_default_pii`` collects today. Used when ``data_collection`` is not
     provided explicitly.
     """
-    kv_mode: "Literal['denylist', 'off']" = "denylist" if send_default_pii else "off"
     terms = [] if send_default_pii else ["forwarded", "-ip", "remote-", "via", "-user"]
 
     return {
         "provided_by_user": False,
         "user_info": send_default_pii,
-        "cookies": {"mode": kv_mode, "terms": terms},
+        "cookies": {"mode": "denylist", "terms": terms},
         # Headers are collected in both PII modes today (sensitive ones filtered
         # when PII is off), so this never maps to "off".
         "http_headers": {
@@ -103,7 +181,7 @@ def _map_from_send_default_pii(
         # Bodies are collected regardless of PII today, bounded by
         # ``max_request_body_size``.
         "http_bodies": list(_ALL_HTTP_BODY_TYPES),
-        "query_params": {"mode": kv_mode, "terms": terms},
+        "url_query_params": {"mode": "denylist", "terms": terms},
         "graphql": {"document": send_default_pii, "variables": send_default_pii},
         "gen_ai": {"inputs": send_default_pii, "outputs": send_default_pii},
         "database_query_data": send_default_pii,
@@ -153,7 +231,7 @@ def _resolve_explicit(
         "cookies": _kvcb_from_value(d.get("cookies") or {}),
         "http_headers": _http_headers_from_value(d.get("http_headers") or {}),
         "http_bodies": http_bodies,
-        "query_params": _kvcb_from_value(d.get("query_params") or {}),
+        "url_query_params": _kvcb_from_value(d.get("url_query_params") or {}),
         "graphql": _graphql_from_value(d.get("graphql") or {}),
         "gen_ai": _gen_ai_from_value(d.get("gen_ai") or {}),
         "database_query_data": d.get("database_query_data", True),

sentry_sdk/integrations/_asgi_common.py

@@ -2,8 +2,11 @@
 from enum import Enum
 from typing import TYPE_CHECKING
 
+import sentry_sdk
+from sentry_sdk.data_collection import _apply_data_collection_filtering_to_query_string
 from sentry_sdk.integrations._wsgi_common import _filter_headers
 from sentry_sdk.scope import should_send_default_pii
+from sentry_sdk.utils import has_data_collection_enabled
 
 if TYPE_CHECKING:
     from typing import Any, Dict, Optional, Union
@@ -75,7 +78,7 @@ def _get_url(
     return path
 
 
-def _get_query(asgi_scope: "Any") -> "Any":
+def _get_query(asgi_scope: "Any") -> "Optional[str]":
     """
     Extract querystring from the ASGI scope, in the format that the Sentry protocol expects.
     """
@@ -115,10 +118,27 @@ def _get_request_data(
     if ty in ("http", "websocket"):
         request_data["method"] = asgi_scope.get("method")
 
-        request_data["headers"] = headers = _filter_headers(
-            _get_headers(asgi_scope),
+        headers = _get_headers(asgi_scope)
+
+        request_data["headers"] = _filter_headers(
+            headers,
+            use_annotated_value=False,
         )
-        request_data["query_string"] = _get_query(asgi_scope)
+
+        client_options = sentry_sdk.get_client().options
+        if has_data_collection_enabled(client_options):
+            qs = _get_query(asgi_scope)
+            if qs:
+                filtered_query_string = (
+                    _apply_data_collection_filtering_to_query_string(
+                        query_string=qs,
+                        behaviour=client_options["data_collection"]["url_query_params"],
+                    )
+                )
+                if filtered_query_string:
+                    request_data["query_string"] = filtered_query_string
+        else:
+            request_data["query_string"] = _get_query(asgi_scope)
 
         request_data["url"] = _get_url(
             asgi_scope,
@@ -148,11 +168,44 @@ def _get_request_attributes(
         if asgi_scope.get("method"):
             attributes["http.request.method"] = asgi_scope["method"].upper()
 
-        headers = _filter_headers(_get_headers(asgi_scope), use_annotated_value=False)
-        for header, value in headers.items():
+        headers = _get_headers(asgi_scope)
+
+        filtered_headers = _filter_headers(headers, use_annotated_value=False)
+        for header, value in filtered_headers.items():
             attributes[f"http.request.header.{header.lower()}"] = value
 
-        if should_send_default_pii():
+        client_options = sentry_sdk.get_client().options
+        if has_data_collection_enabled(client_options):
+            filtered_query_string = None
+            query = _get_query(asgi_scope)
+
+            if query:
+                filtered_query_string = (
+                    _apply_data_collection_filtering_to_query_string(
+                        query_string=query,
+                        behaviour=client_options["data_collection"]["url_query_params"],
+                    )
+                )
+                if filtered_query_string:
+                    attributes["http.query"] = filtered_query_string
+
+            path = _get_path(asgi_scope=asgi_scope, root_path_in_path=root_path_in_path)
+            attributes["url.path"] = path
+
+            url_without_query_string = _get_url(
+                asgi_scope,
+                "http" if ty == "http" else "ws",
+                headers.get("host"),
+                path=path,
+            )
+
+            attributes["url.full"] = (
+                f"{url_without_query_string}?{filtered_query_string}"
+                if filtered_query_string is not None
+                else url_without_query_string
+            )
+
+        elif should_send_default_pii():
             query = _get_query(asgi_scope)
             if query:
                 attributes["http.query"] = query

sentry_sdk/integrations/_wsgi_common.py

@@ -3,8 +3,9 @@
 
 import sentry_sdk
 from sentry_sdk._types import SENSITIVE_DATA_SUBSTITUTE
+from sentry_sdk.data_collection import _apply_key_value_collection_filtering
 from sentry_sdk.scope import should_send_default_pii
-from sentry_sdk.utils import AnnotatedValue, logger
+from sentry_sdk.utils import AnnotatedValue, has_data_collection_enabled, logger
 
 try:
     from django.http.request import RawPostDataException
@@ -88,7 +89,14 @@ def extract_into_event(self, event: "Event") -> None:
         content_length = self.content_length()
         request_info = event.get("request", {})
 
-        if should_send_default_pii():
+        if has_data_collection_enabled(client.options):
+            cookies = _apply_key_value_collection_filtering(
+                items=dict(self.cookies()),
+                behaviour=client.options["data_collection"]["cookies"],
+            )
+            if cookies:
+                request_info["cookies"] = cookies
+        elif should_send_default_pii():
             request_info["cookies"] = dict(self.cookies())
 
         if not request_body_within_bounds(client, content_length):
@@ -206,19 +214,39 @@ def _filter_headers(
     headers: "Mapping[str, str]",
     use_annotated_value: bool = True,
 ) -> "Mapping[str, Union[AnnotatedValue, str]]":
-    if should_send_default_pii():
-        return headers
+    client_options = sentry_sdk.get_client().options
 
-    substitute: "Union[AnnotatedValue, str]" = (
-        SENSITIVE_DATA_SUBSTITUTE
-        if not use_annotated_value
-        else AnnotatedValue.removed_because_over_size_limit()
-    )
+    if has_data_collection_enabled(client_options):
+        data_collection_configuration = client_options["data_collection"]
+
+        filtered = _apply_key_value_collection_filtering(
+            items=headers,
+            behaviour=data_collection_configuration["http_headers"]["request"],
+        )
 
-    return {
-        k: (v if k.upper().replace("-", "_") not in SENSITIVE_HEADERS else substitute)
-        for k, v in headers.items()
-    }
+        for key in filtered:
+            if isinstance(key, str) and key.lower() in ("cookie", "set-cookie"):
+                filtered[key] = SENSITIVE_DATA_SUBSTITUTE
+
+        return filtered
+    else:
+        if should_send_default_pii():
+            return headers
+
+        substitute: "Union[AnnotatedValue, str]" = (
+            SENSITIVE_DATA_SUBSTITUTE
+            if not use_annotated_value
+            else AnnotatedValue.removed_because_over_size_limit()
+        )
+
+        return {
+            k: (
+                v
+                if k.upper().replace("-", "_") not in SENSITIVE_HEADERS
+                else substitute
+            )
+            for k, v in headers.items()
+        }
 
 
 def _in_http_status_code_range(