From bcc5f468426fc55f9a6a7d2b9bce04fb22da08fc Mon Sep 17 00:00:00 2001 From: Jan-Kazlouski-elastic Date: Thu, 9 Jul 2026 13:34:42 +0300 Subject: [PATCH 1/3] [MongoDB] Handle out-of-range BSON datetimes to prevent sync failures Documents containing dates outside the Python datetime range (years 1-9999) caused pymongo to raise InvalidBSON during decode, aborting the entire sync. Configure the client with DATETIME_AUTO so such values are returned as DatetimeMS, and serialize them as raw milliseconds since epoch. Relates to elastic/sdh-search#1924 Co-authored-by: Cursor --- .../connectors/sources/mongo/datasource.py | 19 ++++++++- .../tests/sources/test_mongo.py | 41 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/app/connectors_service/connectors/sources/mongo/datasource.py b/app/connectors_service/connectors/sources/mongo/datasource.py index 2fc9f70d3..5af98aa9c 100644 --- a/app/connectors_service/connectors/sources/mongo/datasource.py +++ b/app/connectors_service/connectors/sources/mongo/datasource.py @@ -10,7 +10,15 @@ from datetime import datetime from tempfile import NamedTemporaryFile -from bson import OLD_UUID_SUBTYPE, Binary, DBRef, Decimal128, ObjectId +from bson import ( + OLD_UUID_SUBTYPE, + Binary, + DatetimeConversion, + DatetimeMS, + DBRef, + Decimal128, + ObjectId, +) from bson.binary import UUID_SUBTYPE from connectors_sdk.source import BaseDataSource, ConfigurableFieldValueError from motor.motor_asyncio import AsyncIOMotorClient @@ -107,6 +115,11 @@ def get_client(self): try: client_params = {} + # Return out-of-range BSON datetimes (years outside 1-9999) as + # DatetimeMS instead of raising InvalidBSON, so a single document + # with an extreme date can't abort the whole sync. + client_params["datetime_conversion"] = DatetimeConversion.DATETIME_AUTO + if self.configuration["direct_connection"]: client_params["directConnection"] = True @@ -172,6 +185,10 @@ def _serialize(value): value = value.to_decimal() elif isinstance(value, DBRef): value = _serialize(value.as_doc().to_dict()) + elif isinstance(value, DatetimeMS): + # Out-of-range dates can't be represented as datetime, so keep + # the raw milliseconds since epoch as an integer. + value = int(value) elif isinstance(value, Binary): # UUID_SUBTYPE is guaranteed to properly be serialized cross-platform and cross-driver if value.subtype == UUID_SUBTYPE: diff --git a/app/connectors_service/tests/sources/test_mongo.py b/app/connectors_service/tests/sources/test_mongo.py index bcf0747ca..d8f7f0213 100644 --- a/app/connectors_service/tests/sources/test_mongo.py +++ b/app/connectors_service/tests/sources/test_mongo.py @@ -11,7 +11,7 @@ from uuid import UUID import pytest -from bson import Binary, DBRef, ObjectId +from bson import Binary, DatetimeConversion, DatetimeMS, DBRef, ObjectId from bson.decimal128 import Decimal128 from connectors_sdk.filtering.validation import Filter from connectors_sdk.source import ConfigurableFieldValueError @@ -457,6 +457,35 @@ async def test_validate_config_when_configuration_valid_then_does_not_raise( }, {"some_binary_uuid": None}, ), + ( + # in-range datetime still serializes to an ISO string + {"created": datetime(2020, 1, 1, 12, 30, 0)}, + {"created": "2020-01-01T12:30:00"}, + ), + ( + # out-of-range date decoded as DatetimeMS -> raw milliseconds + {"too_big": DatetimeMS(2**62)}, + {"too_big": 2**62}, + ), + ( + {"too_small": DatetimeMS(-(2**62))}, + {"too_small": -(2**62)}, + ), + ( + {"epoch": DatetimeMS(0)}, + {"epoch": 0}, + ), + ( + # DatetimeMS nested inside a list and a dict + { + "nested": {"created": DatetimeMS(2**62)}, + "list": [DatetimeMS(0)], + }, + { + "nested": {"created": 2**62}, + "list": [0], + }, + ), ], ) async def test_serialize(raw, output): @@ -464,6 +493,16 @@ async def test_serialize(raw, output): assert source.serialize(raw) == output +@pytest.mark.asyncio +async def test_get_client_uses_datetime_auto_conversion(): + async with create_mongo_source() as source: + with source.get_client() as client: + assert ( + client.codec_options.datetime_conversion + == DatetimeConversion.DATETIME_AUTO + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "certificate_value, tls_insecure", From bf9e606aad76ab5aa0b17342fc52c3069ed88a8b Mon Sep 17 00:00:00 2001 From: Jan-Kazlouski-elastic Date: Thu, 9 Jul 2026 13:47:07 +0300 Subject: [PATCH 2/3] [MongoDB] Shorten inline comments for datetime handling Co-authored-by: Cursor --- .../connectors/sources/mongo/datasource.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app/connectors_service/connectors/sources/mongo/datasource.py b/app/connectors_service/connectors/sources/mongo/datasource.py index 5af98aa9c..30f8025ab 100644 --- a/app/connectors_service/connectors/sources/mongo/datasource.py +++ b/app/connectors_service/connectors/sources/mongo/datasource.py @@ -115,9 +115,7 @@ def get_client(self): try: client_params = {} - # Return out-of-range BSON datetimes (years outside 1-9999) as - # DatetimeMS instead of raising InvalidBSON, so a single document - # with an extreme date can't abort the whole sync. + # Return out-of-range dates as DatetimeMS instead of raising. client_params["datetime_conversion"] = DatetimeConversion.DATETIME_AUTO if self.configuration["direct_connection"]: @@ -186,8 +184,7 @@ def _serialize(value): elif isinstance(value, DBRef): value = _serialize(value.as_doc().to_dict()) elif isinstance(value, DatetimeMS): - # Out-of-range dates can't be represented as datetime, so keep - # the raw milliseconds since epoch as an integer. + # Keep out-of-range dates as raw milliseconds since epoch. value = int(value) elif isinstance(value, Binary): # UUID_SUBTYPE is guaranteed to properly be serialized cross-platform and cross-driver From 4e46f3a61e7fd48933265a0776462319bf432a2d Mon Sep 17 00:00:00 2001 From: Jan-Kazlouski-elastic Date: Thu, 9 Jul 2026 14:38:47 +0300 Subject: [PATCH 3/3] [MongoDB] Clamp out-of-range dates instead of serializing them as longs Switch datetime_conversion from DATETIME_AUTO to DATETIME_CLAMP so out-of-range BSON dates are clamped to datetime.min/max and stay valid dates. This avoids the mixed date/long Elasticsearch mapping conflict that DATETIME_AUTO could cause, and removes the now-unneeded DatetimeMS serialization branch. Co-authored-by: Cursor --- .../connectors/sources/mongo/datasource.py | 10 ++-- .../tests/sources/test_mongo.py | 59 +++++++++++-------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/app/connectors_service/connectors/sources/mongo/datasource.py b/app/connectors_service/connectors/sources/mongo/datasource.py index 30f8025ab..0222f08ca 100644 --- a/app/connectors_service/connectors/sources/mongo/datasource.py +++ b/app/connectors_service/connectors/sources/mongo/datasource.py @@ -14,7 +14,6 @@ OLD_UUID_SUBTYPE, Binary, DatetimeConversion, - DatetimeMS, DBRef, Decimal128, ObjectId, @@ -115,8 +114,10 @@ def get_client(self): try: client_params = {} - # Return out-of-range dates as DatetimeMS instead of raising. - client_params["datetime_conversion"] = DatetimeConversion.DATETIME_AUTO + # Clamp out-of-range dates to datetime.min/max instead of raising, + # so extreme values stay valid dates and don't break the sync or + # the Elasticsearch date mapping. + client_params["datetime_conversion"] = DatetimeConversion.DATETIME_CLAMP if self.configuration["direct_connection"]: client_params["directConnection"] = True @@ -183,9 +184,6 @@ def _serialize(value): value = value.to_decimal() elif isinstance(value, DBRef): value = _serialize(value.as_doc().to_dict()) - elif isinstance(value, DatetimeMS): - # Keep out-of-range dates as raw milliseconds since epoch. - value = int(value) elif isinstance(value, Binary): # UUID_SUBTYPE is guaranteed to properly be serialized cross-platform and cross-driver if value.subtype == UUID_SUBTYPE: diff --git a/app/connectors_service/tests/sources/test_mongo.py b/app/connectors_service/tests/sources/test_mongo.py index d8f7f0213..c171814ca 100644 --- a/app/connectors_service/tests/sources/test_mongo.py +++ b/app/connectors_service/tests/sources/test_mongo.py @@ -10,8 +10,10 @@ from unittest.mock import AsyncMock, MagicMock, Mock from uuid import UUID +import bson import pytest from bson import Binary, DatetimeConversion, DatetimeMS, DBRef, ObjectId +from bson.codec_options import CodecOptions from bson.decimal128 import Decimal128 from connectors_sdk.filtering.validation import Filter from connectors_sdk.source import ConfigurableFieldValueError @@ -458,33 +460,18 @@ async def test_validate_config_when_configuration_valid_then_does_not_raise( {"some_binary_uuid": None}, ), ( - # in-range datetime still serializes to an ISO string + # in-range datetime serializes to an ISO string {"created": datetime(2020, 1, 1, 12, 30, 0)}, {"created": "2020-01-01T12:30:00"}, ), ( - # out-of-range date decoded as DatetimeMS -> raw milliseconds - {"too_big": DatetimeMS(2**62)}, - {"too_big": 2**62}, + # dates clamped to the datetime bounds serialize to ISO strings too + {"clamped_max": datetime.max}, + {"clamped_max": datetime.max.isoformat()}, ), ( - {"too_small": DatetimeMS(-(2**62))}, - {"too_small": -(2**62)}, - ), - ( - {"epoch": DatetimeMS(0)}, - {"epoch": 0}, - ), - ( - # DatetimeMS nested inside a list and a dict - { - "nested": {"created": DatetimeMS(2**62)}, - "list": [DatetimeMS(0)], - }, - { - "nested": {"created": 2**62}, - "list": [0], - }, + {"clamped_min": datetime.min}, + {"clamped_min": datetime.min.isoformat()}, ), ], ) @@ -494,15 +481,41 @@ async def test_serialize(raw, output): @pytest.mark.asyncio -async def test_get_client_uses_datetime_auto_conversion(): +async def test_get_client_clamps_out_of_range_dates(): async with create_mongo_source() as source: with source.get_client() as client: assert ( client.codec_options.datetime_conversion - == DatetimeConversion.DATETIME_AUTO + == DatetimeConversion.DATETIME_CLAMP ) +@pytest.mark.asyncio +@pytest.mark.parametrize("milliseconds", [2**62, -(2**62), 0]) +async def test_out_of_range_dates_are_clamped_and_serialized_as_iso(milliseconds): + # An out-of-range BSON date (outside datetime's 1-9999 year range) must not + # abort the sync: the client clamps it to a valid datetime, which then + # serializes to an ISO string rather than a raw number. + async with create_mongo_source() as source: + with source.get_client() as client: + codec = client.codec_options + + raw = bson.encode( + {"date": DatetimeMS(milliseconds)}, + codec_options=CodecOptions( + datetime_conversion=DatetimeConversion.DATETIME_MS + ), + ) + decoded = bson.decode(raw, codec_options=codec) + + # Clamping yields a regular datetime, never a DatetimeMS. + assert isinstance(decoded["date"], datetime) + + serialized = source.serialize(dict(decoded)) + assert serialized["date"] == decoded["date"].isoformat() + assert isinstance(serialized["date"], str) + + @pytest.mark.asyncio @pytest.mark.parametrize( "certificate_value, tls_insecure",