diff --git a/app/connectors_service/connectors/sources/mongo/datasource.py b/app/connectors_service/connectors/sources/mongo/datasource.py index 2fc9f70d3..0222f08ca 100644 --- a/app/connectors_service/connectors/sources/mongo/datasource.py +++ b/app/connectors_service/connectors/sources/mongo/datasource.py @@ -10,7 +10,14 @@ from datetime import datetime from tempfile import NamedTemporaryFile -from bson import OLD_UUID_SUBTYPE, Binary, DBRef, Decimal128, ObjectId +from bson import ( + OLD_UUID_SUBTYPE, + Binary, + DatetimeConversion, + DBRef, + Decimal128, + ObjectId, +) from bson.binary import UUID_SUBTYPE from connectors_sdk.source import BaseDataSource, ConfigurableFieldValueError from motor.motor_asyncio import AsyncIOMotorClient @@ -107,6 +114,11 @@ def get_client(self): try: client_params = {} + # Clamp out-of-range dates to datetime.min/max instead of raising, + # so extreme values stay valid dates and don't break the sync or + # the Elasticsearch date mapping. + client_params["datetime_conversion"] = DatetimeConversion.DATETIME_CLAMP + if self.configuration["direct_connection"]: client_params["directConnection"] = True diff --git a/app/connectors_service/tests/sources/test_mongo.py b/app/connectors_service/tests/sources/test_mongo.py index bcf0747ca..c171814ca 100644 --- a/app/connectors_service/tests/sources/test_mongo.py +++ b/app/connectors_service/tests/sources/test_mongo.py @@ -10,8 +10,10 @@ from unittest.mock import AsyncMock, MagicMock, Mock from uuid import UUID +import bson import pytest -from bson import Binary, DBRef, ObjectId +from bson import Binary, DatetimeConversion, DatetimeMS, DBRef, ObjectId +from bson.codec_options import CodecOptions from bson.decimal128 import Decimal128 from connectors_sdk.filtering.validation import Filter from connectors_sdk.source import ConfigurableFieldValueError @@ -457,6 +459,20 @@ async def test_validate_config_when_configuration_valid_then_does_not_raise( }, {"some_binary_uuid": None}, ), + ( + # in-range datetime serializes to an ISO string + {"created": datetime(2020, 1, 1, 12, 30, 0)}, + {"created": "2020-01-01T12:30:00"}, + ), + ( + # dates clamped to the datetime bounds serialize to ISO strings too + {"clamped_max": datetime.max}, + {"clamped_max": datetime.max.isoformat()}, + ), + ( + {"clamped_min": datetime.min}, + {"clamped_min": datetime.min.isoformat()}, + ), ], ) async def test_serialize(raw, output): @@ -464,6 +480,42 @@ async def test_serialize(raw, output): assert source.serialize(raw) == output +@pytest.mark.asyncio +async def test_get_client_clamps_out_of_range_dates(): + async with create_mongo_source() as source: + with source.get_client() as client: + assert ( + client.codec_options.datetime_conversion + == DatetimeConversion.DATETIME_CLAMP + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("milliseconds", [2**62, -(2**62), 0]) +async def test_out_of_range_dates_are_clamped_and_serialized_as_iso(milliseconds): + # An out-of-range BSON date (outside datetime's 1-9999 year range) must not + # abort the sync: the client clamps it to a valid datetime, which then + # serializes to an ISO string rather than a raw number. + async with create_mongo_source() as source: + with source.get_client() as client: + codec = client.codec_options + + raw = bson.encode( + {"date": DatetimeMS(milliseconds)}, + codec_options=CodecOptions( + datetime_conversion=DatetimeConversion.DATETIME_MS + ), + ) + decoded = bson.decode(raw, codec_options=codec) + + # Clamping yields a regular datetime, never a DatetimeMS. + assert isinstance(decoded["date"], datetime) + + serialized = source.serialize(dict(decoded)) + assert serialized["date"] == decoded["date"].isoformat() + assert isinstance(serialized["date"], str) + + @pytest.mark.asyncio @pytest.mark.parametrize( "certificate_value, tls_insecure",