Skip to content
Open
40 changes: 32 additions & 8 deletions app/connectors_service/connectors/sources/outlook/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from connectors.sources.outlook.constants import (
API_SCOPE,
CALENDAR_FIELDS,
CONTACT_FIELDS,
CONTACT_FOLDER_FIELDS,
EWS_ENDPOINT,
MAIL_FIELDS,
MAIL_TYPES,
Expand Down Expand Up @@ -226,7 +226,7 @@ async def get_user_accounts(self):
)

async for user in self.get_users():
if "searchResRef" in user["type"]:
if "searchResRef" in user.get("type", ""):
continue

mail = _extract_ldap_mail(user.get("attributes", {}))
Expand Down Expand Up @@ -423,20 +423,42 @@ async def get_mails(self, account):
yield mail, mail_type

async def get_calendars(self, account):
for calendar in await asyncio.to_thread(
account.calendar.all().only, *CALENDAR_FIELDS
):
# Skip instead of aborting if the mailbox has no Calendar folder.
try:
folder = account.calendar

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sanity check - this does not produce network call, folder.all() does, right?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, actually it's the other way around:
account.calendar makes the network call. It is the same as self.root.get_default_folder(Calendar), which calls GetFolder and raises ErrorFolderNotFound.

folder.all() on the other hand is a lazy QuerySet. No network call until it gets iterated, and no ErrorFolderNotFound. So we should be good here.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, then there is a problem here. exchangelib is not async, and the call to account.calendar is gonna block the whole process until it resolves. Then we'd need to wrap it into asyncio.to_thread or something similar

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This then will have to be done to the methods that do network calls below unfortunately

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep. You're right. Done.

except ErrorFolderNotFound:
self._logger.warning(
f"Could not resolve Calendar folder for {account.primary_smtp_address}, skipping."
)
return
for calendar in await asyncio.to_thread(folder.all().only, *CALENDAR_FIELDS):
yield calendar

async def get_child_calendars(self, account):
for child_calendar in account.calendar.children:
try:
child_calendars = account.calendar.children
except ErrorFolderNotFound:
self._logger.warning(
f"Could not resolve Calendar folder for {account.primary_smtp_address}, "
"skipping child calendars."
)
return
for child_calendar in child_calendars:
for calendar in await asyncio.to_thread(
child_calendar.all().only, *CALENDAR_FIELDS
):
yield calendar, child_calendar

async def get_tasks(self, account):
for task in await asyncio.to_thread(account.tasks.all().only, *TASK_FIELDS):
# Skip instead of aborting if the mailbox has no Tasks folder.
try:
folder = account.tasks
except ErrorFolderNotFound:
self._logger.warning(
f"Could not resolve Tasks folder for {account.primary_smtp_address}, skipping."
)
return
for task in await asyncio.to_thread(folder.all().only, *TASK_FIELDS):
yield task

async def get_contacts(self, account):
Expand All @@ -449,5 +471,7 @@ async def get_contacts(self, account):
f"Could not resolve Contacts folder for {account.primary_smtp_address}, skipping."
)
return
for contact in await asyncio.to_thread(folder.all().only, *CONTACT_FIELDS):
for contact in await asyncio.to_thread(
folder.all().only, *CONTACT_FOLDER_FIELDS
):
yield contact
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,13 @@
"company_name",
"birthday",
]
DISTRIBUTION_LIST_FIELDS = [
"last_modified_time",
"display_name",
"members",
]
# Contacts folder holds both item types, so query the union of their fields.
CONTACT_FOLDER_FIELDS = list(dict.fromkeys(CONTACT_FIELDS + DISTRIBUTION_LIST_FIELDS))
TASK_FIELDS = [
"last_modified_time",
"due_date",
Expand Down
133 changes: 112 additions & 21 deletions app/connectors_service/connectors/sources/outlook/datasource.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
iso_utc,
)
from exchangelib.errors import ErrorNonExistentMailbox
from exchangelib.items import DistributionList

from connectors.access_control import ACCESS_CONTROL, es_access_control_query
from connectors.sources.outlook.client import OutlookClient, _extract_ldap_mail
Expand All @@ -34,6 +35,11 @@
)
from connectors.utils import html_to_text

# Per-item errors: a single malformed item is skipped, not fatal. Connection-wide
# failures (transport, SSL, rate limit, missing mailbox) are excluded on purpose
# so they still abort the sync instead of silently emptying the index.
ITEM_SHAPE_ERRORS = (AttributeError, KeyError, ValueError, TypeError)


class OutlookDocFormatter:
"""Format Outlook object documents to Elasticsearch document"""
Expand Down Expand Up @@ -90,11 +96,13 @@ def calendar_doc_formatter(self, calendar, child_calendar, timezone):
}

if child_calendar in ["Folder (Birthdays)", "Birthdays (Birthdays)"]:
# calendar.start may be missing; guard against splitting a None.
birthday = ews_format_to_datetime(
source_datetime=calendar.start, timezone=timezone
)
document.update(
{
"date": ews_format_to_datetime(
source_datetime=calendar.start, timezone=timezone
).split("T", 1)[0],
"date": birthday.split("T", 1)[0] if birthday else None,
}
)
else:
Expand Down Expand Up @@ -168,6 +176,23 @@ def contact_doc_formatter(self, contact, timezone):
),
}

def distribution_list_doc_formatter(self, distribution_list, timezone):
# A contact group has members, not per-contact fields, so it needs its own shape.
return {
"_id": distribution_list.id,
"type": "Distribution List",
"_timestamp": ews_format_to_datetime(
source_datetime=distribution_list.last_modified_time,
timezone=timezone,
),
"name": distribution_list.display_name,
"email_addresses": [
member.mailbox.email_address
for member in (distribution_list.members or [])
if member and member.mailbox and member.mailbox.email_address
],
}

def attachment_doc_formatter(self, attachment, attachment_type, timezone):
attachment_id = (
attachment.attachment_id.id if attachment.attachment_id else None
Expand Down Expand Up @@ -488,6 +513,31 @@ async def download_func(self, content):
"""
yield content

def _format_item(self, formatter, item, account):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This actually has tendency to actually produce syncs that are successful but return 0 items. I'm not really sure we should do this.

We've tried an approach with this class:

Error Monitor swallows errors up to a certain threshold and then raises an exception if too many errors happen. It also has its downsides.

@artem-shelkovnikov artem-shelkovnikov Jul 9, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Continuing the train of thought from above:

I feel this method attempt to help not raise when formatting objects of unexpected types. E.g. you have an API returning types contact and distribution_list and you format them well, but then for some niche case the customer gets an error because an object of type broadcast_group (imaginary name) is returned and the method breaks.

So instead each formatter method can do type checks and warn or error if type is unexpected.

Warn or error? Not sure. It's all about expectation setting to me.

Warns are good because the sync does not crash
Errors are good because they tell what went wrong immediately, so you will know that something is not being synced.

Because of that Error Monitor will not actually help, but will just hide the problem.

All in all I think it's best to do type checks in formatter methods and warn on unexpected types. I am open to a discussion here, though

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. Fixed. Now we're raising instead of silently skipping.

"""Run a document formatter for a single item, isolating item-shape
errors so one malformed Exchange object is skipped (with a warning)
instead of aborting the whole sync.

Args:
formatter (callable): Zero-argument callable that formats the item
(typically a functools.partial around a doc formatter).
item: The Exchange item being formatted, used for logging context.
account: The account the item belongs to, used for logging context.

Returns:
The formatted document, or None if the item was malformed and
skipped. Connection-wide errors are not caught here and propagate.
"""
try:
return formatter()
except ITEM_SHAPE_ERRORS as error:
self._logger.warning(
f"Skipping malformed {type(item).__name__} item "
f"{getattr(item, 'id', 'unknown')} for "
f"{account.primary_smtp_address}: {error}"
)
return None

async def _fetch_attachments(
self, attachment_type, outlook_object, timezone, account
):
Expand All @@ -497,11 +547,18 @@ async def _fetch_attachments(
f"Skipping attachment without an ID on item {outlook_object.id}"
)
continue
document = self.doc_formatter.attachment_doc_formatter(
attachment=attachment,
attachment_type=attachment_type,
timezone=timezone,
document = self._format_item(
partial(
self.doc_formatter.attachment_doc_formatter,
attachment=attachment,
attachment_type=attachment_type,
timezone=timezone,
),
item=attachment,
account=account,
)
if document is None:
continue
yield (
self._decorate_with_access_control(
document, [account.primary_smtp_address]
Expand All @@ -513,11 +570,18 @@ async def _fetch_attachments(

async def _fetch_mails(self, account, timezone):
async for mail, mail_type in self.client.get_mails(account=account):
document = self.doc_formatter.mails_doc_formatter(
mail=mail,
mail_type=mail_type,
timezone=timezone,
document = self._format_item(
partial(
self.doc_formatter.mails_doc_formatter,
mail=mail,
mail_type=mail_type,
timezone=timezone,
),
item=mail,
account=account,
)
if document is None:
continue
yield (
self._decorate_with_access_control(
document, [account.primary_smtp_address]
Expand All @@ -537,10 +601,22 @@ async def _fetch_mails(self, account, timezone):
async def _fetch_contacts(self, account, timezone):
self._logger.debug(f"Fetching contacts for {account.primary_smtp_address}")
async for contact in self.client.get_contacts(account=account):
document = self.doc_formatter.contact_doc_formatter(
contact=contact,
timezone=timezone,
)
# Contacts folder holds both contacts and groups; each has its own shape.
if isinstance(contact, DistributionList):
formatter = partial(
self.doc_formatter.distribution_list_doc_formatter,
distribution_list=contact,
timezone=timezone,
)
else:

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I personally like doing explicit type checks and warn/raise on the case when the type is unexpected.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

formatter = partial(
self.doc_formatter.contact_doc_formatter,
contact=contact,
timezone=timezone,
)
document = self._format_item(formatter, item=contact, account=account)
if document is None:
continue
yield (
self._decorate_with_access_control(
document, [account.primary_smtp_address]
Expand All @@ -551,9 +627,17 @@ async def _fetch_contacts(self, account, timezone):
async def _fetch_tasks(self, account, timezone):
self._logger.debug(f"Fetching tasks for {account.primary_smtp_address}")
async for task in self.client.get_tasks(account=account):
document = self.doc_formatter.task_doc_formatter(
task=task, timezone=timezone
document = self._format_item(
partial(
self.doc_formatter.task_doc_formatter,
task=task,
timezone=timezone,
),
item=task,
account=account,
)
if document is None:
continue
yield (
self._decorate_with_access_control(
document, [account.primary_smtp_address]
Expand Down Expand Up @@ -597,11 +681,18 @@ async def _fetch_child_calendars(self, account, timezone):
yield doc

async def _enqueue_calendars(self, calendar, child_calendar, timezone, account):
document = self.doc_formatter.calendar_doc_formatter(
calendar=calendar,
child_calendar=str(child_calendar),
timezone=timezone,
document = self._format_item(
partial(
self.doc_formatter.calendar_doc_formatter,
calendar=calendar,
child_calendar=str(child_calendar),
timezone=timezone,
),
item=calendar,
account=account,
)
if document is None:
return
yield (
self._decorate_with_access_control(
document, [account.primary_smtp_address]
Expand Down
Loading