diff --git a/.buildkite/nightly_steps.yml b/.buildkite/nightly_steps.yml index 070c3716b..299a77fa6 100644 --- a/.buildkite/nightly_steps.yml +++ b/.buildkite/nightly_steps.yml @@ -349,3 +349,15 @@ steps: matrix: - "3.10" - "3.11" + - label: "🔨 [Python {{ matrix }}] Google Calendar" + <<: *retries + command: ".buildkite/run_functional_test.sh" + env: + PYTHON_VERSION: "{{ matrix }}" + CONNECTOR: "google_calendar" + DATA_SIZE: "small" + artifact_paths: + - "perf8-report-*/**/*" + matrix: + - "3.10" + - "3.11" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index cce64378b..05503ee3c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -599,6 +599,26 @@ steps: artifact_paths: - "perf8-report-*/**/*" + - path: + - "connectors/sources/google_calendar.py" + - "connectors/sources/google.py" + - "tests/sources/fixtures/google_calendar/**" + - "tests/sources/fixtures/fixture.py" + - "${DOCKERFILE_FTEST_PATH}" + - "requirements/**" + config: + label: "🔨 Google Calendar" + <<: *test-agents + <<: *retries + env: + - PYTHON_VERSION=3.11 + - DATA_SIZE=small + - CONNECTOR=google_calendar + command: + - ".buildkite/run_functional_test.sh" + artifact_paths: + - "perf8-report-*/**/*" + # ---- # DRA publishing # ---- diff --git a/connectors/config.py b/connectors/config.py index 17990a120..c299d4964 100644 --- a/connectors/config.py +++ b/connectors/config.py @@ -119,6 +119,7 @@ def _default_config(): "dropbox": "connectors.sources.dropbox:DropboxDataSource", "github": "connectors.sources.github:GitHubDataSource", "gmail": "connectors.sources.gmail:GMailDataSource", + "google_calendar": "connectors.sources.google_calendar:GoogleCalendarDataSource", "google_cloud_storage": "connectors.sources.google_cloud_storage:GoogleCloudStorageDataSource", "google_drive": "connectors.sources.google_drive:GoogleDriveDataSource", "graphql": "connectors.sources.graphql:GraphQLDataSource", diff --git a/connectors/sources/google_calendar.py b/connectors/sources/google_calendar.py new file mode 100644 index 000000000..dd4f1bab5 --- /dev/null +++ b/connectors/sources/google_calendar.py @@ -0,0 +1,399 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; +# you may not use this file except in compliance with the Elastic License 2.0. +# +from datetime import datetime, timedelta + +from connectors.source import BaseDataSource, ConfigurableFieldValueError +from connectors.sources.google import ( + GoogleServiceAccountClient, + load_service_account_json, + remove_universe_domain, + validate_service_account_json, +) + +# Default timeout for Google Calendar API calls (in seconds) +DEFAULT_TIMEOUT = 60 + +# Calendar API scopes +CALENDAR_READONLY_SCOPE = "https://www.googleapis.com/auth/calendar.readonly" + + +class GoogleCalendarClient(GoogleServiceAccountClient): + """A Google Calendar client to handle API calls to the Google Calendar API.""" + + def __init__(self, json_credentials, subject=None): + """Initialize the GoogleCalendarClient. + + Args: + json_credentials (dict): Service account credentials JSON. + subject (str, optional): Email of the user to impersonate. Defaults to None. + """ + remove_universe_domain(json_credentials) + if subject: + json_credentials["subject"] = subject + + super().__init__( + json_credentials=json_credentials, + api="calendar", + api_version="v3", + scopes=[CALENDAR_READONLY_SCOPE], + api_timeout=DEFAULT_TIMEOUT, + ) + + async def ping(self): + """Verify connectivity to Google Calendar API.""" + return await self.api_call(resource="calendarList", method="list", maxResults=1) + + async def list_calendar_list(self): + """Fetch all calendar list entries from Google Calendar. + + Yields: + dict: Calendar list entry. + """ + async for page in self.api_call_paged( + resource="calendarList", + method="list", + maxResults=100, + ): + yield page + + async def get_calendar(self, calendar_id): + """Get a specific calendar by ID. + + Args: + calendar_id (str): The calendar ID. + + Returns: + dict: Calendar resource. + """ + return await self.api_call( + resource="calendars", method="get", calendarId=calendar_id + ) + + async def list_events(self, calendar_id, time_min=None, time_max=None): + """Fetch all events from a specific calendar within the specified time range. + + Args: + calendar_id (str): The calendar ID. + time_min (str): Start time in ISO format. Optional. + time_max (str): End time in ISO format. Optional. + + Yields: + dict: Events page. + """ + params = { + "calendarId": calendar_id, + "maxResults": 100, + } + if time_min: + params["timeMin"] = time_min + if time_max: + params["timeMax"] = time_max + + async for page in self.api_call_paged( + resource="events", method="list", **params + ): + yield page + + +class GoogleCalendarDataSource(BaseDataSource): + """Google Calendar connector for Elastic Enterprise Search. + + This connector fetches data from a user's Google Calendar (read-only mode): + - CalendarList entries (the user's list of calendars) + - Each underlying Calendar resource + - Events belonging to each Calendar + + Reference: + https://developers.google.com/calendar/api/v3/reference + """ + + name = "Google Calendar" + service_type = "google_calendar" + + def __init__(self, configuration): + """Initialize the GoogleCalendarDataSource. + + Args: + configuration (DataSourceConfiguration): Object of DataSourceConfiguration class. + """ + super().__init__(configuration=configuration) + self._calendar_client = None + + @classmethod + def get_default_configuration(cls): + """Returns a dict with a default configuration for the connector.""" + return { + "service_account_credentials": { + "display": "textarea", + "label": "Google Calendar service account JSON", + "order": 1, + "required": True, + "sensitive": True, + "type": "str", + }, + "subject": { + "display": "text", + "label": "User email to impersonate", + "order": 2, + "required": True, + "type": "str", + }, + "days_back": { + "display": "numeric", + "label": "Days back to fetch events", + "order": 3, + "type": "int", + "value": 30, + "validations": [{"type": "greater_than", "constraint": -1}], + }, + "days_forward": { + "display": "numeric", + "label": "Days forward to fetch events", + "order": 4, + "type": "int", + "value": 30, + "validations": [{"type": "greater_than", "constraint": -1}], + }, + } + + def _set_internal_logger(self): + """Set the logger for internal components.""" + if self._calendar_client: + self._calendar_client.set_logger(self._logger) + + @property + def _service_account_credentials(self): + """Load and return the service account credentials. + + Returns: + dict: The loaded service account credentials. + """ + service_account_credentials = self.configuration["service_account_credentials"] + return load_service_account_json(service_account_credentials, "Google Calendar") + + def calendar_client(self): + """Get or create a Google Calendar client. + + Returns: + GoogleCalendarClient: An initialized Google Calendar client. + """ + if not self._calendar_client: + self._calendar_client = GoogleCalendarClient( + json_credentials=self._service_account_credentials, + subject=self.configuration["subject"], + ) + self._calendar_client.set_logger(self._logger) + return self._calendar_client + + async def validate_config(self): + """Validate the configuration. + + Raises: + ConfigurableFieldValueError: If the configuration is invalid. + """ + await super().validate_config() + validate_service_account_json( + self.configuration["service_account_credentials"], "Google Calendar" + ) + + # Verify connectivity to Google Calendar API + try: + await self.ping() + except Exception as e: + msg = f"Google Calendar authentication failed. Please check your service account credentials and subject email. Error: {str(e)}" + raise ConfigurableFieldValueError(msg) from e + + async def ping(self): + """Verify connectivity to Google Calendar API.""" + await self.calendar_client().ping() + + async def get_docs(self, filtering=None): + """Yields documents from Google Calendar API. + + Each document is a tuple with: + - a mapping with the data to index + - an optional mapping with attachment data + """ + client = self.calendar_client() + + # 1) Get the user's calendarList + calendar_list_entries = [] + async for cal_list_doc in self._generate_calendar_list_docs(client): + yield cal_list_doc, None + calendar_list_entries.append(cal_list_doc) + + # 2) For each calendar in the user's calendarList, yield its Calendar resource + for cal_list_doc in calendar_list_entries: + async for calendar_doc in self._generate_calendar_docs( + client, cal_list_doc["calendar_id"] + ): + yield calendar_doc, None + + # 3) For each calendar, yield event documents + for cal_list_doc in calendar_list_entries: + async for event_doc in self._generate_event_docs(client, cal_list_doc): + yield event_doc, None + + async def _generate_calendar_list_docs(self, client): + """Yield documents for each calendar in the user's CalendarList. + + Args: + client (GoogleCalendarClient): The Google Calendar client. + + Yields: + dict: Calendar list entry document. + """ + async for page in client.list_calendar_list(): + items = page.get("items", []) + for cal in items: + doc = { + "_id": cal["id"], + "type": "calendar_list", + "calendar_id": cal["id"], + "summary": cal.get("summary"), + "summary_override": cal.get("summaryOverride"), + "color_id": cal.get("colorId"), + "background_color": cal.get("backgroundColor"), + "foreground_color": cal.get("foregroundColor"), + "hidden": cal.get("hidden", False), + "selected": cal.get("selected", False), + "access_role": cal.get("accessRole"), + "primary": cal.get("primary", False), + "deleted": cal.get("deleted", False), + } + yield doc + + async def _generate_calendar_docs(self, client, calendar_id): + """Yield a document for the specified calendar_id. + + Args: + client (GoogleCalendarClient): The Google Calendar client. + calendar_id (str): The calendar ID. + + Yields: + dict: Calendar document. + """ + try: + data = await client.get_calendar(calendar_id) + doc = { + "_id": data["id"], + "type": "calendar", + "calendar_id": data["id"], + "summary": data.get("summary"), + "description": data.get("description"), + "location": data.get("location"), + "time_zone": data.get("timeZone"), + } + yield doc + except Exception as e: + self._logger.warning(f"Error fetching calendar {calendar_id}: {str(e)}") + + async def _generate_event_docs(self, client, cal_list_doc): + """Yield documents for all events in the given calendar. + + Args: + client (GoogleCalendarClient): The Google Calendar client. + cal_list_doc (dict): Calendar list entry document. + + Yields: + dict: Event document. + """ + calendar_id = cal_list_doc["calendar_id"] + + # Calculate time range based on configuration + now = datetime.utcnow() + days_back = self.configuration.get("days_back", 30) + days_forward = self.configuration.get("days_forward", 30) + + time_min = (now - timedelta(days=days_back)).isoformat() + "Z" + time_max = (now + timedelta(days=days_forward)).isoformat() + "Z" + + # Create calendar reference for events + calendar_ref = { + "id": calendar_id, + "name": cal_list_doc.get("summary_override") + or cal_list_doc.get("summary") + or "", + "type": "calendar", + } + + try: + async for page in client.list_events(calendar_id, time_min, time_max): + for event in page.get("items", []): + event_id = event["id"] + # Extract date/time fields + start_info = event.get("start", {}) + end_info = event.get("end", {}) + start_datetime = start_info.get("dateTime") + start_date = start_info.get("date") + end_datetime = end_info.get("dateTime") + end_date = end_info.get("date") + + # Extract attendee names and emails + attendees_info = [] + if event.get("attendees"): + for attendee in event.get("attendees", []): + attendee_info = {} + if attendee.get("displayName"): + attendee_info["name"] = attendee.get("displayName") + if attendee.get("email"): + attendee_info["email"] = attendee.get("email") + if attendee_info: + attendees_info.append(attendee_info) + + # Extract meeting/zoom links from conferenceData or location + meeting_link = None + if event.get("conferenceData"): + entry_points = event.get("conferenceData", {}).get( + "entryPoints", [] + ) + for entry_point in entry_points: + if entry_point.get("uri"): + meeting_link = entry_point.get("uri") + break + + # Extract attachments + attachments_info = [] + if event.get("attachments"): + for attachment in event.get("attachments", []): + attachment_info = {} + if attachment.get("title"): + attachment_info["title"] = attachment.get("title") + if attachment.get("fileUrl"): + attachment_info["url"] = attachment.get("fileUrl") + if attachment.get("mimeType"): + attachment_info["mime_type"] = attachment.get( + "mimeType" + ) + if attachment_info: + attachments_info.append(attachment_info) + + doc = { + "_id": event_id, + "type": "event", + "event_id": event_id, + "calendar_id": calendar_id, + "calendar": calendar_ref, + "summary": event.get("summary"), + "description": event.get("description"), + "location": event.get("location"), + "meeting_link": meeting_link, + "start_datetime": start_datetime, + "start_date": start_date, + "end_datetime": end_datetime, + "end_date": end_date, + "attendees": attendees_info, + "attachments": attachments_info, + } + yield doc + except Exception as e: + self._logger.warning( + f"Error fetching events for calendar {calendar_id}: {str(e)}" + ) + + async def close(self): + """Close any resources.""" + pass diff --git a/docs/GOOGLE_CALENDAR_AUTH.md b/docs/GOOGLE_CALENDAR_AUTH.md new file mode 100644 index 000000000..661a8509a --- /dev/null +++ b/docs/GOOGLE_CALENDAR_AUTH.md @@ -0,0 +1,114 @@ +# Google Calendar Authentication Guide + +This guide explains how to set up service account authentication for the Google Calendar connector. + +## Prerequisites + +- A Google account with administrative privileges for your Google Workspace domain +- Access to [Google Cloud Console](https://console.cloud.google.com/) + +## Step 1: Create a Google Cloud Project + +1. Go to the [Google Cloud Console](https://console.cloud.google.com/) +2. Click on the project dropdown at the top of the page +3. Click on "New Project" +4. Enter a name for your project and click "Create" +5. Wait for the project to be created and then select it from the project dropdown + +## Step 2: Enable the Google Calendar API + +1. In your Google Cloud project, navigate to "APIs & Services" > "Library" +2. Search for "Google Calendar API" +3. Click on the Google Calendar API card +4. Click "Enable" + +## Step 3: Create a Service Account + +1. Navigate to "APIs & Services" > "Credentials" +2. Click "Create Credentials" and select "Service Account" +3. Enter a name for your service account +4. (Optional) Add a description +5. Click "Create and Continue" +6. In the "Grant this service account access to project" section, you can skip this step by clicking "Continue" +7. In the "Grant users access to this service account" section, you can skip this step by clicking "Done" +8. Your service account has now been created + +## Step 4: Create a Service Account Key + +1. In the "Service Accounts" section, click on the service account you just created +2. Click on the "Keys" tab +3. Click "Add Key" and select "Create new key" +4. Select "JSON" as the key type +5. Click "Create" +6. The JSON key file will be downloaded to your computer +7. Keep this file secure, as it contains sensitive information + +## Step 5: Set Up Domain-Wide Delegation + +To allow the service account to access user data in your Google Workspace domain, you need to set up domain-wide delegation: + +1. In the Google Cloud Console, navigate to "APIs & Services" > "Credentials" +2. Click on the service account you created +3. Click on the "Details" tab +4. Scroll down to "Show domain-wide delegation" +5. Enable "Domain-wide delegation" +6. Save the changes + +Next, you need to authorize the service account in your Google Workspace Admin Console: + +1. Go to your [Google Workspace Admin Console](https://admin.google.com/) +2. Navigate to "Security" > "API controls" +3. In the "Domain-wide delegation" section, click "Manage Domain Wide Delegation" +4. Click "Add new" +5. Enter the Client ID of your service account (this is a numeric ID found in the service account details) +6. In the "OAuth scopes" field, enter: `https://www.googleapis.com/auth/calendar.readonly` +7. Click "Authorize" + +## Step 6: Configure the Connector + +To use the service account with the Google Calendar connector, you need to provide: + +1. The service account JSON key file +2. The email address of a user to impersonate + +Configure the connector with the following settings: + +```yaml +service_account_credentials: | + { + "type": "service_account", + "project_id": "your-project-id", + "private_key_id": "your-private-key-id", + "private_key": "your-private-key", + "client_email": "your-service-account-email", + "client_id": "your-client-id", + ... + } +subject: "user@yourdomain.com" # Email address of the user to impersonate +include_freebusy: false # Set to true if you want to include free/busy data +``` + +Replace the `service_account_credentials` with the contents of your JSON key file, and `subject` with the email address of the user whose calendars you want to access. + +## Important Notes + +1. **Security**: + - Keep your service account key file secure + - Do not commit it to version control + - Consider using environment variables or secure vaults to store the credentials + +2. **Scopes**: + - The Google Calendar connector requires the `https://www.googleapis.com/auth/calendar.readonly` scope + - If you need additional permissions, add the appropriate scopes in the Google Workspace Admin Console + +3. **User Impersonation**: + - The service account will impersonate the user specified in the `subject` field + - This user must be a member of your Google Workspace domain + - The connector will only be able to access calendars that this user has permission to view + +## Troubleshooting + +- **Authentication Failed**: Ensure that the service account has been properly set up with domain-wide delegation and the correct scopes. +- **Access Denied**: Check that the impersonated user has access to the calendars you're trying to fetch. +- **Invalid Service Account JSON**: Verify that the service account JSON is correctly formatted and contains all required fields. +- **Missing Scopes**: Ensure that the service account has been granted the necessary OAuth scopes in the Google Workspace Admin Console. \ No newline at end of file diff --git a/tests/sources/fixtures/google_calendar/.env b/tests/sources/fixtures/google_calendar/.env new file mode 100644 index 000000000..73eb3baad --- /dev/null +++ b/tests/sources/fixtures/google_calendar/.env @@ -0,0 +1 @@ +GOOGLE_API_FTEST_HOST=http://localhost:10340 \ No newline at end of file diff --git a/tests/sources/fixtures/google_calendar/config.yml b/tests/sources/fixtures/google_calendar/config.yml new file mode 100644 index 000000000..8ded6aa5a --- /dev/null +++ b/tests/sources/fixtures/google_calendar/config.yml @@ -0,0 +1,6 @@ +service.idling: 1 + +connectors: + - + connector_id: 'google_calendar' + service_type: 'google_calendar' \ No newline at end of file diff --git a/tests/sources/fixtures/google_calendar/connector.json b/tests/sources/fixtures/google_calendar/connector.json new file mode 100644 index 000000000..9798e7ba6 --- /dev/null +++ b/tests/sources/fixtures/google_calendar/connector.json @@ -0,0 +1,74 @@ +{ + "configuration": { + "service_account_credentials": { + "depends_on": [], + "display": "textarea", + "tooltip": null, + "default_value": null, + "label": "Google Calendar service account JSON", + "sensitive": true, + "type": "str", + "required": true, + "options": [], + "validations": [], + "value": "{\"type\":\"service_account\",\"project_id\":\"project_id\",\"private_key_id\":\"abc\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDY3E8o1NEFcjMM\\nHW/5ZfFJw29/8NEqpViNjQIx95Xx5KDtJ+nWn9+OW0uqsSqKlKGhAdAo+Q6bjx2c\\nuXVsXTu7XrZUY5Kltvj94DvUa1wjNXs606r/RxWTJ58bfdC+gLLxBfGnB6CwK0YQ\\nxnfpjNbkUfVVzO0MQD7UP0Hl5ZcY0Puvxd/yHuONQn/rIAieTHH1pqgW+zrH/y3c\\n59IGThC9PPtugI9ea8RSnVj3PWz1bX2UkCDpy9IRh9LzJLaYYX9RUd7++dULUlat\\nAaXBh1U6emUDzhrIsgApjDVtimOPbmQWmX1S60mqQikRpVYZ8u+NDD+LNw+/Eovn\\nxCj2Y3z1AgMBAAECggEAWDBzoqO1IvVXjBA2lqId10T6hXmN3j1ifyH+aAqK+FVl\\nGjyWjDj0xWQcJ9ync7bQ6fSeTeNGzP0M6kzDU1+w6FgyZqwdmXWI2VmEizRjwk+/\\n/uLQUcL7I55Dxn7KUoZs/rZPmQDxmGLoue60Gg6z3yLzVcKiDc7cnhzhdBgDc8vd\\nQorNAlqGPRnm3EqKQ6VQp6fyQmCAxrr45kspRXNLddat3AMsuqImDkqGKBmF3Q1y\\nxWGe81LphUiRqvqbyUlh6cdSZ8pLBpc9m0c3qWPKs9paqBIvgUPlvOZMqec6x4S6\\nChbdkkTRLnbsRr0Yg/nDeEPlkhRBhasXpxpMUBgPywKBgQDs2axNkFjbU94uXvd5\\nznUhDVxPFBuxyUHtsJNqW4p/ujLNimGet5E/YthCnQeC2P3Ym7c3fiz68amM6hiA\\nOnW7HYPZ+jKFnefpAtjyOOs46AkftEg07T9XjwWNPt8+8l0DYawPoJgbM5iE0L2O\\nx8TU1Vs4mXc+ql9F90GzI0x3VwKBgQDqZOOqWw3hTnNT07Ixqnmd3dugV9S7eW6o\\nU9OoUgJB4rYTpG+yFqNqbRT8bkx37iKBMEReppqonOqGm4wtuRR6LSLlgcIU9Iwx\\nyfH12UWqVmFSHsgZFqM/cK3wGev38h1WBIOx3/djKn7BdlKVh8kWyx6uC8bmV+E6\\nOoK0vJD6kwKBgHAySOnROBZlqzkiKW8c+uU2VATtzJSydrWm0J4wUPJifNBa/hVW\\ndcqmAzXC9xznt5AVa3wxHBOfyKaE+ig8CSsjNyNZ3vbmr0X04FoV1m91k2TeXNod\\njMTobkPThaNm4eLJMN2SQJuaHGTGERWC0l3T18t+/zrDMDCPiSLX1NAvAoGBAN1T\\nVLJYdjvIMxf1bm59VYcepbK7HLHFkRq6xMJMZbtG0ryraZjUzYvB4q4VjHk2UDiC\\nlhx13tXWDZH7MJtABzjyg+AI7XWSEQs2cBXACos0M4Myc6lU+eL+iA+OuoUOhmrh\\nqmT8YYGu76/IBWUSqWuvcpHPpwl7871i4Ga/I3qnAoGBANNkKAcMoeAbJQK7a/Rn\\nwPEJB+dPgNDIaboAsh1nZhVhN5cvdvCWuEYgOGCPQLYQF0zmTLcM+sVxOYgfy8mV\\nfbNgPgsP5xmu6dw2COBKdtozw0HrWSRjACd1N4yGu75+wPCcX/gQarcjRcXXZeEa\\nNtBLSfcqPULqD+h7br9lEJio\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"123-abc@developer.gserviceaccount.com\",\"client_id\":\"123-abc.apps.googleusercontent.com\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"http://localhost:10340/token\"}", + "order": 1, + "ui_restrictions": [] + }, + "subject": { + "depends_on": [], + "display": "text", + "tooltip": null, + "default_value": null, + "label": "User email to impersonate", + "sensitive": false, + "type": "str", + "required": true, + "options": [], + "validations": [], + "value": "test@example.com", + "order": 2, + "ui_restrictions": [] + }, + "days_back": { + "depends_on": [], + "display": "numeric", + "tooltip": null, + "default_value": null, + "label": "Days back to fetch events", + "sensitive": false, + "type": "int", + "required": false, + "options": [], + "validations": [ + { + "constraint": -1, + "type": "greater_than" + } + ], + "value": 30, + "order": 3, + "ui_restrictions": [] + }, + "days_forward": { + "depends_on": [], + "display": "numeric", + "tooltip": null, + "default_value": null, + "label": "Days forward to fetch events", + "sensitive": false, + "type": "int", + "required": false, + "options": [], + "validations": [ + { + "constraint": -1, + "type": "greater_than" + } + ], + "value": 30, + "order": 4, + "ui_restrictions": [] + } + } +} \ No newline at end of file diff --git a/tests/sources/fixtures/google_calendar/docker-compose.yml b/tests/sources/fixtures/google_calendar/docker-compose.yml new file mode 100644 index 000000000..f50d06892 --- /dev/null +++ b/tests/sources/fixtures/google_calendar/docker-compose.yml @@ -0,0 +1,46 @@ +version: '3.9' + +services: + elasticsearch: + image: ${ELASTICSEARCH_DRA_DOCKER_IMAGE} + container_name: elasticsearch + environment: + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - ES_JAVA_OPTS=-Xms2g -Xmx2g + - ELASTIC_PASSWORD=changeme + - xpack.security.enabled=true + - xpack.security.authc.api_key.enabled=true + - discovery.type=single-node + - action.destructive_requires_name=false + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - esdata:/usr/share/elasticsearch/data + ports: + - 9200:9200 + networks: + - esnet + + google_calendar: + build: + context: ../../../../ + dockerfile: ${DOCKERFILE_FTEST_PATH} + command: .venv/bin/python tests/sources/fixtures/google_calendar/fixture.py + ports: + - "10340:10340" + volumes: + - .:/python-flask + restart: always + environment: + - DATA_SIZE=${DATA_SIZE} + +volumes: + esdata: + driver: local + +networks: + esnet: + driver: bridge \ No newline at end of file diff --git a/tests/sources/fixtures/google_calendar/fixture.py b/tests/sources/fixtures/google_calendar/fixture.py new file mode 100644 index 000000000..7e7de5660 --- /dev/null +++ b/tests/sources/fixtures/google_calendar/fixture.py @@ -0,0 +1,345 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; +# you may not use this file except in compliance with the Elastic License 2.0. +# +# ruff: noqa: T201 +"""Module to handle API calls received from Google Calendar connector.""" + +import os +import time +from datetime import datetime, timedelta + +from flask import Flask, request + +from tests.commons import WeightedFakeProvider + +fake_provider = WeightedFakeProvider() + +DATA_SIZE = os.environ.get("DATA_SIZE", "medium").lower() + +match DATA_SIZE: + case "small": + CALENDARS_COUNT = 5 + EVENTS_PER_CALENDAR = 50 + case "medium": + CALENDARS_COUNT = 10 + EVENTS_PER_CALENDAR = 100 + case "large": + CALENDARS_COUNT = 20 + EVENTS_PER_CALENDAR = 250 + +# Total docs = CALENDARS_COUNT (calendar_list) + CALENDARS_COUNT (calendar) + (CALENDARS_COUNT * EVENTS_PER_CALENDAR) +DOCS_COUNT = CALENDARS_COUNT + CALENDARS_COUNT + (CALENDARS_COUNT * EVENTS_PER_CALENDAR) + +app = Flask(__name__) + +PRE_REQUEST_SLEEP = float(os.environ.get("PRE_REQUEST_SLEEP", "0.1")) + + +def get_num_docs(): + print(DOCS_COUNT) + + +@app.before_request +def before_request(): + time.sleep(PRE_REQUEST_SLEEP) + + +# Mock OAuth2 token endpoint +@app.route("/token", methods=["POST"]) +def post_auth_token(): + """OAuth2 token endpoint mock.""" + return { + "access_token": "ya29.a0AfH6SMBxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "token_type": "Bearer", + "expires_in": 3600, + "refresh_token": "1//0xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + } + + +# Calendar API endpoints +@app.route("/calendar/v3/users/me/calendarList", methods=["GET"]) +def calendar_list(): + """Mock calendar list endpoint.""" + calendars = [] + for i in range(CALENDARS_COUNT): + calendar_id = f"cal_{i}@example.com" # Simple calendar ID + calendars.append({ + "kind": "calendar#calendarListEntry", + "etag": f"\"etag_{i}\"", + "id": calendar_id, + "summary": f"Calendar {i}", + "description": f"Test calendar {i} for functional testing", + "timeZone": "America/Los_Angeles", + "colorId": str((i % 24) + 1), # Google Calendar has 24 color options + "backgroundColor": f"#{'%06x' % (hash(str(i)) & 0xFFFFFF)}", + "foregroundColor": "#000000", + "selected": True, + "accessRole": "owner" if i == 0 else "reader", + "primary": i == 0, + "defaultReminders": [ + {"method": "popup", "minutes": 30} + ] + }) + + return { + "kind": "calendar#calendarList", + "etag": "\"calendar_list_etag\"", + "items": calendars + } + + +@app.route("/calendar/v3/calendars/", methods=["GET"]) +def get_calendar(calendar_id): + """Mock get calendar endpoint.""" + # Extract index from calendar_id (e.g., "cal_0@example.com" -> 0) + try: + index = int(calendar_id.split("_")[-1].split("@")[0]) + except (IndexError, ValueError): + index = 0 + + # Return the same calendar ID as the calendar list entry + # The _id collision will be handled by document type differentiation + return { + "kind": "calendar#calendar", + "etag": f"\"calendar_etag_{index}\"", + "id": calendar_id, + "summary": f"Calendar {index}", + "description": f"Test calendar {index} for functional testing", + "location": f"Building {index}, Test City", + "timeZone": "America/Los_Angeles" + } + + +@app.route("/calendar/v3/calendars//events", methods=["GET"]) +def list_events(calendar_id): + """Mock events list endpoint.""" + # Extract index from calendar_id (e.g., "cal_0@example.com" -> 0) + try: + calendar_index = int(calendar_id.split("_")[-1].split("@")[0]) + except (IndexError, ValueError): + calendar_index = 0 + + # Get query parameters + time_min = request.args.get("timeMin") + time_max = request.args.get("timeMax") + max_results = int(request.args.get("maxResults", 100)) + + # Generate events for this calendar + events = [] + now = datetime.utcnow() + + # Parse time range if provided by connector + if time_min and time_max: + try: + start_time = datetime.fromisoformat(time_min.replace('Z', '+00:00')) + end_time = datetime.fromisoformat(time_max.replace('Z', '+00:00')) + time_range_days = (end_time - start_time).days + except: + start_time = now - timedelta(days=30) + end_time = now + timedelta(days=30) + time_range_days = 60 + else: + start_time = now - timedelta(days=30) + end_time = now + timedelta(days=30) + time_range_days = 60 + + for i in range(min(EVENTS_PER_CALENDAR, max_results)): + event_id = f"event_{calendar_index}_{i}" + # Spread events evenly across the requested time range + if EVENTS_PER_CALENDAR > 1: + days_offset = (i / (EVENTS_PER_CALENDAR - 1)) * time_range_days + event_start = start_time + timedelta(days=days_offset, hours=i % 24) + else: + event_start = start_time + timedelta(hours=i % 24) + event_end = event_start + timedelta(hours=1) + + # Create some variety in event types + event_type = i % 4 + + if event_type == 0: + # All-day event + event = { + "kind": "calendar#event", + "etag": f"\"event_etag_{calendar_index}_{i}\"", + "id": event_id, + "status": "confirmed", + "htmlLink": f"https://calendar.google.com/event?eid={event_id}", + "created": (now - timedelta(days=30)).isoformat() + "Z", + "updated": (now - timedelta(days=1)).isoformat() + "Z", + "summary": f"All-day Event {i} in Calendar {calendar_index}", + "description": f"This is a test all-day event {i} for functional testing", + "location": f"Conference Room {i}", + "creator": { + "email": "test@example.com", + "displayName": "Test User" + }, + "organizer": { + "email": "test@example.com", + "displayName": "Test User" + }, + "start": { + "date": event_start.date().isoformat(), + "timeZone": "America/Los_Angeles" + }, + "end": { + "date": (event_start + timedelta(days=1)).date().isoformat(), + "timeZone": "America/Los_Angeles" + }, + "transparency": "transparent", + "visibility": "default" + } + elif event_type == 1: + # Meeting with attendees and conference data + event = { + "kind": "calendar#event", + "etag": f"\"event_etag_{calendar_index}_{i}\"", + "id": event_id, + "status": "confirmed", + "htmlLink": f"https://calendar.google.com/event?eid={event_id}", + "created": (now - timedelta(days=30)).isoformat() + "Z", + "updated": (now - timedelta(days=1)).isoformat() + "Z", + "summary": f"Team Meeting {i} - Calendar {calendar_index}", + "description": f"This is a test meeting {i} for functional testing with agenda items", + "location": f"Zoom Meeting Room {i}", + "creator": { + "email": "test@example.com", + "displayName": "Test User" + }, + "organizer": { + "email": "test@example.com", + "displayName": "Test User" + }, + "start": { + "dateTime": event_start.isoformat() + "Z", + "timeZone": "America/Los_Angeles" + }, + "end": { + "dateTime": event_end.isoformat() + "Z", + "timeZone": "America/Los_Angeles" + }, + "attendees": [ + { + "email": f"attendee1_{i}@example.com", + "displayName": f"Attendee One {i}", + "responseStatus": "accepted" + }, + { + "email": f"attendee2_{i}@example.com", + "displayName": f"Attendee Two {i}", + "responseStatus": "tentative" + } + ], + "conferenceData": { + "entryPoints": [ + { + "entryPointType": "video", + "uri": f"https://zoom.us/j/{1234567890 + i}", + "label": f"zoom.us/j/{1234567890 + i}" + } + ], + "conferenceId": f"zoom-meeting-{i}", + "signature": f"signature_{i}" + }, + "reminders": { + "useDefault": False, + "overrides": [ + {"method": "email", "minutes": 1440}, + {"method": "popup", "minutes": 30} + ] + } + } + elif event_type == 2: + # Event with attachments + event = { + "kind": "calendar#event", + "etag": f"\"event_etag_{calendar_index}_{i}\"", + "id": event_id, + "status": "confirmed", + "htmlLink": f"https://calendar.google.com/event?eid={event_id}", + "created": (now - timedelta(days=30)).isoformat() + "Z", + "updated": (now - timedelta(days=1)).isoformat() + "Z", + "summary": f"Document Review {i} - Calendar {calendar_index}", + "description": f"Review session {i} with attached documents for functional testing", + "location": f"Office Building {i}", + "creator": { + "email": "test@example.com", + "displayName": "Test User" + }, + "organizer": { + "email": "test@example.com", + "displayName": "Test User" + }, + "start": { + "dateTime": event_start.isoformat() + "Z", + "timeZone": "America/Los_Angeles" + }, + "end": { + "dateTime": event_end.isoformat() + "Z", + "timeZone": "America/Los_Angeles" + }, + "attachments": [ + { + "fileId": f"file_id_{i}_1", + "title": f"Document {i}_1.pdf", + "mimeType": "application/pdf", + "fileUrl": f"https://drive.google.com/file/d/file_id_{i}_1/view" + }, + { + "fileId": f"file_id_{i}_2", + "title": f"Presentation {i}_2.pptx", + "mimeType": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "fileUrl": f"https://drive.google.com/file/d/file_id_{i}_2/view" + } + ] + } + else: + # Simple event + event = { + "kind": "calendar#event", + "etag": f"\"event_etag_{calendar_index}_{i}\"", + "id": event_id, + "status": "confirmed", + "htmlLink": f"https://calendar.google.com/event?eid={event_id}", + "created": (now - timedelta(days=30)).isoformat() + "Z", + "updated": (now - timedelta(days=1)).isoformat() + "Z", + "summary": f"Simple Event {i} - Calendar {calendar_index}", + "description": f"This is a simple test event {i} for functional testing", + "creator": { + "email": "test@example.com", + "displayName": "Test User" + }, + "organizer": { + "email": "test@example.com", + "displayName": "Test User" + }, + "start": { + "dateTime": event_start.isoformat() + "Z", + "timeZone": "America/Los_Angeles" + }, + "end": { + "dateTime": event_end.isoformat() + "Z", + "timeZone": "America/Los_Angeles" + } + } + + events.append(event) + + return { + "kind": "calendar#events", + "etag": f"\"events_etag_{calendar_index}\"", + "summary": f"Calendar {calendar_index}", + "description": f"Test calendar {calendar_index} events", + "updated": now.isoformat() + "Z", + "timeZone": "America/Los_Angeles", + "accessRole": "owner", + "defaultReminders": [ + {"method": "popup", "minutes": 30} + ], + "items": events + } + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=10340, debug=True) \ No newline at end of file diff --git a/tests/sources/test_google_calendar.py b/tests/sources/test_google_calendar.py new file mode 100644 index 000000000..085cc3c1f --- /dev/null +++ b/tests/sources/test_google_calendar.py @@ -0,0 +1,360 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; +# you may not use this file except in compliance with the Elastic License 2.0. +# +"""Tests the Google Calendar source class methods.""" + +import asyncio +from contextlib import asynccontextmanager +from unittest import mock +from unittest.mock import patch + +import pytest +from aiogoogle import Aiogoogle + +from connectors.source import ConfigurableFieldValueError, DataSourceConfiguration +from connectors.sources.google_calendar import ( + GoogleCalendarClient, + GoogleCalendarDataSource, +) +from tests.commons import AsyncIterator +from tests.sources.support import create_source + +SERVICE_ACCOUNT_CREDENTIALS = '{"project_id": "dummy123"}' + + +@asynccontextmanager +async def create_gcal_source(**kwargs): + """Create a Google Calendar source for testing""" + async with create_source( + GoogleCalendarDataSource, + service_account_credentials=SERVICE_ACCOUNT_CREDENTIALS, + subject="test@example.com", + **kwargs, + ) as source: + yield source + + +@pytest.mark.asyncio +async def test_empty_configuration(): + """Tests the validity of the configurations passed to the Google Calendar source class.""" + + configuration = DataSourceConfiguration({"service_account_credentials": ""}) + gcal_object = GoogleCalendarDataSource(configuration=configuration) + + with pytest.raises( + ConfigurableFieldValueError, + match="Field validation errors: 'Service_account_credentials' cannot be empty.", + ): + await gcal_object.validate_config() + + +@pytest.mark.asyncio +async def test_raise_on_invalid_configuration(): + """Test if invalid configuration raises an expected Exception""" + + configuration = DataSourceConfiguration( + {"service_account_credentials": "{'abc':'bcd','cd'}"} + ) + gcal_object = GoogleCalendarDataSource(configuration=configuration) + + with pytest.raises( + ConfigurableFieldValueError, + match="Google Calendar service account is not a valid JSON", + ): + await gcal_object.validate_config() + + +@pytest.mark.asyncio +async def test_get_default_configuration(): + """Test the default configuration for Google Calendar connector""" + config = GoogleCalendarDataSource.get_default_configuration() + + assert "service_account_credentials" in config + assert config["service_account_credentials"]["type"] == "str" + assert config["service_account_credentials"]["sensitive"] is True + + assert "subject" in config + assert config["subject"]["type"] == "str" + + assert "days_back" in config + assert config["days_back"]["type"] == "int" + assert config["days_back"]["value"] == 30 + assert config["days_back"]["validations"] == [ + {"type": "greater_than", "constraint": -1} + ] + + assert "days_forward" in config + assert config["days_forward"]["type"] == "int" + assert config["days_forward"]["value"] == 30 + assert config["days_forward"]["validations"] == [ + {"type": "greater_than", "constraint": -1} + ] + + +@pytest.mark.asyncio +async def test_ping_for_successful_connection(): + """Tests the ping functionality for ensuring connection to Google Calendar.""" + + expected_response = { + "kind": "calendar#calendarList", + "items": [], + } + async with create_gcal_source() as source: + as_service_account_response = asyncio.Future() + as_service_account_response.set_result(expected_response) + + with mock.patch.object( + Aiogoogle, "as_service_account", return_value=as_service_account_response + ): + await source.ping() + + +@patch("connectors.utils.time_to_sleep_between_retries", mock.Mock(return_value=0)) +@pytest.mark.asyncio +async def test_ping_for_failed_connection(): + """Tests the ping functionality when connection can not be established to Google Calendar.""" + + async with create_gcal_source() as source: + with mock.patch.object( + Aiogoogle, "discover", side_effect=Exception("Something went wrong") + ): + with pytest.raises(Exception): + await source.ping() + + +@pytest.mark.asyncio +async def test_get_docs(): + """Tests the module responsible to fetch and yield documents from Google Calendar.""" + + async with create_gcal_source() as source: + # Mock responses for calendar list, calendar details, and events + calendar_list_response = { + "kind": "calendar#calendarList", + "items": [ + { + "id": "calendar1", + "summary": "Calendar 1", + "summaryOverride": "Calendar 1 Override", + "colorId": "1", + "backgroundColor": "#ffffff", + "foregroundColor": "#000000", + "accessRole": "owner", + "primary": True, + } + ], + } + + calendar_response = { + "id": "calendar1", + "summary": "Calendar 1", + "description": "Calendar 1 Description", + "location": "Location 1", + "timeZone": "UTC", + } + + events_response = { + "kind": "calendar#events", + "items": [ + { + "id": "event1", + "summary": "Event 1", + "description": "Event 1 Description", + "location": "Event Location", + "colorId": "1", + "start": {"dateTime": "2025-07-23T10:00:00Z"}, + "end": {"dateTime": "2025-07-23T11:00:00Z"}, + "created": "2025-07-20T10:00:00Z", + "updated": "2025-07-21T10:00:00Z", + "status": "confirmed", + "organizer": {"email": "organizer@example.com"}, + "creator": {"email": "creator@example.com"}, + "attendees": [ + {"displayName": "John Doe", "email": "attendee@example.com"} + ], + } + ], + } + + # Mock the client methods + mock_client = mock.MagicMock() + mock_client.list_calendar_list = AsyncIterator([calendar_list_response]) + mock_client.get_calendar = mock.AsyncMock(return_value=calendar_response) + mock_client.list_events = AsyncIterator([events_response]) + + # Mock the calendar_client method to return our mock client + with mock.patch.object(source, "calendar_client", return_value=mock_client): + # Collect the documents yielded by get_docs + documents = [] + async for doc, _ in source.get_docs(): + documents.append(doc) + + # We should have 3 documents: 1 calendar list entry, 1 calendar, and 1 event + assert len(documents) == 3 + + # Verify the calendar list entry + calendar_list_doc = next( + (doc for doc in documents if doc["type"] == "calendar_list"), None + ) + assert calendar_list_doc is not None + assert calendar_list_doc["calendar_id"] == "calendar1" + assert calendar_list_doc["summary"] == "Calendar 1" + assert calendar_list_doc["summary_override"] == "Calendar 1 Override" + + # Verify the calendar + calendar_doc = next( + (doc for doc in documents if doc["type"] == "calendar"), None + ) + assert calendar_doc is not None + assert calendar_doc["calendar_id"] == "calendar1" + assert calendar_doc["summary"] == "Calendar 1" + assert calendar_doc["description"] == "Calendar 1 Description" + + # Verify the event + event_doc = next((doc for doc in documents if doc["type"] == "event"), None) + assert event_doc is not None + assert event_doc["event_id"] == "event1" + assert event_doc["summary"] == "Event 1" + assert event_doc["description"] == "Event 1 Description" + + # Verify the event has the simplified fields + assert "attendees" in event_doc + assert "attachments" in event_doc + assert "meeting_link" in event_doc + # Check that attendees structure is simplified + assert len(event_doc["attendees"]) == 1 + assert event_doc["attendees"][0]["name"] == "John Doe" + assert event_doc["attendees"][0]["email"] == "attendee@example.com" + # Check that unnecessary fields are removed + assert "color_id" not in event_doc + assert "transparency" not in event_doc + assert "visibility" not in event_doc + assert "conference_data" not in event_doc + + +@pytest.mark.asyncio +async def test_get_docs_with_time_range(): + """Tests the get_docs method with time range configuration.""" + + async with create_gcal_source(days_back=7, days_forward=14) as source: + # Mock responses for calendar list, calendar details, and events + calendar_list_response = { + "kind": "calendar#calendarList", + "items": [{"id": "calendar1", "summary": "Calendar 1"}], + } + + calendar_response = {"id": "calendar1", "summary": "Calendar 1"} + + events_response = {"kind": "calendar#events", "items": []} + + # Mock the client methods + mock_client = mock.MagicMock() + mock_client.list_calendar_list = AsyncIterator([calendar_list_response]) + mock_client.get_calendar = mock.AsyncMock(return_value=calendar_response) + mock_client.list_events = AsyncIterator([events_response]) + + # Mock the calendar_client method to return our mock client + with mock.patch.object(source, "calendar_client", return_value=mock_client): + # Collect the documents yielded by get_docs + documents = [] + async for doc, _ in source.get_docs(): + documents.append(doc) + + # We should have 2 documents: 1 calendar list entry and 1 calendar + assert len(documents) == 2 + + # Verify the calendar list entry + calendar_list_doc = next( + (doc for doc in documents if doc["type"] == "calendar_list"), None + ) + assert calendar_list_doc is not None + assert calendar_list_doc["calendar_id"] == "calendar1" + assert calendar_list_doc["summary"] == "Calendar 1" + + # Verify the calendar + calendar_doc = next( + (doc for doc in documents if doc["type"] == "calendar"), None + ) + assert calendar_doc is not None + assert calendar_doc["calendar_id"] == "calendar1" + assert calendar_doc["summary"] == "Calendar 1" + + # Verify that list_events was called with time parameters + mock_client.list_events.assert_called_once() + call_args = mock_client.list_events.call_args + assert len(call_args[0]) == 3 # calendar_id, time_min, time_max + assert call_args[0][0] == "calendar1" + # Verify time_min and time_max are provided (exact values depend on when test runs) + assert call_args[0][1] is not None # time_min + assert call_args[0][2] is not None # time_max + + +@pytest.mark.asyncio +async def test_client_methods(): + """Test the GoogleCalendarClient methods.""" + + client = GoogleCalendarClient( + json_credentials={"project_id": "dummy123"}, subject="test@example.com" + ) + + # Mock the api_call and api_call_paged methods + with mock.patch.object( + client, "api_call", mock.AsyncMock(return_value={"id": "calendar1"}) + ): + with mock.patch.object( + client, + "api_call_paged", + side_effect=lambda *args, **kwargs: AsyncIterator([{"items": []}]), + ): + # Test ping + result = await client.ping() + assert result == {"id": "calendar1"} + client.api_call.assert_called_once_with( + resource="calendarList", method="list", maxResults=1 + ) + + # Reset mock + client.api_call.reset_mock() + + # Test get_calendar + result = await client.get_calendar("calendar1") + assert result == {"id": "calendar1"} + client.api_call.assert_called_once_with( + resource="calendars", method="get", calendarId="calendar1" + ) + + # Test list_calendar_list + async for page in client.list_calendar_list(): + assert page == {"items": []} + client.api_call_paged.assert_called_once_with( + resource="calendarList", method="list", maxResults=100 + ) + + # Reset mock + client.api_call_paged.reset_mock() + + # Test list_events with time range + async for page in client.list_events( + "calendar1", "2025-07-23T00:00:00Z", "2025-07-30T23:59:59Z" + ): + assert page == {"items": []} + client.api_call_paged.assert_called_once_with( + resource="events", + method="list", + calendarId="calendar1", + maxResults=100, + timeMin="2025-07-23T00:00:00Z", + timeMax="2025-07-30T23:59:59Z", + ) + + # Reset mock + client.api_call.reset_mock() + + +@pytest.mark.asyncio +async def test_close(): + """Test the close method""" + async with create_gcal_source() as source: + # close should not raise an exception + await source.close()