diff --git a/backend/src/baserow/contrib/database/fields/dependencies/handler.py b/backend/src/baserow/contrib/database/fields/dependencies/handler.py index 208d7a2467..1876620d8d 100644 --- a/backend/src/baserow/contrib/database/fields/dependencies/handler.py +++ b/backend/src/baserow/contrib/database/fields/dependencies/handler.py @@ -711,9 +711,21 @@ def get_via_dependants_of_link_field(cls, field: "LinkRowField") -> FieldDependa dependant_id=related_field_id ) + queryset = FieldDependency.objects.filter(broken_via_dep_filter).select_related( + "dependant", "dependant__table" + ) + specific_dependants = { + dependant_field.id: dependant_field + for dependant_field in specific_iterator( + [via_dep.dependant for via_dep in queryset], + base_model=Field, + select_related=["table"], + ) + } + dependants = [] - for via_dep in FieldDependency.objects.filter(broken_via_dep_filter): - dependant_field = via_dep.dependant.specific + for via_dep in queryset: + dependant_field = specific_dependants[via_dep.dependant_id] dependant_field_type = field_type_registry.get_by_model(dependant_field) field_dep_tuple = ( dependant_field, diff --git a/backend/src/baserow/contrib/database/rows/handler.py b/backend/src/baserow/contrib/database/rows/handler.py index 107056b3f2..ac5afc02ce 100644 --- a/backend/src/baserow/contrib/database/rows/handler.py +++ b/backend/src/baserow/contrib/database/rows/handler.py @@ -687,7 +687,7 @@ def get_row_names( field_ids=[], fields=[primary_field], add_dependencies=False ) - queryset = model.objects.filter(pk__in=row_ids) + queryset = model.objects.filter(pk__in=row_ids).enhance_by_fields() return {row.id: str(row) for row in queryset} diff --git a/backend/src/baserow/contrib/database/webhooks/handler.py b/backend/src/baserow/contrib/database/webhooks/handler.py index 891eab295a..1d30ce1a1d 100644 --- a/backend/src/baserow/contrib/database/webhooks/handler.py +++ b/backend/src/baserow/contrib/database/webhooks/handler.py @@ -117,7 +117,7 @@ def _get_table_webhook( try: webhook = ( base_queryset.select_related("table__database__workspace") - .prefetch_related("events", "events__fields") + .prefetch_related("events", "events__fields", "events__views") .get(id=webhook_id) ) except TableWebhook.DoesNotExist: @@ -145,7 +145,7 @@ def get_all_table_webhooks(self, user: any, table: Table) -> QuerySet: ) return TableWebhook.objects.prefetch_related( - "events", "headers", "calls" + "events", "events__fields", "events__views", "headers", "calls" ).filter(table_id=table.id) def _update_webhook_event_config( diff --git a/backend/src/baserow/core/handler.py b/backend/src/baserow/core/handler.py index dfbd9c563b..f57ad40095 100755 --- a/backend/src/baserow/core/handler.py +++ b/backend/src/baserow/core/handler.py @@ -611,9 +611,20 @@ def get_workspaceuser_workspace_queryset(self) -> QuerySet[WorkspaceUser]: Returns WorkspaceUser queryset that will prefetch workspaces and their users. """ - workspaceusers_with_user_and_profile = WorkspaceUser.objects.select_related( - "user" - ).select_related("user__profile") + from baserow.core.two_factor_auth.models import TwoFactorAuthProviderModel + + workspaceusers_with_user_and_profile = ( + WorkspaceUser.objects.select_related("user") + .select_related("user__profile") + .prefetch_related( + Prefetch( + "user__two_factor_auth_provider", + queryset=specific_queryset( + TwoFactorAuthProviderModel.objects.all() + ), + ) + ) + ) workspaceuser_workspaces = WorkspaceUser.objects.select_related( "workspace" ).prefetch_related( diff --git a/backend/src/baserow/core/trash/handler.py b/backend/src/baserow/core/trash/handler.py index 475f357ec2..1f42f7cf93 100755 --- a/backend/src/baserow/core/trash/handler.py +++ b/backend/src/baserow/core/trash/handler.py @@ -569,7 +569,7 @@ def get_trash_contents( if application: trash_contents = trash_contents.filter(application=application) - return trash_contents.order_by("-trashed_at") + return trash_contents.select_related("user_who_trashed").order_by("-trashed_at") @staticmethod def item_has_a_trashed_parent(item, check_item_also=False): diff --git a/backend/src/baserow/ws/consumers.py b/backend/src/baserow/ws/consumers.py index 6789a170b7..17d61c7846 100644 --- a/backend/src/baserow/ws/consumers.py +++ b/backend/src/baserow/ws/consumers.py @@ -53,6 +53,11 @@ unit="1", description="Closed realtime WebSocket connections, labelled by close code.", ) +websocket_active_connections = meter.create_up_down_counter( + "baserow.websocket_active_connections", + unit="1", + description="Currently open realtime WebSocket connections.", +) # Known close codes pass through; anything else becomes "other" to keep the # metric's label cardinality bounded. @@ -185,6 +190,7 @@ class CoreConsumer(AsyncJsonWebsocketConsumer): async def connect(self): await self.accept() websocket_connections_counter.add(1) + websocket_active_connections.add(1) user = self.scope["user"] await self.send_json( @@ -225,6 +231,7 @@ async def disconnect(self, code): await self._discard_all_channel_groups() await self.channel_layer.group_discard("users", self.channel_name) finally: + websocket_active_connections.add(-1) websocket_disconnects_counter.add( 1, attributes={"code": _bucket_close_code(code)} ) diff --git a/backend/tests/baserow/api/groups/test_workspace_views.py b/backend/tests/baserow/api/groups/test_workspace_views.py index 454324b1b3..7b507b6a57 100755 --- a/backend/tests/baserow/api/groups/test_workspace_views.py +++ b/backend/tests/baserow/api/groups/test_workspace_views.py @@ -1,4 +1,6 @@ +from django.db import connection from django.shortcuts import reverse +from django.test.utils import CaptureQueriesContext import pytest from rest_framework.status import HTTP_200_OK, HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND @@ -499,3 +501,43 @@ def test_create_initial_workspace(api_client, data_fixture): assert workspace_user.workspace.id == json_response["id"] assert workspace_user.workspace.name == "Test1's workspace" assert workspace_user.user == user + + +@pytest.mark.django_db +def test_list_workspaces_two_factor_auth_no_n_plus_one(api_client, data_fixture): + def build_workspace_and_get_token(member_count): + owner, token = data_fixture.create_user_and_token() + workspace = data_fixture.create_workspace(user=owner) + for _ in range(member_count - 1): + member = data_fixture.create_user() + data_fixture.create_user_workspace( + workspace=workspace, user=member, permissions="MEMBER" + ) + # exercise the prefetched reverse relation with real providers + data_fixture.configure_base_totp(member) + return token + + token_few = build_workspace_and_get_token(member_count=2) + token_many = build_workspace_and_get_token(member_count=6) + + url = reverse("api:workspaces:list") + + api_client.get(url, **{"HTTP_AUTHORIZATION": f"JWT {token_few}"}) + api_client.get(url, **{"HTTP_AUTHORIZATION": f"JWT {token_many}"}) + + with CaptureQueriesContext(connection) as few_queries: + response_few = api_client.get(url, **{"HTTP_AUTHORIZATION": f"JWT {token_few}"}) + with CaptureQueriesContext(connection) as many_queries: + response_many = api_client.get( + url, **{"HTTP_AUTHORIZATION": f"JWT {token_many}"} + ) + + assert response_few.status_code == HTTP_200_OK + assert response_many.status_code == HTTP_200_OK + assert len(response_few.json()[0]["users"]) == 2 + assert len(response_many.json()[0]["users"]) == 6 + + assert len(many_queries.captured_queries) == len(few_queries.captured_queries), ( + f"N+1 on two_factor_auth: {len(few_queries.captured_queries)} queries for " + f"2 members vs {len(many_queries.captured_queries)} for 6 members" + ) diff --git a/backend/tests/baserow/api/trash/test_trash_views.py b/backend/tests/baserow/api/trash/test_trash_views.py index f47d70f573..f15516efe7 100755 --- a/backend/tests/baserow/api/trash/test_trash_views.py +++ b/backend/tests/baserow/api/trash/test_trash_views.py @@ -1,6 +1,8 @@ from datetime import datetime, timedelta, timezone +from django.db import connection from django.shortcuts import reverse +from django.test.utils import CaptureQueriesContext import pytest from freezegun import freeze_time @@ -1009,3 +1011,45 @@ def test_managed_trash_entries_excluded_from_contents( } ], } + + +@pytest.mark.django_db +def test_getting_trash_contents_does_not_scale_queries_with_number_of_entries( + api_client, data_fixture +): + user, token = data_fixture.create_user_and_token() + + def trash_rows(count): + workspace = data_fixture.create_workspace(user=user) + database = data_fixture.create_database_application( + user=user, workspace=workspace + ) + table = data_fixture.create_database_table(user=user, database=database) + model = table.get_model() + for _ in range(count): + TrashHandler.trash(user, workspace, database, model.objects.create()) + return workspace + + def get_contents(workspace): + return api_client.get( + reverse( + "api:trash:contents", + kwargs={"workspace_id": workspace.id}, + ), + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + few_workspace = trash_rows(5) + many_workspace = trash_rows(10) + + # Warm up so cold-cache and model generation don't skew the measured counts. + assert get_contents(few_workspace).status_code == HTTP_200_OK + assert get_contents(many_workspace).status_code == HTTP_200_OK + + with CaptureQueriesContext(connection) as few_queries: + assert get_contents(few_workspace).status_code == HTTP_200_OK + + with CaptureQueriesContext(connection) as many_queries: + assert get_contents(many_workspace).status_code == HTTP_200_OK + + assert len(few_queries.captured_queries) == len(many_queries.captured_queries) diff --git a/backend/tests/baserow/contrib/database/api/webhooks/test_webhook_views.py b/backend/tests/baserow/contrib/database/api/webhooks/test_webhook_views.py index 4a639981b7..01573d99a4 100644 --- a/backend/tests/baserow/contrib/database/api/webhooks/test_webhook_views.py +++ b/backend/tests/baserow/contrib/database/api/webhooks/test_webhook_views.py @@ -1,5 +1,6 @@ +from django.db import connection from django.shortcuts import reverse -from django.test.utils import override_settings +from django.test.utils import CaptureQueriesContext, override_settings import pytest import responses @@ -719,3 +720,47 @@ def test_can_query_private_http_addresses_when_env_var_on(api_client, data_fixtu HTTP_AUTHORIZATION=f"JWT {jwt_token}", ) assert response.status_code == HTTP_200_OK, response_json + + +@pytest.mark.django_db +def test_list_webhooks_query_count_does_not_scale_with_webhooks( + api_client, data_fixture +): + user, jwt_token = data_fixture.create_user_and_token() + + def build_table_with_webhooks(webhook_count): + table = data_fixture.create_database_table(user=user) + fields = [data_fixture.create_text_field(table=table) for _ in range(2)] + views = [data_fixture.create_grid_view(table=table) for _ in range(2)] + for _ in range(webhook_count): + webhook = data_fixture.create_table_webhook( + table=table, + include_all_events=False, + events=["rows.created", "rows.updated"], + ) + for event in webhook.events.all(): + event.fields.set(fields) + event.views.set(views) + return table + + def list_webhooks(table): + return api_client.get( + reverse("api:database:webhooks:list", kwargs={"table_id": table.id}), + format="json", + HTTP_AUTHORIZATION=f"JWT {jwt_token}", + ) + + few_table = build_table_with_webhooks(3) + many_table = build_table_with_webhooks(6) + + # warm up so cold-cache/model-generation queries don't skew the comparison + assert list_webhooks(few_table).status_code == HTTP_200_OK + assert list_webhooks(many_table).status_code == HTTP_200_OK + + with CaptureQueriesContext(connection) as few_queries: + assert list_webhooks(few_table).status_code == HTTP_200_OK + + with CaptureQueriesContext(connection) as many_queries: + assert list_webhooks(many_table).status_code == HTTP_200_OK + + assert len(few_queries.captured_queries) == len(many_queries.captured_queries) diff --git a/backend/tests/baserow/contrib/database/field/dependencies/test_field_dependency_handler.py b/backend/tests/baserow/contrib/database/field/dependencies/test_field_dependency_handler.py index 8e278eaad8..42f76bb81b 100644 --- a/backend/tests/baserow/contrib/database/field/dependencies/test_field_dependency_handler.py +++ b/backend/tests/baserow/contrib/database/field/dependencies/test_field_dependency_handler.py @@ -1,4 +1,6 @@ from django.contrib.contenttypes.models import ContentType +from django.db import connection +from django.test.utils import CaptureQueriesContext import pytest from pytest_unordered import unordered @@ -818,3 +820,48 @@ def test_can_import_database_with_formula_dependencies(data_fixture): assert r2.lookup == [{"id": 1, "value": "A"}] assert r2.lookup2 == [{"id": 1, "value": "A"}] + + +@pytest.mark.django_db +@pytest.mark.field_link_row +def test_get_via_dependants_of_link_field_num_queries_does_not_scale(data_fixture): + table = data_fixture.create_database_table() + linked_table = data_fixture.create_database_table(database=table.database) + link_field = data_fixture.create_link_row_field( + table=table, link_row_table=linked_table + ) + dependency_field = data_fixture.create_text_field(table=table) + + def add_via_dependants(count): + created_ids = [] + for _ in range(count): + dependant = data_fixture.create_text_field(table=linked_table) + FieldDependency.objects.create( + dependency=dependency_field, dependant=dependant, via=link_field + ) + created_ids.append(dependant.id) + return created_ids + + def measure(): + with CaptureQueriesContext(connection) as ctx: + dependants = FieldDependencyHandler.get_via_dependants_of_link_field( + link_field.specific + ) + return len(ctx.captured_queries), dependants + + first_ids = add_via_dependants(3) + FieldDependencyHandler.get_via_dependants_of_link_field(link_field.specific) + few_queries, few_dependants = measure() + + second_ids = add_via_dependants(3) + FieldDependencyHandler.get_via_dependants_of_link_field(link_field.specific) + more_queries, more_dependants = measure() + + assert more_queries == few_queries + + assert len(more_dependants) == len(few_dependants) + 3 + result_ids = {field.id for field, _, _ in more_dependants} + assert set(first_ids) | set(second_ids) <= result_ids + for field, field_type, via_path in more_dependants: + assert via_path == [] + assert field_type == field_type_registry.get_by_model(field) diff --git a/backend/tests/baserow/contrib/database/rows/test_rows_handler.py b/backend/tests/baserow/contrib/database/rows/test_rows_handler.py index 53fb773e8e..0ead5f41b1 100644 --- a/backend/tests/baserow/contrib/database/rows/test_rows_handler.py +++ b/backend/tests/baserow/contrib/database/rows/test_rows_handler.py @@ -2132,3 +2132,42 @@ def test_create_row_handler_default_value_priority(data_fixture): user, table, values={f"field_{text_field.id}": "user value"}, view=view ) assert getattr(row3, f"field_{text_field.id}") == "user value" + + +@pytest.mark.django_db +def test_get_row_names_does_not_scale_queries_with_relational_primary(data_fixture): + user = data_fixture.create_user() + table = data_fixture.create_database_table(user=user) + single_select_field = data_fixture.create_single_select_field( + table=table, name="Status", primary=True + ) + options = [ + data_fixture.create_select_option( + field=single_select_field, value=f"option-{i}", order=i + ) + for i in range(3) + ] + + model = table.get_model() + row_ids = [] + for i in range(20): + row = model.objects.create( + **{f"field_{single_select_field.id}_id": options[i % len(options)].id} + ) + row_ids.append(row.id) + + n = 5 + small_ids = row_ids[:n] + large_ids = row_ids[: 2 * n] + + # Warm up to prime content-type/model caches so only the row count differs. + RowHandler().get_row_names(table, small_ids) + RowHandler().get_row_names(table, large_ids) + + with CaptureQueriesContext(connection) as small_captured: + RowHandler().get_row_names(table, small_ids) + + with CaptureQueriesContext(connection) as large_captured: + RowHandler().get_row_names(table, large_ids) + + assert len(small_captured.captured_queries) == len(large_captured.captured_queries) diff --git a/changelog/entries/unreleased/bug/fixes_several_n1_query_problems_that_slowed_down_some_list_e.json b/changelog/entries/unreleased/bug/fixes_several_n1_query_problems_that_slowed_down_some_list_e.json new file mode 100644 index 0000000000..23e8cd8663 --- /dev/null +++ b/changelog/entries/unreleased/bug/fixes_several_n1_query_problems_that_slowed_down_some_list_e.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Fixes several N+1 query problems that slowed down some list endpoints", + "issue_origin": "github", + "issue_number": null, + "domain": "database", + "bullet_points": [], + "created_at": "2026-07-10" +}