Paginate Data Overview for wide datasets

CHANGELOG.md

@@ -32,6 +32,15 @@ All notable changes to AIDRIN are documented here. This project loosely follows
   `Overall Completeness (column-wise)` (present briefly on this branch) were
   removed in favor of the single `Overall Completeness` key.
 
+### Changed
+
+- **Data Overview is paged by feature.** `/summary-statistics` accepts `page`
+  and `page_size` (defaults 1 and 25, maximum page size 100) and adds a
+  `pagination` object. `summary_statistics`, `categorical_summary`, and
+  `histograms` are page-local; dataset-wide counts and feature lists stay
+  complete. Globus workers need the `summary_pagination_v1` capability. An
+  older worker gets an upgrade message instead of computing every feature.
+
 ### Added
 
 - **New data-quality completeness metrics** (CLI, Python library, batch, MCP,

aidrin/compute/remote.py

@@ -187,14 +187,28 @@ def _file_reference_validation():
         )
 
     def _summary_statistics():
-        """Compute summary statistics + histograms on the remote file."""
+        """Compute summary statistics + histograms on the remote file.
+
+        ``summary_statistics``, ``categorical_summary``, and ``histograms``
+        cover only the requested page. Dataset-wide counts and feature lists
+        stay complete. ``class_imbalance_features`` still scans every column.
+        """
         import io
         import base64
         import pandas as pd
         import matplotlib.pyplot as plt
         import seaborn as sns
         from aidrin.file_handling.file_parser import read_file as _read_file
         from aidrin.file_handling.hashable_utils import hashable_series, safe_nunique
+        from aidrin.structured_data_metrics.summary_pagination import (
+            paginate_features,
+            parse_page_params,
+        )
+
+        try:
+            page, page_size = parse_page_params(params.get("page"), params.get("page_size"))
+        except ValueError as exc:
+            return {"error": str(exc)}
 
         df = _read_file(file_info)
         if isinstance(df, str):
@@ -212,15 +226,18 @@ def _summary_statistics():
             if pd.api.types.is_string_dtype(dtype) or pd.api.types.is_bool_dtype(dtype)
         ]
         all_features = numerical_columns + categorical_columns
-
-        # Restrict to numeric columns: describe() would otherwise fall back to
-        # object-column stats (top/freq are strings) and the numeric formatter
-        # below would fail on them when there are no numerical features (#125).
-        numeric_df = df.select_dtypes(include="number")
-        if numeric_df.shape[1] == 0:
+        page_features = paginate_features(numerical_columns, categorical_columns, page, page_size)
+        page_numeric = page_features["numerical"]
+        page_categorical = page_features["categorical"]
+
+        # Restrict to this page's numeric columns: describe() would otherwise
+        # fall back to object-column stats (top/freq are strings) and the
+        # numeric formatter below would fail on them when there are no
+        # numerical features (#125).
+        if not page_numeric:
             summary = {}
         else:
-            summary = numeric_df.describe().map(
+            summary = df[page_numeric].describe().map(
                 lambda x: round(x, 2) if x == 0 or abs(x) >= 0.001 else f"{x:.2e}"
             ).to_dict()
 
@@ -230,10 +247,10 @@ def _summary_statistics():
                 if old_key in ["25%", "50%", "75%"]:
                     v[old_key.replace("%", "th percentile")] = v.pop(old_key)
 
-        # Per-column summary for categorical features (parity with the local
-        # /summary-statistics panel).
+        # Per-column summary for categorical features on this page (parity
+        # with the local /summary-statistics panel).
         categorical_summary = {}
-        for col in categorical_columns:
+        for col in page_categorical:
             # value_counts() and nunique() both hash, so a column holding
             # arrays/lists/dicts (parquet/HDF5/JSON) would raise and take the
             # whole summary down. Normalize such columns first.
@@ -249,11 +266,11 @@ def _summary_statistics():
                 "freq_pct": round(freq / count * 100, 1) if count else 0.0,
             }
 
-        # Generate histograms (transparent, same as local)
+        # Generate histograms for this page's numeric columns (transparent, same as local).
         text_color = "#6b7280"
         curve_color = "#4485F4"
         histograms = {}
-        for column in df.select_dtypes(include="number").columns:
+        for column in page_numeric:
             try:
                 fig, ax = plt.subplots(figsize=(4, 3))
                 fig.patch.set_alpha(0)
@@ -284,6 +301,7 @@ def _summary_statistics():
             "summary_statistics": summary,
             "categorical_summary": categorical_summary,
             "histograms": histograms,
+            "pagination": page_features["pagination"],
             "class_imbalance_features": [
                 col for col in all_features if safe_nunique(df[col]) <= 30
             ],
@@ -448,6 +466,7 @@ def remote_env_probe():
     capabilities = ["file_reference_validation_v1"]
     if hasattr(aidrin, "calculate_variable_unit_validation"):
         capabilities.append("variable_unit_metadata_v1")
+    capabilities.append("summary_pagination_v1")
 
     return {
         "aidrin_version": aidrin.__version__,

aidrin/structured_data_metrics/summary_pagination.py

@@ -0,0 +1,106 @@
+"""Page the Data Overview feature list.
+
+The ordered list is numerical features, then categorical features, matching
+the order the Data Overview already uses.
+
+``start`` and ``end`` are one-based, inclusive positions in that ordered list.
+
+``total_pages = max(1, ceil(total_features / page_size))``, so page 1 is
+always valid.
+
+An empty dataset has ``total_features = 0``, ``total_pages = 1``,
+``start = end = 0``, and empty feature lists.
+
+An out-of-range page is not an error. The call succeeds with empty lists,
+``pagination.page`` echoes the request, and ``start = end = 0``. The page is
+not clamped, so a cache key matches the page that was requested.
+"""
+
+import math
+
+DEFAULT_PAGE = 1
+DEFAULT_PAGE_SIZE = 25
+MAX_PAGE_SIZE = 100
+
+
+def _parse_positive_int(value, name):
+    """Return a positive int, or None when the caller should use the default."""
+    if value is None:
+        return None
+    if isinstance(value, bool) or isinstance(value, float):
+        raise ValueError(f"{name} must be a positive integer")
+    if isinstance(value, int):
+        number = value
+    elif isinstance(value, str) and value.strip().isdigit():
+        number = int(value.strip())
+    else:
+        raise ValueError(f"{name} must be a positive integer")
+    if number < 1:
+        raise ValueError(f"{name} must be a positive integer")
+    return number
+
+
+def parse_page_params(page, page_size):
+    """Validate a Data Overview page request.
+
+    ``None`` uses the default. Both values must be positive integers. Bools,
+    floats (including ``1.5``), zero, negatives, and non-numeric text are
+    rejected. ``page_size`` must be at most ``MAX_PAGE_SIZE``.
+
+    Query strings arrive as text, so only a digit string is accepted. JSON
+    booleans are rejected because ``bool`` is a subclass of ``int``.
+    """
+    parsed_page = _parse_positive_int(page, "page")
+    parsed_size = _parse_positive_int(page_size, "page_size")
+    if parsed_page is None:
+        parsed_page = DEFAULT_PAGE
+    if parsed_size is None:
+        parsed_size = DEFAULT_PAGE_SIZE
+    if parsed_size > MAX_PAGE_SIZE:
+        raise ValueError(f"page_size must be at most {MAX_PAGE_SIZE}")
+    return parsed_page, parsed_size
+
+
+def paginate_features(numerical, categorical, page, page_size):
+    """Return one page of features plus the pagination block.
+
+    ``numerical`` and ``categorical`` are the dataset-wide lists. The page
+    lists preserve that order and contain only the requested slice.
+    """
+    numerical = list(numerical)
+    categorical = list(categorical)
+    ordered = numerical + categorical
+    total_features = len(ordered)
+    total_pages = max(1, math.ceil(total_features / page_size))
+    start_index = (page - 1) * page_size
+
+    if total_features == 0 or start_index >= total_features:
+        page_numerical = []
+        page_categorical = []
+        start = 0
+        end = 0
+    else:
+        page_items = ordered[start_index:start_index + page_size]
+        numerical_count = len(numerical)
+        page_numerical = []
+        page_categorical = []
+        for offset, name in enumerate(page_items):
+            if start_index + offset < numerical_count:
+                page_numerical.append(name)
+            else:
+                page_categorical.append(name)
+        start = start_index + 1
+        end = start_index + len(page_items)
+
+    return {
+        "pagination": {
+            "page": page,
+            "page_size": page_size,
+            "total_pages": total_pages,
+            "total_features": total_features,
+            "start": start,
+            "end": end,
+        },
+        "numerical": page_numerical,
+        "categorical": page_categorical,
+    }