Experimental lazy geff

README.md

@@ -11,7 +11,7 @@ A common data structure and basic tools for multi-object tracking.
 ## Features
 
 - Graph-based representation of tracking problems
-- In-memory (RustWorkX) and database-backed (SQL) graph backends
+- In-memory (RustWorkX), database-backed (SQL), and lazy read-only GEFF/Zarr graph backends
 - Nodes and edges can take arbitrary attributes
 - SQLGraph backend can index frequently queried attributes for faster filtering
 - Standardize API for node operators (e.g. defining objects and their attributes)
@@ -35,7 +35,7 @@ It uses graphs to represent detections (nodes) and their connections (edges), ma
 Key benefits:
 - Consistent data representation for tracking problems
 - Modular components that can be combined as needed
-- Support for both small datasets (in-memory) and large datasets (database)
+- Support for small in-memory datasets and large database- or GEFF-backed datasets
 
 ## Documentation
 

benchmarks/geff_loading.py

@@ -0,0 +1,188 @@
+"""ASV benchmarks for loading and querying GEFF graph backends.
+
+The suite separates construction from steady-state read queries because
+``ZarrSQLGraph`` intentionally defers array reads until a query is collected,
+whereas the other backends materialize the GEFF during ``from_geff``.
+"""
+
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+
+import numpy as np
+from asv_runner.benchmarks.mark import SkipNotImplemented
+from geff.core_io import write_arrays
+from geff_spec import Axis, GeffMetadata, PropMetadata
+
+import tracksdata as td
+from benchmarks.common import IS_CI
+from tracksdata.attrs import EdgeAttr, NodeAttr
+from tracksdata.constants import DEFAULT_ATTR_KEYS
+
+if IS_CI:
+    NODE_SIZES = (10_000,)
+else:
+    NODE_SIZES = (1_000, 100_000)
+
+BACKEND_NAMES = (
+    "RustWorkXGraph",
+    "IndexedRXGraph",
+    "SQLGraphMemory",
+    "ZarrSQLGraph",
+)
+N_TIME_POINTS = 100
+N_SUCCESSOR_SEEDS = 256
+
+
+def _write_geff(path: Path, n_nodes: int) -> None:
+    """Write a deterministic scalar-property GEFF benchmark fixture."""
+    node_ids = np.arange(n_nodes, dtype=np.uint64)
+    node_props = {
+        DEFAULT_ATTR_KEYS.T: {
+            "values": np.arange(n_nodes, dtype=np.int32) % N_TIME_POINTS,
+            "missing": None,
+        },
+        "score": {
+            "values": np.linspace(0.0, 1.0, n_nodes, dtype=np.float32),
+            "missing": None,
+        },
+        "label": {
+            "values": np.arange(n_nodes, dtype=np.int32),
+            "missing": None,
+        },
+    }
+
+    edge_source = np.arange(max(0, n_nodes - 1), dtype=np.uint64)
+    edge_target = edge_source + 1
+    edge_ids = np.column_stack((edge_source, edge_target))
+    edge_props = {
+        "weight": {
+            "values": np.linspace(0.0, 1.0, len(edge_ids), dtype=np.float32),
+            "missing": None,
+        }
+    }
+    metadata = GeffMetadata(
+        directed=True,
+        axes=[Axis(name=DEFAULT_ATTR_KEYS.T, type="time")],
+        node_props_metadata={
+            DEFAULT_ATTR_KEYS.T: PropMetadata(identifier=DEFAULT_ATTR_KEYS.T, dtype="int32"),
+            "score": PropMetadata(identifier="score", dtype="float32"),
+            "label": PropMetadata(identifier="label", dtype="int32"),
+        },
+        edge_props_metadata={"weight": PropMetadata(identifier="weight", dtype="float32")},
+        extra={"tracksdata": {"benchmark_nodes": n_nodes}},
+    )
+    write_arrays(
+        path,
+        node_ids=node_ids,
+        node_props=node_props,
+        edge_ids=edge_ids,
+        edge_props=edge_props,
+        metadata=metadata,
+        overwrite=True,
+        zarr_format=3,
+    )
+
+
+def _require_backend(backend_name: str) -> None:
+    """Skip benchmarks for backends absent from the compared revision."""
+    if backend_name != "SQLGraphMemory" and not hasattr(td.graph, backend_name):
+        raise SkipNotImplemented(f"{backend_name} is unavailable in this revision.")
+
+
+def _load_graph(backend_name: str, path: str) -> td.graph.BaseGraph:
+    """Load one backend from the common GEFF fixture."""
+    if backend_name == "SQLGraphMemory":
+        graph, _ = td.graph.SQLGraph.from_geff(
+            path,
+            drivername="sqlite",
+            database=":memory:",
+            engine_kwargs={"connect_args": {"check_same_thread": False}},
+        )
+        return graph
+
+    backend = getattr(td.graph, backend_name)
+    graph, _ = backend.from_geff(path)
+    return graph
+
+
+def _dispose_graph(graph: td.graph.BaseGraph) -> None:
+    """Release SQLAlchemy resources without assuming every backend has an engine."""
+    if type(graph) is td.graph.SQLGraph:
+        graph._engine.dispose()
+
+
+class _GeffFixture:
+    """Shared ASV cache containing identical GEFF inputs for every backend."""
+
+    param_names = ("backend", "n_nodes")
+    params = (BACKEND_NAMES, NODE_SIZES)
+    timeout = 300
+
+    def setup_cache(self) -> dict[int, str]:
+        root = Path(tempfile.mkdtemp(prefix="tracksdata_geff_benchmark_"))
+        paths: dict[int, str] = {}
+        for n_nodes in NODE_SIZES:
+            path = root / f"graph_{n_nodes}.geff"
+            _write_geff(path, n_nodes)
+            paths[n_nodes] = str(path)
+        return paths
+
+
+class GeffLoadBenchmark(_GeffFixture):
+    """Fresh-object ``from_geff`` construction cost for each backend."""
+
+    number = 1
+    warmup_time = 0
+
+    def setup(self, paths: dict[int, str], backend_name: str, n_nodes: int) -> None:
+        _require_backend(backend_name)
+        self.path = paths[n_nodes]
+        self.graph: td.graph.BaseGraph | None = None
+
+    def teardown(self, paths: dict[int, str], backend_name: str, n_nodes: int) -> None:
+        graph = getattr(self, "graph", None)
+        if graph is not None:
+            _dispose_graph(graph)
+
+    def time_from_geff(self, paths: dict[int, str], backend_name: str, n_nodes: int) -> None:
+        self.graph = _load_graph(backend_name, self.path)
+
+
+class GeffQueryBenchmark(_GeffFixture):
+    """Read-query costs after backend construction has completed."""
+
+    def setup(self, paths: dict[int, str], backend_name: str, n_nodes: int) -> None:
+        _require_backend(backend_name)
+        self.graph = _load_graph(backend_name, paths[n_nodes])
+        self.filter_time = N_TIME_POINTS // 2
+        seed_count = min(N_SUCCESSOR_SEEDS, max(1, n_nodes - 1))
+        self.successor_seeds = np.linspace(0, max(0, n_nodes - 2), seed_count, dtype=np.int64).tolist()
+
+    def teardown(self, paths: dict[int, str], backend_name: str, n_nodes: int) -> None:
+        graph = getattr(self, "graph", None)
+        if graph is not None:
+            _dispose_graph(graph)
+
+    def time_node_attrs(self, paths: dict[int, str], backend_name: str, n_nodes: int) -> None:
+        self.graph.node_attrs(attr_keys=[DEFAULT_ATTR_KEYS.NODE_ID, DEFAULT_ATTR_KEYS.T, "score", "label"])
+
+    def time_edge_attrs(self, paths: dict[int, str], backend_name: str, n_nodes: int) -> None:
+        self.graph.edge_attrs(attr_keys=["weight"])
+
+    def time_filter_projected_node_attrs(
+        self,
+        paths: dict[int, str],
+        backend_name: str,
+        n_nodes: int,
+    ) -> None:
+        self.graph.filter(NodeAttr(DEFAULT_ATTR_KEYS.T) == self.filter_time).node_attrs(
+            attr_keys=[DEFAULT_ATTR_KEYS.NODE_ID, "score"]
+        )
+
+    def time_filter_edge_ids(self, paths: dict[int, str], backend_name: str, n_nodes: int) -> None:
+        self.graph.filter(EdgeAttr("weight") >= 0.5).edge_ids()
+
+    def time_successors_batch(self, paths: dict[int, str], backend_name: str, n_nodes: int) -> None:
+        self.graph.successors(self.successor_seeds)

docs/concepts.md

@@ -30,8 +30,31 @@ graph.create_node_attr_index(["t", "label"])  # composite index
 graph.create_edge_attr_index("score", unique=True)
 ```
 
+### ZarrSQLGraph
+- **Use case**: Querying an existing directed GEFF Zarr store without first loading the complete graph into memory
+- **Performance**: Lazy, chunked reads through xarray-sql and DataFusion
+- **Features**: Read-only scalar filtering and row-selected loading of array properties
+
+```python
+from tracksdata.attrs import NodeAttr
+from tracksdata.graph import ZarrSQLGraph
+
+graph, geff_metadata = ZarrSQLGraph.from_geff("tracks.geff")
+selected = graph.filter(NodeAttr("t") == 10).node_attrs(attr_keys=["node_id", "score"])
+```
+
+`ZarrSQLGraph` supports Zarr v2 and v3 paths or configured Zarr stores. GEFF
+payload arrays remain lazy, although xarray-sql currently creates in-memory integer
+row coordinates proportional to the number of nodes and edges during registration.
+The backend does not support undirected GEFF stores. Numeric and boolean scalar
+properties can be used in SQL filters, except integer or boolean properties that have missing-value
+masks. Strings, nullable integer/boolean properties, fixed-shape arrays, and
+variable-length arrays remain retrievable, but cannot be filter predicates. When
+edits are required, materialize a mutable backend with, for example,
+`RustWorkXGraph.from_other(graph)`.
+
 ### GraphView
-- **Use case**: Results subgraph either backends
+- **Use case**: Result subgraphs from any backend
 - **Performance**: Low overhead, similar to RustWorkXGraph
 - **Features**: Maintains connection to root graph, all operations are mirrored to the root graph
 

docs/faq.md

@@ -22,7 +22,8 @@ TracksData focuses on providing a **general** unified data structure and modular
 ### Which graph backend should I use?
 
 - **RustWorkXGraph**: For most applications where data fits in memory
-- **SQLGraph**: For large datasets or when you need persistent storage
+- **SQLGraph**: For large mutable datasets or when you need persistent database storage
+- **ZarrSQLGraph**: For lazy, read-only queries against an existing directed GEFF Zarr store
 - **GraphView**: You shouldn't instantiate this directly, it is used internally by the library when you use `graph.subgraph()`
 
 ### Can TracksData handle cell divisions?

pyproject.toml

@@ -57,6 +57,7 @@ dependencies = [
   "numcodecs>=0.13",
   "numcodecs>=0.13,<0.16; python_version > '3.10,<=3.11'", # TODO: remove pin once the 16 release is stable
   "numcodecs>=0.15; python_version >= '3.13'", # TODO: remove pin once the 16 release is stable
+  "xarray-sql>=0.3.3",
 ]
 
 [project.optional-dependencies]

src/tracksdata/graph/__init__.py

@@ -4,7 +4,17 @@
 from tracksdata.graph._graph_view import GraphView
 from tracksdata.graph._rustworkx_graph import IndexedRXGraph, RustWorkXGraph
 from tracksdata.graph._sql_graph import SQLGraph
+from tracksdata.graph._zarr_sql_graph import ZarrSQLGraph
 
 InMemoryGraph = RustWorkXGraph
 
-__all__ = ["BaseGraph", "GraphView", "InMemoryGraph", "IndexedRXGraph", "MetadataView", "RustWorkXGraph", "SQLGraph"]
+__all__ = [
+    "BaseGraph",
+    "GraphView",
+    "InMemoryGraph",
+    "IndexedRXGraph",
+    "MetadataView",
+    "RustWorkXGraph",
+    "SQLGraph",
+    "ZarrSQLGraph",
+]