feat(htsget): c4gh-encrypted streams

.gitignore

@@ -1,3 +1,9 @@
+# agents
+.agents
+AGENTS.md
+.claude
+CLAUDE.md
+
 # zarr
 .zmetadata
 # Byte-compiled / optimized / DLL files

docs/specs/2026-06-12-encrypted-htsget-streams-design.md

@@ -0,0 +1,59 @@
+# Client-side support for encrypted htsget streams
+
+Date: 2026-06-12
+
+## Problem
+
+The htsget client in `modos.genomics.htsget` streams genomic regions by
+fetching a ticket of byte ranges and concatenating them into a single stream.
+The [htsget-rs](https://github.com/umccr/htsget-rs) server can serve
+crypt4gh-encrypted streams, but the client cannot yet decrypt them (we only
+do crypt4gh on *local* files, in `modos.genomics.c4gh`).
+
+## Server protocol (htsget-rs, experimental)
+
+The client sends a `Client-Public-Key: <base64 crypt4gh public key>` header and
+an `encryptionScheme=C4GH` query parameter. The server returns byte ranges that
+concatenate into a valid crypt4gh file (header re-encrypted to that public key,
+plus edit lists). The client decrypts the assembled stream with the matching
+private key.
+
+The `encryptionScheme=C4GH` parameter is experimental and subject to change.
+
+## Decisions
+
+- User inputs: A `--secret-key` path (plus optional passphrase). The
+  public key is derived from it if possible.
+- **Surface:** both the CLI `modos stream` and the Python API
+  (`HtsgetConnection` / `MODO.stream_genomics`).
+- **Output:** decrypt transparently; the user gets the plaintext region.
+
+## Approach
+
+Decrypt at the `HtsgetConnection.open()` boundary: when a secret key is set,
+`open()` returns a decrypted readable and every consumer (CLI, `to_pysam`,
+`to_file`) is unchanged. The encrypted stream is buffered to a temp file before
+decryption (consistent with `to_pysam`, which already spools).
+
+Rejected: decrypting in each consumer (duplication, leaks encryption awareness);
+a lazy streaming-decrypt wrapper (crypt4gh has no clean incremental reader).
+
+## Flow
+
+```mermaid
+sequenceDiagram
+    participant C as client
+    participant H as htsget-rs
+    participant S as store
+    C->>H: ticket (public key, C4GH)
+    H-->>C: byte ranges
+    C->>S: fetch ranges
+    S-->>C: crypt4gh blocks
+    Note over C: open(): assemble + decrypt
+    C->>C: plaintext region
+```
+
+## Out of scope
+
+- Server-side / deployment configuration of htsget-rs C4GH.
+- Encrypting or decrypting remote objects at rest (client-side only).

docs/tutorials/genomics_streaming.md

@@ -36,6 +36,40 @@ modos --endpoint http://localhost stream --region BA000007.3 s3://modos-demo/ex
 We highly recommend using the `MODOs` CLI for streaming. The output can directly be passed to tools like <a href="https://www.htslib.org/" target="_blank">samtools</a>. Streaming using the `MODOs` python api will return a <a href="https://pysam.readthedocs.io/en/stable/" target="_blank">pysam</a> object. `pysam` does not allow reading from byte-streams and thus the streamed region will be written into an temporary file before parsing to `pysam`. For large files/regions this can cause issues.
 :::
 
+### Streaming encrypted data
+
+When the htsget server stores crypt4gh-encrypted data, pass your secret key to
+decrypt the stream on the fly. The matching public key is derived and sent to
+the server automatically; the decrypted region is returned transparently.
+
+:::::{tab-set}
+
+::::{tab-item} python
+:sync: python
+```{code-block} python
+from modos.api import MODO
+
+modo = MODO(path='s3://modos-demo/ex', endpoint='http://localhost')
+modo.stream_genomics(
+    file_path="demo1.cram",
+    region="BA000007.3",
+    secret_key_path="path/to/recipient.sec",
+)
+```
+::::
+
+::::{tab-item} cli
+:sync: cli
+```{code-block} console
+modos --endpoint http://localhost remote stream \
+  --region BA000007.3 \
+  --secret-key path/to/recipient.sec \
+  s3://modos-demo/ex demo1.cram
+```
+::::
+
+:::::
+
 ## Data encryption and decryption
 
 Genomic data is typically sensitive, and data sharing increases the risk to data security.

src/modos/api.py

@@ -475,6 +475,8 @@ def stream_genomics(
         file_path: str,
         region: str | None = None,
         reference_filename: str | None = None,
+        secret_key_path: Path | None = None,
+        passphrase: str | None = None,
     ) -> Iterator[AlignedSegment | VariantRecord]:
         """Slices both local and remote CRAM, VCF (.vcf.gz), and BCF
         files returning an iterator over records.
@@ -487,6 +489,10 @@ def stream_genomics(
             Genomic region in UCSC format (e.g. chr1:1000-200
         reference_filename
             Path to the reference genome file.
+        secret_key_path
+            Path to a crypt4gh secret key to decrypt an encrypted stream.
+        passphrase
+            Passphrase to unlock the secret key, if it is protected.
 
         Returns
         -------
@@ -503,6 +509,8 @@ def stream_genomics(
                 self.endpoint.htsget,
                 Path(*self.path.parts[1:]) / file_path,
                 region=_region,
+                secret_key_path=secret_key_path,
+                passphrase=passphrase,
             )
             stream = con.to_pysam(reference_filename=reference_filename)
         else:

src/modos/cli/remote.py

@@ -1,5 +1,6 @@
 import sys
 from pathlib import Path
+from typing import Optional
 from typing_extensions import Annotated
 from loguru import logger
 import typer
@@ -169,6 +170,23 @@ def stream(
             help="Restrict stream to genomic region (chr:start-end).",
         ),
     ] = None,
+    secret_key_path: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--secret-key",
+            "-s",
+            help="Secret key to decrypt an encrypted stream. Its public "
+            "key is sent to the htsget server.",
+        ),
+    ] = None,
+    passphrase: Annotated[
+        Optional[Path],
+        typer.Option(
+            "--passphrase",
+            "-pw",
+            help="Path to file with passphrase to unlock the secret key.",
+        ),
+    ] = None,
 ):
     """Stream genomic file from a remote modo into stdout."""
     from modos.genomics.htsget import HtsgetConnection
@@ -189,7 +207,13 @@ def stream(
     if not endpoint.htsget:
         raise ValueError("No htsget service found.")
 
-    con = HtsgetConnection(endpoint.htsget, source, _region)
+    con = HtsgetConnection(
+        endpoint.htsget,
+        source,
+        _region,
+        secret_key_path=secret_key_path,
+        passphrase=passphrase.read_text() if passphrase else None,
+    )
     with con.open() as f:
         for chunk in f:
             sys.stdout.buffer.write(chunk)

src/modos/genomics/c4gh.py

@@ -61,6 +61,15 @@ def get_secret_key(
     return get_private_key(seckey_path, cb)
 
 
+def derive_public_key(seckey: bytes) -> bytes:
+    """Derive the raw 32-byte X25519 public key from secret key bytes.
+
+    Used to advertise the client public key to an htsget server via the
+    Client-Public-Key header without requiring a separate public key file.
+    """
+    return bytes(PrivateKey(seckey).public_key)
+
+
 def get_keys(
     recipient_pubkeys: List[os.PathLike] | os.PathLike, seckey: bytes
 ) -> Set[Tuple[int, bytes, bytes]]:

src/modos/genomics/htsget.py

@@ -45,18 +45,26 @@
 from typing import Any
 from urllib.parse import urlparse, parse_qs
 
+from crypt4gh import CIPHER_SEGMENT_SIZE
+from crypt4gh.lib import decrypt
 from pydantic import HttpUrl, validate_call
 from pydantic.dataclasses import dataclass
 import pysam
 import requests
 
 from modos.remote import get_session
+from modos.genomics.c4gh import derive_public_key, get_secret_key
 from modos.genomics.region import Region
 from modos.genomics.formats import GenomicFileSuffix, read_pysam
 
 
 @validate_call
-def build_htsget_url(host: HttpUrl, path: Path, region: Region | None) -> str:
+def build_htsget_url(
+    host: HttpUrl,
+    path: Path,
+    region: Region | None,
+    encrypted: bool = False,
+) -> str:
     """Build an htsget URL from a host, path, and region.
 
     Examples
@@ -67,6 +75,13 @@ def build_htsget_url(host: HttpUrl, path: Path, region: Region | None) -> str:
     ...   Region("chr1", 0, 1000)
     ... )
     'http://localhost:8000/reads/file?format=BAM&referenceName=chr1&start=0&end=1000'
+    >>> build_htsget_url(
+    ...   "http://localhost:8000",
+    ...   Path("file.bam"),
+    ...   Region("chr1", 0, 1000),
+    ...   encrypted=True,
+    ... )
+    'http://localhost:8000/reads/file?format=BAM&referenceName=chr1&start=0&end=1000&encryptionScheme=C4GH'
     """
     format = GenomicFileSuffix.from_path(path)
     endpoint = format.to_htsget_endpoint()
@@ -79,6 +94,8 @@ def build_htsget_url(host: HttpUrl, path: Path, region: Region | None) -> str:
     url = f"{netloc}{endpoint}/{stem}?format={format.name}"
     if region:
         url += f"&{region.to_htsget_query()}"
+    if encrypted:
+        url += "&encryptionScheme=C4GH"
     return url
 
 
@@ -231,29 +248,94 @@ class HtsgetConnection:
     host: HttpUrl
     path: Path
     region: Region | None
+    secret_key_path: Path | None = None
+    passphrase: str | None = None
+
+    @property
+    def _encrypted(self) -> bool:
+        return self.secret_key_path is not None
 
     @property
     def url(self) -> str:
         """URL to fetch the ticket."""
-        return build_htsget_url(self.host, Path(self.path), self.region)
+        return build_htsget_url(
+            self.host, Path(self.path), self.region, encrypted=self._encrypted
+        )
+
+    @cached_property
+    def _seckey(self) -> bytes:
+        return get_secret_key(self.secret_key_path, self.passphrase)
 
     @cached_property
     def ticket(self) -> dict[str, Any]:
         """Ticket containing the URLs to fetch the data."""
-        return get_session().get(self.url).json()
-
-    def open(self) -> io.RawIOBase:
-        """Open a connection to the stream data."""
+        headers = {}
+        if self._encrypted:
+            headers["Client-Public-Key"] = base64.b64encode(
+                derive_public_key(self._seckey)
+            ).decode()
+        return get_session().get(self.url, headers=headers).json()
+
+    def _stream(self) -> HtsgetStream:
+        """Assemble the raw (still encrypted) htsget stream from the ticket."""
         try:
             return HtsgetStream(self.ticket["htsget"]["urls"])
         except KeyError:
             raise KeyError(f"No htsget urls found in ticket: {self.ticket}")
 
+    def open(self) -> io.IOBase:
+        """Open a connection to the stream data (decrypted if a key is set).
+
+        Encrypted streams are buffered to a temporary file for decryption,
+        so the requested region is materialized before this returns.
+        """
+        stream = self._stream()
+        if not self._encrypted:
+            return stream
+
+        # TODO: decrypt on the stream, and return a wrapped stream instead
+        # of using a temp file
+        plaintext = tempfile.TemporaryFile("w+b")
+        try:
+            self._decrypt_into(stream, plaintext)
+        except BaseException:
+            plaintext.close()
+            raise
+        plaintext.seek(0)
+        return plaintext
+
+    def _decrypt_into(self, stream: io.RawIOBase, outfile: io.IOBase) -> None:
+        """Decrypt the whole stream at once, writing plaintext to outfile."""
+        # decrypt expects a full cipher segment per read; BufferedReader
+        # wraps HtsgetStream's per-block reads to deliver one.
+        with io.BufferedReader(
+            stream, buffer_size=CIPHER_SEGMENT_SIZE
+        ) as encrypted:
+            try:
+                decrypt(
+                    keys=[(0, self._seckey, None)],
+                    infile=encrypted,
+                    outfile=outfile,
+                )
+            except Exception as err:
+                raise ValueError(
+                    "Failed to decrypt htsget stream. Ensure the "
+                    "secret key matches the public key registered "
+                    "with the server."
+                ) from err
+
     def to_file(self, path: Path):
-        """Save all data from the stream to a file."""
-        with self.open() as source, open(path, "wb") as sink:
-            for block in source:
-                sink.write(block)
+        """Save all data from the stream to a file.
+
+        Decryption writes straight into the destination, so an encrypted
+        stream is never materialized to an intermediate temporary file.
+        """
+        with self._stream() as stream, open(path, "wb") as sink:
+            if self._encrypted:
+                self._decrypt_into(stream, sink)
+            else:
+                for block in stream:
+                    sink.write(block)
 
     @classmethod
     def from_url(cls, url: str):

tests/conftest.py

@@ -2,14 +2,21 @@
 
 import os
 from pathlib import Path
-import pytest
 
-import modos_schema.datamodel as model
 import crypt4gh.keys.c4gh as c4gh
+import modos_schema.datamodel as model
+import pytest
 from testcontainers.minio import MinioContainer
 
 from modos.api import MODO
 
+
+@pytest.fixture(scope="session")
+def httpserver_listen_address():
+    """Bind to 127.0.0.1; 'localhost' may resolve to ::1 and break the client."""
+    return ("127.0.0.1", 0)
+
+
 ## Add --remote option
 # see: https://docs.pytest.org/en/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option
 

tests/test_c4gh.py

@@ -0,0 +1,13 @@
+"""Tests for crypt4gh helpers."""
+
+from crypt4gh.keys import get_private_key, get_public_key
+
+from modos.genomics.c4gh import derive_public_key
+
+
+def test_derive_public_key_matches_keypair(c4gh_keypair):
+    """The derived public key equals the keypair's own public key."""
+    seckey = get_private_key(str(c4gh_keypair["private_key"]), lambda: None)
+    expected = get_public_key(str(c4gh_keypair["public_key"]))
+
+    assert derive_public_key(seckey) == expected