feat(htsget): c4gh-encrypted streams

.gitignore

@@ -1,3 +1,9 @@
+# agents
+.agents
+AGENTS.md
+.claude
+CLAUDE.md
+
 # zarr
 .zmetadata
 # Byte-compiled / optimized / DLL files

docs/specs/2026-06-12-encrypted-htsget-streams-design.md

@@ -0,0 +1,59 @@
+# Client-side support for encrypted htsget streams
+
+Date: 2026-06-12
+
+## Problem
+
+The htsget client in `modos.genomics.htsget` streams genomic regions by
+fetching a ticket of byte ranges and concatenating them into a single stream.
+The [htsget-rs](https://github.com/umccr/htsget-rs) server can serve
+crypt4gh-encrypted streams, but the client cannot yet decrypt them (we only
+do crypt4gh on *local* files, in `modos.genomics.c4gh`).
+
+## Server protocol (htsget-rs, experimental)
+
+The client sends a `Client-Public-Key: <base64 crypt4gh public key>` header and
+an `encryptionScheme=C4GH` query parameter. The server returns byte ranges that
+concatenate into a valid crypt4gh file (header re-encrypted to that public key,
+plus edit lists). The client decrypts the assembled stream with the matching
+private key.
+
+The `encryptionScheme=C4GH` parameter is experimental and subject to change.
+
+## Decisions
+
+- User inputs: A `--secret-key` path (plus optional passphrase). The
+  public key is derived from it if possible.
+- **Surface:** both the CLI `modos stream` and the Python API
+  (`HtsgetConnection` / `MODO.stream_genomics`).
+- **Output:** decrypt transparently; the user gets the plaintext region.
+
+## Approach
+
+Decrypt at the `HtsgetConnection.open()` boundary: when a secret key is set,
+`open()` returns a decrypted readable and every consumer (CLI, `to_pysam`,
+`to_file`) is unchanged. The encrypted stream is buffered to a temp file before
+decryption (consistent with `to_pysam`, which already spools).
+
+Rejected: decrypting in each consumer (duplication, leaks encryption awareness);
+a lazy streaming-decrypt wrapper (crypt4gh has no clean incremental reader).
+
+## Flow
+
+```mermaid
+sequenceDiagram
+    participant C as client
+    participant H as htsget-rs
+    participant S as store
+    C->>H: ticket (public key, C4GH)
+    H-->>C: byte ranges
+    C->>S: fetch ranges
+    S-->>C: crypt4gh blocks
+    Note over C: open(): assemble + decrypt
+    C->>C: plaintext region
+```
+
+## Out of scope
+
+- Server-side / deployment configuration of htsget-rs C4GH.
+- Encrypting or decrypting remote objects at rest (client-side only).

docs/tutorials/genomics_streaming.md

@@ -36,6 +36,40 @@ modos --endpoint http://localhost stream --region BA000007.3 s3://modos-demo/ex
 We highly recommend using the `MODOs` CLI for streaming. The output can directly be passed to tools like <a href="https://www.htslib.org/" target="_blank">samtools</a>. Streaming using the `MODOs` python api will return a <a href="https://pysam.readthedocs.io/en/stable/" target="_blank">pysam</a> object. `pysam` does not allow reading from byte-streams and thus the streamed region will be written into an temporary file before parsing to `pysam`. For large files/regions this can cause issues.
 :::
 
+### Streaming encrypted data
+
+When the htsget server stores crypt4gh-encrypted data, pass your secret key to
+decrypt the stream on the fly. The matching public key is derived and sent to
+the server automatically; the decrypted region is returned transparently.
+
+:::::{tab-set}
+
+::::{tab-item} python
+:sync: python
+```{code-block} python
+from modos.api import MODO
+
+modo = MODO(path='s3://modos-demo/ex', endpoint='http://localhost')
+modo.stream_genomics(
+    file_path="demo1.cram",
+    region="BA000007.3",
+    secret_key_path="path/to/recipient.sec",
+)
+```
+::::
+
+::::{tab-item} cli
+:sync: cli
+```{code-block} console
+modos --endpoint http://localhost remote stream \
+  --region BA000007.3 \
+  --secret-key path/to/recipient.sec \
+  s3://modos-demo/ex demo1.cram
+```
+::::
+
+:::::
+
 ## Data encryption and decryption
 
 Genomic data is typically sensitive, and data sharing increases the risk to data security.

src/modos/api.py

@@ -475,6 +475,8 @@ def stream_genomics(
         file_path: str,
         region: str | None = None,
         reference_filename: str | None = None,
+        secret_key_path: Path | None = None,
+        passphrase: str | None = None,
     ) -> Iterator[AlignedSegment | VariantRecord]:
         """Slices both local and remote CRAM, VCF (.vcf.gz), and BCF
         files returning an iterator over records.
@@ -487,6 +489,10 @@ def stream_genomics(
             Genomic region in UCSC format (e.g. chr1:1000-200
         reference_filename
             Path to the reference genome file.
+        secret_key_path
+            Path to a crypt4gh secret key to decrypt an encrypted stream.
+        passphrase
+            Passphrase to unlock the secret key, if it is protected.
 
         Returns
         -------
@@ -503,6 +509,8 @@ def stream_genomics(
                 self.endpoint.htsget,
                 Path(*self.path.parts[1:]) / file_path,
                 region=_region,
+                secret_key_path=secret_key_path,
+                passphrase=passphrase,
             )
             stream = con.to_pysam(reference_filename=reference_filename)
         else:

src/modos/cli/remote.py

@@ -169,6 +169,23 @@ def stream(
             help="Restrict stream to genomic region (chr:start-end).",
         ),
     ] = None,
+    secret_key_path: Annotated[
+        Path | None,
+        typer.Option(
+            "--secret-key",
+            "-s",
+            help="Secret key to decrypt an encrypted stream. Its public "
+            "key is sent to the htsget server.",
+        ),
+    ] = None,
+    passphrase: Annotated[
+        Path | None,
+        typer.Option(
+            "--passphrase",
+            "-pw",
+            help="Path to file with passphrase to unlock the secret key.",
+        ),
+    ] = None,
 ):
     """Stream genomic file from a remote modo into stdout."""
     from modos.genomics.htsget import HtsgetConnection
@@ -189,7 +206,13 @@ def stream(
     if not endpoint.htsget:
         raise ValueError("No htsget service found.")
 
-    con = HtsgetConnection(endpoint.htsget, source, _region)
+    con = HtsgetConnection(
+        endpoint.htsget,
+        source,
+        _region,
+        secret_key_path=secret_key_path,
+        passphrase=passphrase.read_text() if passphrase else None,
+    )
     with con.open() as f:
         for chunk in f:
             sys.stdout.buffer.write(chunk)

src/modos/genomics/c4gh.py

@@ -11,7 +11,6 @@
 .. [2] https://github.com/EGA-archive/crypt4gh
 """
 
-from typing import List, Optional, Set, Tuple
 import os
 from pathlib import Path
 
@@ -24,8 +23,8 @@
 
 
 def get_secret_key(
-    seckey_path: Optional[os.PathLike] = None,
-    passphrase: Optional[str] = None,
+    seckey_path: os.PathLike | None = None,
+    passphrase: str | None = None,
 ) -> bytes:
     """
     Get the secret key for encryption/decryption.
@@ -61,9 +60,18 @@ def get_secret_key(
     return get_private_key(seckey_path, cb)
 
 
+def derive_public_key(seckey: bytes) -> bytes:
+    """Derive the raw 32-byte X25519 public key from secret key bytes.
+
+    Used to advertise the client public key to an htsget server via the
+    Client-Public-Key header without requiring a separate public key file.
+    """
+    return bytes(PrivateKey(seckey).public_key)
+
+
 def get_keys(
-    recipient_pubkeys: List[os.PathLike] | os.PathLike, seckey: bytes
-) -> Set[Tuple[int, bytes, bytes]]:
+    recipient_pubkeys: list[os.PathLike] | os.PathLike, seckey: bytes
+) -> set[tuple[int, bytes, bytes]]:
     """Retrieves recipient public keys and builds a collection of "key tuples".
 
     Parameters
@@ -78,7 +86,7 @@ def get_keys(
     {(method, seckey, recipient_pubkey)}
         Set of key triplets, one for each recipient.
     """
-    if not isinstance(recipient_pubkeys, List):
+    if not isinstance(recipient_pubkeys, list):
         recipient_pubkeys = [recipient_pubkeys]
     recipient_list = []
     for pubkey in recipient_pubkeys:
@@ -87,15 +95,17 @@ def get_keys(
             raise ValueError(f"Recipient public key not found: {pubkey}")
         recipient_list.append((0, seckey, get_public_key(pubkey_path)))
 
+    recipient_list.append((0, seckey, derive_public_key(seckey)))
+
     return set(recipient_list)
 
 
 def encrypt_file(
-    recipient_pubkeys: List[os.PathLike] | os.PathLike,
+    recipient_pubkeys: list[os.PathLike] | os.PathLike,
     infile: Path | str,
     outfile: Path | str,
-    seckey_path: Optional[os.PathLike] = None,
-    passphrase: Optional[str] = None,
+    seckey_path: os.PathLike | None = None,
+    passphrase: str | None = None,
 ):
     """Encrypt a file using the crypt4gh algorithm (authenticated encryption)."""
     seckey = get_secret_key(seckey_path, passphrase=passphrase)
@@ -108,8 +118,8 @@ def decrypt_file(
     seckey_path: os.PathLike,
     infile: Path | str,
     outfile: Path | str,
-    sender_pubkey: Optional[os.PathLike] = None,
-    passphrase: Optional[str] = None,
+    sender_pubkey: os.PathLike | None = None,
+    passphrase: str | None = None,
 ):
     if not seckey_path:
         raise ValueError(