diff --git a/.gitignore b/.gitignore index a283fb7..8d58c34 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ aegle_patch_viewer/ debug_synthetic_previews *.pkl *.csv +!exps/templates/oocyte_samples_template.csv +!exps/configs/oocyte_d11_d13_panel1/samples.csv notebooks # Ignore specific files NW_1_Scan1 copy.qptiff diff --git a/AGENTS.md b/AGENTS.md index 323b87f..39dd00a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,7 @@ # Repository Guidelines ## Project Structure & Module Organization -The Git repo root is `/workspaces/codex-analysis/0-phenocycler-penntmc-pipeline`, so assume all relative paths start there unless otherwise noted. Core Python modules that drive preprocessing, segmentation, and reporting live in `aegle/`, while downstream analytics sit in `aegle_analysis/`. Command-line entry points (e.g., `src/main.py`, `src/run_analysis.py`, `src/extract_tissue_regions.py`) wrap the package functions so agents can wire configs without editing the package. Shell harnesses such as `launcher/run_main_ft.sh`, `launcher/run_preprocess_ft.sh`, and the helper scripts under `scripts/` orchestrate typical PennTMC scenarios. Images, masks, and experiment outputs are staged in `data/`, `out/`, and `logs/`, and reusable YAML configurations reside in `exps/configs/` with templates under `exps/templates/`. Unit and integration tests are grouped under `tests/`, mirroring the pipeline stages (`tests/preprocess`, `tests/main`, `tests/analysis`, plus shared helpers in `tests/utils`). Keep notebooks and scratch analyses inside `notebooks/` or `debug/` to avoid mixing exploratory code with shipping modules. +The Git repo root is the directory containing this `AGENTS.md`; assume all relative paths start there unless otherwise noted. Core Python modules that drive preprocessing, segmentation, and reporting live in `aegle/`, while downstream analytics sit in `aegle_analysis/`. Command-line entry points (e.g., `src/main.py`, `src/run_analysis.py`, `src/extract_tissue_regions.py`) wrap the package functions so agents can wire configs without editing the package. Shell harnesses such as `launcher/run_main_ft.sh`, `launcher/run_preprocess_ft.sh`, and the helper scripts under `scripts/` orchestrate typical PennTMC scenarios. Images, masks, and experiment outputs are staged in `data/`, `out/`, and `logs/`, and reusable YAML configurations reside in `exps/configs/` with templates under `exps/templates/`. Unit and integration tests are grouped under `tests/`, mirroring the pipeline stages (`tests/preprocess`, `tests/main`, `tests/analysis`, plus shared helpers in `tests/utils`). Keep notebooks and scratch analyses inside `notebooks/` or `debug/` to avoid mixing exploratory code with shipping modules. ## Build, Test, and Development Commands - `python -m pip install -e .` — install the `aegle` package in editable mode so module imports (e.g., `from aegle.pipeline import run_pipeline`) resolve consistently. @@ -21,5 +21,14 @@ Follow the short imperative commit style already in `git log` (e.g., “Added nu ## Data & Configuration Tips Treat `exps/templates/*.yaml` as the canonical starting point, copying them into `exps/configs////config.yaml` when onboarding a new sample. Keep raw PhenoCycler deliveries in `data/` and ensure derived artifacts (`out/`, `debug/`, `logs/`) stay git-ignored—share reproducibility details via config diffs rather than uploading binary tiles. Scripts assume relative paths inside the repo root; when building new orchestrators, echo both `--data_dir` and `--out_dir` so automated agents can rediscover the run context. Launcher scripts (e.g., `launcher/run_main_ft.sh`) provide cohort-specific wrappers. Verify that secrets (API tokens for optional LLM-assisted annotation in `src/run_analysis.py`) are provided through environment variables or `.env` files listed in `.gitignore`, never in tracked configs. +## Oocyte Agent Workflow +Before detecting, reviewing, profiling, or releasing raw-UCHL1 oocytes, read +`aegle/oocyte/AGENTS.md` and `docs/oocyte_detection.md`. The nested guide is the +Codex orchestration contract for this human-in-the-loop workflow. It defines +which actions Codex may perform, which decisions require a biologist, durable +review JSON requirements, immutable output rules, the reference panel1 run, and +the handoff prompt for a new Codex session. Do not substitute DeepCell masks for +the raw-UCHL1 input or describe unreviewed detector accepts as final oocytes. + ## Runtime Defaults Most production runs use `patching.split_mode: full_image`; prefer optimizations that keep the single-patch path efficient and avoid per-patch chatter that adds little value in that mode. diff --git a/aegle/oocyte/AGENTS.md b/aegle/oocyte/AGENTS.md new file mode 100644 index 0000000..2abd00a --- /dev/null +++ b/aegle/oocyte/AGENTS.md @@ -0,0 +1,286 @@ +# Oocyte Module Collaboration Guide + +This directory implements standalone raw-UCHL1 oocyte detection, review, final +mask export, expression profiling, and release packaging. It does not require +DeepCell labels or another Aegle pipeline output. Keep the module runnable from +raw registered OME-TIFFs plus the acquisition antibody table. + +## Delivery Model + +This module is intentionally delivered as a deterministic pipeline with +human-in-the-loop biological review and optional Codex orchestration. Codex is +allowed to inspect state, run deterministic commands, validate identities, +generate review consoles, ingest exported review JSON, preserve versioned +artifacts, and explain the next checkpoint. A biologist remains the authority +for whether an object is an oocyte and whether its boundary is acceptable. + +Do not make the current delivery depend on an autonomous workflow engine. The +purpose of these instructions is to let another researcher give Codex the repo, +raw-image paths, and review exports and reproduce the same sequence safely. +Algorithm optimization, threshold retuning, and automatic replacement of human +choices are follow-up work unless the user explicitly opens a separate issue. + +## Sources Of Truth + +Use these sources in order rather than relying on conversation history: + +1. `aegle/oocyte/AGENTS.md` defines the orchestration and safety contract. +2. `docs/oocyte_detection.md` defines command syntax and artifact semantics. +3. The selected YAML config, sample CSV, run manifests, profile fingerprint, + candidate tables, and NPZ metadata define the computation that actually ran. +4. Exported identity-bound review JSON defines durable human decisions. Browser + local storage, screenshots, chat messages, and CSV exports are not final + review authority. +5. Finalization, profiling, and release manifests plus their SHA-256 records + define the delivered labels and expression tables. + +If prose conflicts with an identity-validated manifest, stop and investigate; +do not silently choose whichever artifact is more convenient. + +## Starting A Codex Session + +At the beginning of a new oocyte session, Codex must: + +1. Read this file and the relevant sections of `docs/oocyte_detection.md`. +2. Inspect `git status --short` and avoid staging, reverting, or modifying + unrelated worktree changes. +3. Resolve the requested sample IDs, raw OME-TIFFs, antibody table, detector + config, output root, and all review JSON paths from disk. +4. Audit existing manifests and versioned directories instead of assuming the + last conversational step completed successfully. +5. Report the current phase for every requested sample, the evidence used to + determine it, unresolved biological decisions, and the exact next action. +6. Confirm the output destination is new before running any finalizer, profiler, + or release builder. Never overwrite an existing reviewed version. + +For a code change, run the focused tests before handing work back: + +```bash +python -m unittest discover -s tests/oocyte -v +``` + +Routine operation on unchanged code does not require rerunning the entire suite, +but every generated release must pass the independent release validator. + +## Human And Codex Responsibilities + +Codex owns the mechanical work: + +- Validate input paths, channel resolution, image geometry, and profile identity. +- Run detection, review generation, review ingestion, finalization, profiling, + release construction, and independent validation commands. +- Keep detector outputs, reviewed deltas, final labels, and releases in separate + versioned directories. +- Summarize counts, incomplete decisions, identity mismatches, overlap failures, + and generated artifact paths without changing biological classifications. +- Start the dynamic review server on `0.0.0.0:8767` when Precision or Recall + needs native raw-image access and tell the user which URL to open. + +The biologist owns the scientific decisions: + +- Precision: Accept, Reject, or Unsure for each proposed object. +- Boundary review: choose a safe proposal, request a manual contour, or exclude. +- Recall: classify every survey window and click every visible oocyte lacking a + satisfactory current mask. +- Provisional-mask and polygon review: select the intended boundary or leave the + object unresolved. + +Codex must never convert `Unsure`, `Neither`, an incomplete queue, a note alone, +or unexported browser state into a final label. When a human checkpoint is +pending, provide the console URL and expected export filename, then stop the +scientific transition until the exported JSON is available. + +## Orchestration State Model + +Treat each sample as moving through these explicit phases. Some samples have no +boundary-recovery subphase, but phase ordering must not be reversed. + +| Phase | Codex action | Durable evidence | Human checkpoint | +| --- | --- | --- | --- | +| Input audit | Validate manifest, raw image, antibodies, pixel size, profile, and output destination | Resolved config and sample manifest | Confirm sample scope and profile | +| Detection | Run `src/run_oocyte.py` and generate exact candidate NPZs, labels, reports, and manifests | `run_manifest.json`, `candidates.csv`, `masks/`, `oocyte_labels.ome.tiff` | None | +| Precision | Serve the candidate console and preserve its exported JSON | Identity-bound Precision JSON | Classify every candidate and mask quality | +| Precision boundary resolution | Generate threshold alternatives or a polygon queue, then run the matching finalizer | Versioned `precision_resolved_vN` manifest and copied review JSON | Select alternatives or draw contours | +| Recall | Generate the reviewed-overlay survey, serve all windows, and analyze the exported JSON | Geometry-bound Recall JSON and analysis manifest | Classify every window and click all misses | +| Miss boundary resolution | Generate conservative/expanded masks and, when needed, manual contours; finalize accepted choices | Versioned reviewed manual-seed directory and decision audit | Select or draw each missing boundary | +| Final labels | Verify there are no unresolved required decisions and identify the exact final OME label and mapping | Finalization manifest, label OME-TIFF, mapping CSV, exact NPZ masks | Confirm release set | +| Profiling | Measure raw within-mask means from final labels only | `profiling_manifest.json`, marker and metadata tables | None | +| Release | Build into a new immutable directory and run `validate` independently | Release and sample manifests, SHA-256 artifact index | Approve package for sharing | + +After the user reports that a review is complete, Codex must inspect the stated +JSON file, verify sample and review identity, summarize completion counts, run +only the matching analyzer/finalizer, and report the next human checkpoint. Do +not skip directly from a browser export to profiling or release. + +## Operating Principles + +- Treat detector accepts as review candidates, not final biological labels. +- Never overwrite detector, review, or finalized outputs. Use a new versioned + directory for every iteration. +- Keep precision and recall evidence distinct. Precision review decides whether + proposed objects and boundaries are acceptable. Recall review surveys the + whole tissue and records missing objects. +- A manual center is not a final mask. Generate provisional boundaries, obtain a + second review, and finalize only explicit accepted choices. +- Do not silently convert `Unsure`, `Neither`, incomplete reviews, or browser + local storage into final labels. +- Preserve source identities, exported review JSON, artifact hashes, and final + mapping tables so every label can be audited. +- Keep donor11 sections as zero-oocyte negative controls for the panel1 release. + Any accepted detector or rescue object in donor11 must block release creation. + +## Standard Workflow + +1. Run standalone detection from the raw OME-TIFF and antibody table with + `src/run_oocyte.py`. Use the frozen study profile unless an issue explicitly + defines and validates a new profile. +2. Open the sample review console. Complete Precision first, rejecting false + objects and flagging true oocytes with unacceptable boundaries. +3. Resolve precision boundary cases through the generated boundary review. Use + manual contours only when threshold-derived alternatives cannot represent the + intended cell. +4. Complete the whole-slide Recall survey. Click every oocyte that lacks a + satisfactory current mask and export the review JSON. +5. Analyze manual centers into conservative and expanded provisional masks, + review every proposal, and finalize only accepted masks. Resolve remaining + confirmed boundary failures with the manual-contour workflow. +6. Run `src/run_oocyte_profile.py` only against the final reviewed label image + and mapping. The default measurement is the raw within-mask mean for every + registered channel. +7. Build a release with `src/run_oocyte_release.py build`, then run the separate + `validate` command. Never edit files inside a completed release. + +The command-level options for every boundary and Recall subphase are maintained +in `docs/oocyte_detection.md`; do not duplicate or invent flags from memory. + +## Reference Panel1 Run + +The completed D11/D13 panel1 run is the reproducibility reference, not a claim +that the same profile is validated for every ovary cohort: + +```text +Detector config: exps/configs/oocyte_d11_d13_panel1/config.yaml +Sample manifest: exps/configs/oocyte_d11_d13_panel1/samples.csv +Detector profile: donor13_v6 +Secondary profile: donor13_v6_rescue_v1 +Final release spec: exps/configs/oocyte_d11_d13_panel1/release_v6.yaml +Final donor13 counts: 13-21=129, 13-22=68, 13-23=219, 13-24=43 +Negative controls: 11-21, 11-22, 11-23, 11-24, all with zero final labels +``` + +The final reviewed release contains 459 donor13 labels, raw within-mask values +for 36 channels, four zero-label donor11 controls, complete review evidence, and +static per-sample consoles. The release validator reports 598 hashed artifacts. +Use these invariants to detect accidental orchestration or packaging drift. + +The detector and release directories used in one workstation are not portable +inputs. A colleague should provide their own raw-data root and output root while +retaining the checked-in config/profile semantics. The sample CSV resolves raw +paths relative to its own location; the release v6 spec is cohort provenance and +must be adapted only when rebuilding from another filesystem layout. + +## Reusing The Workflow On New Samples + +Copy both templates rather than editing the reference cohort in place: + +```text +exps/templates/oocyte_template.yaml +exps/templates/oocyte_samples_template.csv +``` + +A new sample needs a registered CYX OME-TIFF, an acquisition antibody table or +explicit UCHL1 channel index, pixel size, a unique sample ID, and a new output +directory. The detector can use `donor13_v6` as an initial candidate generator, +but a different donor, panel, staining run, or tissue context requires complete +Precision and whole-slide Recall before any sensitivity claim. If review across +multiple samples justifies new numerical behavior, create a new named profile +and a separate algorithm-validation issue; never mutate `donor13_v6`. + +## Prompt For A Colleague + +Use this prompt when handing the repository to another Codex session. Replace +the bracketed values with real paths and sample IDs: + +```text +Operate the standalone raw-UCHL1 oocyte workflow in this Aegle checkout. +First read aegle/oocyte/AGENTS.md and docs/oocyte_detection.md. Do not optimize +the detector or change frozen profiles in this task. Audit the on-disk state for +samples [SAMPLE_IDS], using config [CONFIG], manifest [SAMPLE_CSV], output root +[OUTPUT_ROOT], and review exports under [REVIEW_DIR]. Report each sample's +current workflow phase and evidence before running anything. Then execute the +next deterministic step, preserve all existing outputs, and stop whenever a +biological review is required. After each exported review JSON is provided, +validate its identity, ingest it through the documented command, and generate +the next review checkpoint. Profile only final reviewed labels. Build releases +in new versioned directories and independently validate them before sharing. +``` + +For an interrupted session, append the exact last successful output directory, +review JSON path, and unresolved choice. Codex must re-audit those artifacts and +must not trust the stated phase without checking manifests. + +## Review Iteration + +Use review outcomes in two ways: + +- Delivery correction: reviewed false positives, misses, and manual boundaries + directly determine the current sample's final masks. +- Algorithm improvement: aggregate failure classes across completed samples, + change detector parameters in a new named profile, and rerun precision plus + whole-slide recall. Do not tune against one clicked object and claim a global + detector improvement. + +Before changing shared segmentation behavior, compare the proposed profile with +the frozen baseline on all reviewed donor13 sections and all donor11 negative +controls. Record count deltas, gained and lost reviewed objects, duplicate +behavior, and boundary regressions. + +For the initial publication PR, algorithm improvement is explicitly deferred. +Review exports are included as provenance in the release package, not converted +into a learned model or global parameter update. A colleague may open a new +issue for optimization after reproducing the current reference workflow. + +## Delivery Contract + +The release builder consumes a YAML or JSON spec and creates an immutable tree: + +```text +release/ + release_manifest.json + batch_summary.csv + batch_oocyte_by_marker.csv + batch_oocyte_metadata.csv + samples// + final/oocyte_labels.ome.tiff + final/oocyte_labels.csv + final/oocyte_candidates.csv + final/masks/*.npz + profiling/*.csv + profiling/profiling_manifest.json + review/review_manifest.json + review_console.html + sample_release_manifest.json +``` + +`release_manifest.json` is the authority for file hashes and sample totals. +Validation must confirm label-to-mapping pixel counts, unique IDs, profiling ID +alignment, uniform marker schemas, all package hashes, positive sample counts, +and zero labels plus zero accepted diagnostics for negative controls. + +## Commands + +```bash +python src/run_oocyte_release.py build \ + --spec exps/configs/oocyte_d11_d13_panel1/release_v6.yaml \ + --out-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_delivery_v6 + +python src/run_oocyte_release.py validate \ + --release-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_delivery_v6 + +python -m unittest discover -s tests/oocyte -v +``` + +When handing work to another Codex session, provide the active issue, exact +sample IDs, current versioned output directories, exported review JSON paths, +known unresolved choices, and the validation command. Do not describe an +unreviewed detector directory as a final delivery. diff --git a/aegle/oocyte/__init__.py b/aegle/oocyte/__init__.py new file mode 100644 index 0000000..13a37e9 --- /dev/null +++ b/aegle/oocyte/__init__.py @@ -0,0 +1,222 @@ +"""Standalone raw-UCHL1 oocyte detection.""" + +from .batch import ( + OocyteBatchResult, + OocyteSampleInput, + detect_oocyte_batch, + load_sample_manifest, +) +from .config import ( + DONOR13_V6, + DONOR13_V6_RESCUE_V1, + OOCYTE_IMPLEMENTATION_VERSION, + OocyteDetectionConfig, + available_profiles, + get_profile, +) +from .detection import ( + CoarseDetectionResult, + OocyteDetectionResult, + RefinedDetectionResult, + candidate_score, + detect_coarse_candidates, + detect_oocytes, + refine_candidates_from_array, + scan_coarse_candidates, + scan_refined_candidates, +) +from .delta import RescueDeltaBatchResult, generate_rescue_delta_batch +from .io import ( + find_channel_index, + load_candidate_mask, + read_ome_channel_patch, + save_candidate_mask, +) +from .manual_seed_finalize import ( + MANUAL_SEED_PROFILE_NAME, + ManualSeedFinalizeResult, + finalize_manual_seed_review, +) +from .export import LabelExportResult, export_whole_slide_labels +from .models import ( + BoundingBox, + LocalSegmentationResult, + ScoredCandidateMask, + SegmentationMetrics, +) +from .review import ReviewPackResult, generate_review_pack +from .report import HtmlReportResult, algorithm_document_html, generate_html_reports +from .release import ( + OOCYTE_RELEASE_IMPLEMENTATION_VERSION, + OOCYTE_RELEASE_SCHEMA_VERSION, + OocyteReleaseResult, + OocyteReleaseSample, + OocyteReleaseSpec, + build_oocyte_release, + load_oocyte_release_spec, + validate_oocyte_release, +) +from .recall_review import ( + RecallReviewBundle, + RecallReviewRuntime, + analyze_recall_review, + classify_recall_failure, + generate_recall_review_bundle, + serve_recall_review, +) +from .recall_review_batch import ( + BATCH_REVIEW_SCHEMA_VERSION, + BatchRecallReviewBundle, + generate_batch_recall_review_bundle, + serve_batch_recall_review, +) +from .recall_overlay import ( + REVIEWED_OVERLAY_CANDIDATE_FILES, + RecallMaskOverlay, + load_recall_mask_overlay, + overlay_dir_from_identity, +) +from .qc import ( + BatchSpatialQcResult, + SpatialQcResult, + accepted_duplicate_suspects, + compile_batch_spatial_qc, + render_spatial_overview, +) +from .segmentation import segment_oocyte_patch +from .profiling import ( + OOCYTE_PROFILING_VERSION, + OocyteProfilingResult, + profile_oocyte_labels, +) +from .precision_boundary_review import ( + BOUNDARY_RECOVERY_PARAMETERS, + PrecisionBoundaryReviewResult, + generate_precision_boundary_review, +) +from .precision_boundary_finalize import ( + PRECISION_BOUNDARY_CHOICES, + PRECISION_RESOLVED_PROFILE_NAME, + PrecisionBoundaryFinalizeResult, + finalize_precision_boundary_review, +) +from .precision_manual_boundary_review import ( + PRECISION_MANUAL_BOUNDARY_RENDERER_VERSION, + PRECISION_MANUAL_BOUNDARY_REVIEW_SCHEMA_VERSION, + PrecisionManualBoundaryReviewResult, + generate_precision_manual_boundary_review, +) +from .precision_manual_boundary_finalize import ( + MANUAL_CONTOUR_MAX_DIAMETER_UM, + MANUAL_CONTOUR_MIN_DIAMETER_UM, + PRECISION_MANUAL_BOUNDARY_CHOICES, + PRECISION_RESOLVED_V2_PROFILE_NAME, + PrecisionManualBoundaryFinalizeResult, + finalize_precision_manual_boundary_review, +) +from .shape_recovery_review import ( + SHAPE_RECOVERY_PARAMETERS, + ShapeRecoveryReviewResult, + generate_shape_recovery_review, +) +from .shape_recovery_finalize import ( + MANUAL_SEED_V2_PROFILE_NAME, + SHAPE_REVIEW_CHOICES, + finalize_shape_recovery_review, +) + +__all__ = [ + "DONOR13_V6", + "DONOR13_V6_RESCUE_V1", + "OocyteBatchResult", + "OOCYTE_IMPLEMENTATION_VERSION", + "OOCYTE_PROFILING_VERSION", + "OOCYTE_RELEASE_IMPLEMENTATION_VERSION", + "OOCYTE_RELEASE_SCHEMA_VERSION", + "OocyteDetectionConfig", + "BoundingBox", + "BATCH_REVIEW_SCHEMA_VERSION", + "BatchRecallReviewBundle", + "BatchSpatialQcResult", + "BOUNDARY_RECOVERY_PARAMETERS", + "CoarseDetectionResult", + "LocalSegmentationResult", + "LabelExportResult", + "MANUAL_SEED_PROFILE_NAME", + "MANUAL_SEED_V2_PROFILE_NAME", + "MANUAL_CONTOUR_MAX_DIAMETER_UM", + "MANUAL_CONTOUR_MIN_DIAMETER_UM", + "ManualSeedFinalizeResult", + "HtmlReportResult", + "OocyteDetectionResult", + "OocyteProfilingResult", + "OocyteReleaseResult", + "OocyteReleaseSample", + "OocyteReleaseSpec", + "OocyteSampleInput", + "PRECISION_BOUNDARY_CHOICES", + "PRECISION_MANUAL_BOUNDARY_CHOICES", + "PRECISION_MANUAL_BOUNDARY_RENDERER_VERSION", + "PRECISION_MANUAL_BOUNDARY_REVIEW_SCHEMA_VERSION", + "PRECISION_RESOLVED_PROFILE_NAME", + "PRECISION_RESOLVED_V2_PROFILE_NAME", + "PrecisionBoundaryFinalizeResult", + "PrecisionBoundaryReviewResult", + "PrecisionManualBoundaryFinalizeResult", + "PrecisionManualBoundaryReviewResult", + "RefinedDetectionResult", + "RecallReviewBundle", + "RecallReviewRuntime", + "RecallMaskOverlay", + "REVIEWED_OVERLAY_CANDIDATE_FILES", + "RescueDeltaBatchResult", + "ReviewPackResult", + "ScoredCandidateMask", + "SHAPE_REVIEW_CHOICES", + "SHAPE_RECOVERY_PARAMETERS", + "ShapeRecoveryReviewResult", + "SpatialQcResult", + "SegmentationMetrics", + "available_profiles", + "accepted_duplicate_suspects", + "analyze_recall_review", + "build_oocyte_release", + "candidate_score", + "classify_recall_failure", + "compile_batch_spatial_qc", + "detect_coarse_candidates", + "detect_oocyte_batch", + "detect_oocytes", + "export_whole_slide_labels", + "finalize_manual_seed_review", + "finalize_precision_boundary_review", + "finalize_precision_manual_boundary_review", + "finalize_shape_recovery_review", + "find_channel_index", + "get_profile", + "generate_review_pack", + "generate_rescue_delta_batch", + "generate_html_reports", + "generate_batch_recall_review_bundle", + "generate_recall_review_bundle", + "generate_precision_boundary_review", + "generate_precision_manual_boundary_review", + "generate_shape_recovery_review", + "algorithm_document_html", + "load_candidate_mask", + "load_oocyte_release_spec", + "load_recall_mask_overlay", + "load_sample_manifest", + "profile_oocyte_labels", + "overlay_dir_from_identity", + "read_ome_channel_patch", + "refine_candidates_from_array", + "render_spatial_overview", + "save_candidate_mask", + "scan_coarse_candidates", + "scan_refined_candidates", + "segment_oocyte_patch", + "serve_recall_review", + "serve_batch_recall_review", + "validate_oocyte_release", +] diff --git a/aegle/oocyte/batch.py b/aegle/oocyte/batch.py new file mode 100644 index 0000000..bf81a48 --- /dev/null +++ b/aegle/oocyte/batch.py @@ -0,0 +1,471 @@ +"""Manifest-driven standalone oocyte batch execution.""" + +from __future__ import annotations + +import json +import os +import tempfile +import time +import traceback +from concurrent.futures import ProcessPoolExecutor, as_completed +from dataclasses import asdict, dataclass, replace +from pathlib import Path +from typing import Any, Dict, List + +import pandas as pd + +from .config import OOCYTE_IMPLEMENTATION_VERSION, get_profile +from .detection import detect_oocytes +from .io import find_channel_index +from .qc import compile_batch_spatial_qc + + +@dataclass(frozen=True) +class OocyteSampleInput: + sample_id: str + image_path: Path + pixel_size_um: float + enabled: bool + profile: str + channel_index: int | None + channel_name: str + antibodies_path: Path | None + + +@dataclass(frozen=True) +class OocyteBatchResult: + manifest_path: Path + out_dir: Path + summary: pd.DataFrame + artifact_paths: Dict[str, Path] + + @property + def failed_count(self) -> int: + return int((self.summary["status"] == "failed").sum()) + + +def _parse_bool(value: Any, *, default: bool) -> bool: + if value is None or (isinstance(value, float) and pd.isna(value)): + return default + if isinstance(value, bool): + return value + text = str(value).strip().casefold() + if text in {"1", "true", "yes", "y"}: + return True + if text in {"0", "false", "no", "n"}: + return False + raise ValueError(f"invalid boolean value: {value!r}") + + +def _optional_text(value: Any) -> str | None: + if value is None or (isinstance(value, float) and pd.isna(value)): + return None + text = str(value).strip() + return text or None + + +def _optional_int(value: Any) -> int | None: + if value is None or (isinstance(value, float) and pd.isna(value)): + return None + number = float(value) + if not number.is_integer(): + raise ValueError(f"expected an integer, got {value!r}") + return int(number) + + +def _resolve_path(value: Any, manifest_dir: Path) -> Path | None: + text = _optional_text(value) + if text is None: + return None + path = Path(text).expanduser() + if not path.is_absolute(): + path = manifest_dir / path + return path.resolve() + + +def load_sample_manifest( + manifest_path: Path, + *, + default_profile: str = "donor13_v6", + default_channel_name: str = "UCHL1", +) -> List[OocyteSampleInput]: + """Parse and structurally validate an oocyte sample CSV manifest.""" + + path = Path(manifest_path).resolve() + table = pd.read_csv(path, keep_default_na=True) + required = {"sample_id", "image_path", "pixel_size_um"} + missing = required.difference(table.columns) + if missing: + raise ValueError(f"sample manifest missing columns: {sorted(missing)}") + sample_ids = table["sample_id"].astype(str).str.strip() + if (sample_ids == "").any(): + raise ValueError("sample manifest contains an empty sample_id") + duplicates = sorted(sample_ids[sample_ids.duplicated()].unique()) + if duplicates: + raise ValueError(f"sample manifest contains duplicate sample IDs: {duplicates}") + + rows = [] + for row_number, record in enumerate(table.to_dict("records"), start=2): + try: + sample_id = str(record["sample_id"]).strip() + image_path = _resolve_path(record["image_path"], path.parent) + if image_path is None: + raise ValueError("image_path is empty") + pixel_size_um = float(record["pixel_size_um"]) + if pixel_size_um <= 0: + raise ValueError("pixel_size_um must be positive") + enabled = _parse_bool(record.get("enabled"), default=True) + profile = _optional_text(record.get("profile")) or default_profile + get_profile(profile) + channel_index = _optional_int(record.get("channel_index")) + if channel_index is not None and channel_index < 0: + raise ValueError("channel_index must not be negative") + channel_name = ( + _optional_text(record.get("channel_name")) or default_channel_name + ) + antibodies_path = _resolve_path( + record.get("antibodies_path"), + path.parent, + ) + if enabled and channel_index is None and antibodies_path is None: + raise ValueError( + "enabled rows require channel_index or antibodies_path" + ) + except (TypeError, ValueError) as exc: + raise ValueError(f"invalid sample manifest row {row_number}: {exc}") from exc + rows.append( + OocyteSampleInput( + sample_id=sample_id, + image_path=image_path, + pixel_size_um=pixel_size_um, + enabled=enabled, + profile=profile, + channel_index=channel_index, + channel_name=channel_name, + antibodies_path=antibodies_path, + ) + ) + return rows + + +def _complete_record( + sample: OocyteSampleInput, + sample_out_dir: Path, +) -> Dict[str, Any]: + started = time.perf_counter() + try: + result = detect_oocytes( + sample.image_path, + sample_id=sample.sample_id, + out_dir=sample_out_dir, + config=get_profile(sample.profile), + channel_name=sample.channel_name, + channel_index=sample.channel_index, + antibodies_path=sample.antibodies_path, + pixel_size_um=sample.pixel_size_um, + ) + return { + "sample_id": sample.sample_id, + "status": "complete", + "profile": result.profile_name, + "profile_fingerprint": result.profile_fingerprint, + "implementation_version": result.implementation_version, + "image_path": str(result.image_path), + "image_height": int(result.image_shape_yx[0]), + "image_width": int(result.image_shape_yx[1]), + "channel_index": int(result.channel_index), + "coarse_candidate_count": int(len(result.coarse_candidates)), + "refined_candidate_count": int(len(result.candidates)), + "accepted_candidate_count": int(result.candidates["accepted"].sum()) + if not result.candidates.empty + else 0, + "runtime_seconds": float(result.runtime_seconds["total"]), + "output_dir": str(sample_out_dir), + "candidates_path": str(result.artifact_paths["candidates"]), + "labels_path": str(result.artifact_paths["labels"]), + "error_type": "", + "error_message": "", + "traceback": "", + "resumed": False, + } + except Exception as exc: + return { + "sample_id": sample.sample_id, + "status": "failed", + "profile": sample.profile, + "profile_fingerprint": "", + "implementation_version": OOCYTE_IMPLEMENTATION_VERSION, + "image_path": str(sample.image_path), + "image_height": None, + "image_width": None, + "channel_index": sample.channel_index, + "coarse_candidate_count": None, + "refined_candidate_count": None, + "accepted_candidate_count": None, + "runtime_seconds": float(time.perf_counter() - started), + "output_dir": str(sample_out_dir), + "candidates_path": "", + "labels_path": "", + "error_type": type(exc).__name__, + "error_message": str(exc), + "traceback": traceback.format_exc(), + "resumed": False, + } + + +def _skipped_record(sample: OocyteSampleInput, sample_out_dir: Path) -> Dict[str, Any]: + return { + "sample_id": sample.sample_id, + "status": "skipped", + "profile": sample.profile, + "profile_fingerprint": "", + "implementation_version": OOCYTE_IMPLEMENTATION_VERSION, + "image_path": str(sample.image_path), + "image_height": None, + "image_width": None, + "channel_index": sample.channel_index, + "coarse_candidate_count": None, + "refined_candidate_count": None, + "accepted_candidate_count": None, + "runtime_seconds": 0.0, + "output_dir": str(sample_out_dir), + "candidates_path": "", + "labels_path": "", + "error_type": "", + "error_message": "disabled in manifest", + "traceback": "", + "resumed": False, + } + + +def _resume_record( + sample: OocyteSampleInput, + sample_out_dir: Path, +) -> Dict[str, Any] | None: + summary_path = sample_out_dir / "summary.json" + manifest_path = sample_out_dir / "run_manifest.json" + candidates_path = sample_out_dir / "candidates.csv" + labels_path = sample_out_dir / "oocyte_labels.ome.tiff" + if not all( + path.is_file() + for path in (summary_path, manifest_path, candidates_path, labels_path) + ): + return None + try: + summary = json.loads(summary_path.read_text()) + manifest = json.loads(manifest_path.read_text()) + config = get_profile(sample.profile) + if sample.pixel_size_um != config.pixel_size_um: + config = replace(config, pixel_size_um=sample.pixel_size_um) + expected_channel_index = sample.channel_index + if expected_channel_index is None: + if sample.antibodies_path is None: + return None + expected_channel_index = find_channel_index( + sample.antibodies_path, + sample.channel_name, + ) + if str(Path(manifest["source_image"]).resolve()) != str( + sample.image_path.resolve() + ): + return None + if manifest.get("profile_fingerprint") != config.fingerprint(): + return None + if manifest.get("implementation_version") != OOCYTE_IMPLEMENTATION_VERSION: + return None + if int(manifest["resolved_channel_index"]) != int(expected_channel_index): + return None + if summary.get("status") != "complete": + return None + return { + "sample_id": sample.sample_id, + "status": "complete", + "profile": str(summary["profile_name"]), + "profile_fingerprint": str(summary["profile_fingerprint"]), + "implementation_version": str(summary["implementation_version"]), + "image_path": str(sample.image_path), + "image_height": int(summary["image_shape_yx"][0]), + "image_width": int(summary["image_shape_yx"][1]), + "channel_index": int(summary["channel_index"]), + "coarse_candidate_count": int(summary["coarse_candidate_count"]), + "refined_candidate_count": int(summary["refined_candidate_count"]), + "accepted_candidate_count": int(summary["accepted_candidate_count"]), + "runtime_seconds": float(summary["runtime_seconds"]["total"]), + "output_dir": str(sample_out_dir), + "candidates_path": str(candidates_path), + "labels_path": str(labels_path), + "error_type": "", + "error_message": "", + "traceback": "", + "resumed": True, + } + except (KeyError, OSError, TypeError, ValueError, json.JSONDecodeError): + return None + + +def _atomic_write_json(payload: Any, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".json", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + json.dump(payload, handle, indent=2, sort_keys=True, default=str) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _atomic_write_csv(table: pd.DataFrame, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".csv", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + table.to_csv(handle, index=False) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _resolved_manifest_row(sample: OocyteSampleInput) -> Dict[str, Any]: + row = asdict(sample) + row["image_path"] = str(sample.image_path) + row["antibodies_path"] = ( + None if sample.antibodies_path is None else str(sample.antibodies_path) + ) + return row + + +def detect_oocyte_batch( + manifest_path: Path, + *, + out_dir: Path, + jobs: int = 1, + continue_on_error: bool = True, + resume_completed: bool = True, + default_profile: str = "donor13_v6", + default_channel_name: str = "UCHL1", +) -> OocyteBatchResult: + """Run all enabled manifest rows and preserve one status per input sample.""" + + samples = load_sample_manifest( + manifest_path, + default_profile=default_profile, + default_channel_name=default_channel_name, + ) + destination = Path(out_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + enabled = [sample for sample in samples if sample.enabled] + records_by_sample = { + sample.sample_id: _skipped_record(sample, destination / sample.sample_id) + for sample in samples + if not sample.enabled + } + pending = [] + for sample in enabled: + resumed = ( + _resume_record(sample, destination / sample.sample_id) + if resume_completed + else None + ) + if resumed is None: + pending.append(sample) + else: + records_by_sample[sample.sample_id] = resumed + worker_count = max(1, int(jobs)) + worker_count = min(worker_count, len(pending)) if pending else 1 + + if worker_count == 1: + for sample in pending: + record = _complete_record(sample, destination / sample.sample_id) + records_by_sample[sample.sample_id] = record + if record["status"] == "failed" and not continue_on_error: + break + else: + with ProcessPoolExecutor(max_workers=worker_count) as executor: + futures = { + executor.submit( + _complete_record, + sample, + destination / sample.sample_id, + ): sample + for sample in pending + } + stop_requested = False + for future in as_completed(futures): + sample = futures[future] + record = future.result() + records_by_sample[sample.sample_id] = record + if record["status"] == "failed" and not continue_on_error: + stop_requested = True + for pending_future in futures: + pending_future.cancel() + break + if stop_requested: + for sample in pending: + if sample.sample_id not in records_by_sample: + records_by_sample[sample.sample_id] = { + **_skipped_record(sample, destination / sample.sample_id), + "error_message": "not run after fail-fast error", + } + + for sample in pending: + if sample.sample_id not in records_by_sample: + records_by_sample[sample.sample_id] = { + **_skipped_record(sample, destination / sample.sample_id), + "error_message": "not run after fail-fast error", + } + records = [records_by_sample[sample.sample_id] for sample in samples] + summary = pd.DataFrame(records) + summary_csv = destination / "batch_summary.csv" + summary_json = destination / "batch_summary.json" + resolved_manifest = destination / "batch_manifest.json" + _atomic_write_csv(summary, summary_csv) + _atomic_write_json(records, summary_json) + _atomic_write_json( + { + "schema_version": 1, + "source_manifest": str(Path(manifest_path).resolve()), + "jobs": worker_count, + "continue_on_error": bool(continue_on_error), + "resume_completed": bool(resume_completed), + "samples": [_resolved_manifest_row(sample) for sample in samples], + }, + resolved_manifest, + ) + spatial_qc = compile_batch_spatial_qc(destination, summary) + artifact_paths = { + "batch_summary_csv": summary_csv, + "batch_summary_json": summary_json, + "batch_manifest": resolved_manifest, + "spatial_qc_dir": spatial_qc.qc_dir, + "spatial_qc_index": spatial_qc.overview_index_path, + "duplicate_suspects": spatial_qc.duplicate_suspects_path, + } + if spatial_qc.overview_atlas_path is not None: + artifact_paths["spatial_qc_atlas"] = spatial_qc.overview_atlas_path + return OocyteBatchResult( + manifest_path=Path(manifest_path).resolve(), + out_dir=destination, + summary=summary, + artifact_paths=artifact_paths, + ) diff --git a/aegle/oocyte/config.py b/aegle/oocyte/config.py new file mode 100644 index 0000000..5db2009 --- /dev/null +++ b/aegle/oocyte/config.py @@ -0,0 +1,354 @@ +"""Versioned configuration for standalone oocyte detection.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass +from typing import Any, Dict, Optional, Tuple + + +OOCYTE_IMPLEMENTATION_VERSION = "donor13_v6_engineering_1" + + +def _positive(name: str, value: float) -> None: + if value <= 0: + raise ValueError(f"{name} must be positive, got {value}") + + +def _ordered_pair(name: str, values: Tuple[float, float]) -> None: + if values[0] >= values[1]: + raise ValueError(f"{name} must be increasing, got {values}") + + +@dataclass(frozen=True) +class CoarseDetectionConfig: + downsample_factor: int = 8 + strip_height: int = 1024 + gaussian_sigma_small: float = 1.0 + gaussian_sigma_large: float = 10.0 + intensity_percentile: float = 99.9 + intensity_floor_percentile: float = 99.5 + intensity_floor_multiplier: float = 1.2 + contrast_percentile: float = 99.9 + min_component_area_ds: int = 20 + closing_radius_ds: int = 2 + fallback_min_candidate_count: int = 5 + min_diameter_um: float = 15.0 + max_diameter_um: float = 120.0 + max_peaks_per_component: int = 8 + peak_min_distance_ds: int = 6 + peak_percentile: float = 92.0 + global_peak_min_distance_ds: int = 6 + global_peak_max_count: int = 5000 + blob_threshold: float = 0.12 + blob_min_sigma_ds: float = 2.5 + blob_max_sigma_ds: float = 5.5 + blob_num_sigma: int = 5 + blob_overlap: float = 0.5 + blob_response_sigma_ds: float = 0.8 + blob_intensity_percentile: float = 99.5 + blob_contrast_percentile: float = 99.5 + blob_intensity_relax_multiplier: float = 0.35 + blob_contrast_relax_multiplier: float = 0.55 + blob_max_new_candidates: int = 120 + seed_merge_distance_px: float = 60.0 + + def __post_init__(self) -> None: + _positive("downsample_factor", self.downsample_factor) + _positive("strip_height", self.strip_height) + if self.gaussian_sigma_small >= self.gaussian_sigma_large: + raise ValueError("coarse Gaussian sigmas must satisfy small < large") + if self.min_diameter_um >= self.max_diameter_um: + raise ValueError("coarse diameter bounds must satisfy min < max") + if self.blob_min_sigma_ds > self.blob_max_sigma_ds: + raise ValueError("blob sigma bounds must satisfy min <= max") + + +@dataclass(frozen=True) +class LocalSegmentationConfig: + window_radius_px: int = 180 + threshold_method: str = "triangle" + gaussian_sigma: float = 1.0 + annulus_inner_px: int = 80 + annulus_outer_px: int = 170 + annulus_floor_percentile: float = 99.0 + annulus_floor_multiplier: float = 1.0 + min_component_size_px: int = 200 + closing_radius_px: int = 3 + max_peak_seeds_per_candidate: int = 8 + peak_gaussian_sigma: float = 1.0 + peak_search_radius_px: int = 175 + peak_min_distance_px: int = 24 + peak_percentile: float = 98.5 + max_broad_peak_seeds_per_candidate: int = 3 + broad_peak_gaussian_sigma: float = 8.0 + broad_peak_min_distance_px: int = 36 + broad_peak_percentile: float = 99.0 + max_offset_ring_seeds: int = 8 + offset_ring_step_px: int = 24 + seed_merge_radius_px: float = 12.0 + max_centroid_reseed_iterations: int = 1 + centroid_reseed_min_shift_px: float = 8.0 + centroid_reseed_max_shift_px: float = 80.0 + compact_window_radius_px: int = 140 + compact_annulus_inner_px: int = 60 + compact_annulus_outer_px: int = 120 + compact_retry_max_brightness_ratio: float = 2.5 + max_instances_per_coarse_candidate: int = 6 + instance_merge_distance_px: float = 50.0 + + def __post_init__(self) -> None: + if self.threshold_method not in {"triangle", "yen"}: + raise ValueError(f"unsupported threshold method: {self.threshold_method}") + if not 0 < self.annulus_inner_px < self.annulus_outer_px < self.window_radius_px: + raise ValueError("default annulus must fit inside the local window") + if not ( + 0 + < self.compact_annulus_inner_px + < self.compact_annulus_outer_px + < self.compact_window_radius_px + ): + raise ValueError("compact annulus must fit inside the compact window") + if self.compact_window_radius_px >= self.window_radius_px: + raise ValueError("compact window must be smaller than the default window") + if self.centroid_reseed_min_shift_px >= self.centroid_reseed_max_shift_px: + raise ValueError("centroid reseed bounds must satisfy min < max") + + +@dataclass(frozen=True) +class ScoringConfig: + size_core_range_um: Tuple[float, float] = (25.0, 80.0) + size_soft_range_um: Tuple[float, float] = (15.0, 110.0) + circularity_floor: float = 0.45 + circularity_span: float = 0.40 + solidity_floor: float = 0.80 + solidity_span: float = 0.18 + brightness_ratio_floor: float = 3.0 + brightness_ratio_span: float = 12.0 + centered_offset_limit_px: float = 40.0 + brightness_weight: float = 0.35 + shape_weight: float = 0.30 + size_weight: float = 0.20 + centered_weight: float = 0.15 + shape_circularity_weight: float = 0.60 + shape_solidity_weight: float = 0.40 + + def __post_init__(self) -> None: + _ordered_pair("size_core_range_um", self.size_core_range_um) + _ordered_pair("size_soft_range_um", self.size_soft_range_um) + if self.size_soft_range_um[0] > self.size_core_range_um[0]: + raise ValueError("soft size range must include the core lower bound") + if self.size_soft_range_um[1] < self.size_core_range_um[1]: + raise ValueError("soft size range must include the core upper bound") + total = self.brightness_weight + self.shape_weight + self.size_weight + self.centered_weight + if abs(total - 1.0) > 1e-9: + raise ValueError(f"candidate score weights must sum to 1, got {total}") + shape_total = self.shape_circularity_weight + self.shape_solidity_weight + if abs(shape_total - 1.0) > 1e-9: + raise ValueError(f"shape score weights must sum to 1, got {shape_total}") + + +@dataclass(frozen=True) +class AcceptanceConfig: + score_threshold: float = 0.45 + strict_min_diameter_um: float = 15.0 + rescue_min_diameter_um: float = 24.0 + rescue_max_diameter_um: float = 60.0 + rescue_min_circularity: float = 0.48 + rescue_min_solidity: float = 0.84 + rescue_min_brightness_ratio: float = 1.75 + rescue_max_offset_px: float = 12.0 + + def __post_init__(self) -> None: + if not 0.0 <= self.score_threshold <= 1.0: + raise ValueError("acceptance score threshold must be in [0, 1]") + if self.rescue_min_diameter_um >= self.rescue_max_diameter_um: + raise ValueError("rescue diameter bounds must satisfy min < max") + + +@dataclass(frozen=True) +class DeduplicationConfig: + refined_center_distance_px: float = 60.0 + + +@dataclass(frozen=True) +class ComparisonConfig: + reference_min_final_score: float = 0.35 + match_radius_px: float = 100.0 + + +@dataclass(frozen=True) +class RuntimeConfig: + automatic_sample_worker_cap: int = 4 + + +@dataclass(frozen=True) +class ExperimentalConfig: + border_rescue_enabled: bool = False + + +@dataclass(frozen=True) +class SecondaryRescueConfig: + """Conservative second pass for crowded fields missed by the v6 refinement.""" + + discovery_window_radius_px: int = 240 + discovery_annulus_inner_px: int = 170 + discovery_annulus_outer_px: int = 230 + annulus_floor_percentile: float = 95.0 + compact_relaxed_annulus_floor_percentile: float = 80.0 + brightness_reference_percentile: float = 95.0 + discovery_min_diameter_um: float = 14.0 + discovery_max_diameter_um: float = 115.0 + discovery_min_circularity: float = 0.30 + discovery_min_solidity: float = 0.70 + final_min_diameter_um: float = 18.0 + final_max_diameter_um: float = 100.0 + final_min_circularity: float = 0.65 + final_min_solidity: float = 0.88 + final_min_brightness_ratio: float = 1.10 + final_min_max_intensity: float = 1500.0 + final_score_threshold: float = 0.45 + low_intensity_shape_max_intensity: float = 3000.0 + low_intensity_shape_max_eccentricity: float = 0.82 + small_candidate_diameter_um: float = 20.0 + small_candidate_min_score: float = 0.48 + small_candidate_min_max_intensity: float = 2000.0 + bright_irregular_min_diameter_um: float = 20.0 + bright_irregular_max_diameter_um: float = 60.0 + bright_irregular_min_circularity: float = 0.58 + bright_irregular_min_solidity: float = 0.82 + bright_irregular_min_max_intensity: float = 6000.0 + bright_irregular_min_score: float = 0.43 + bright_irregular_max_centroid_offset_px: float = 50.0 + bright_fragment_min_diameter_um: float = 18.0 + bright_fragment_max_diameter_um: float = 30.0 + bright_fragment_min_circularity: float = 0.40 + bright_fragment_min_solidity: float = 0.70 + bright_fragment_min_max_intensity: float = 8000.0 + bright_fragment_min_brightness_ratio: float = 5.0 + bright_fragment_min_score: float = 0.38 + bright_fragment_max_centroid_offset_px: float = 15.0 + baseline_fallback_min_diameter_um: float = 20.0 + baseline_fallback_max_diameter_um: float = 60.0 + baseline_fallback_min_circularity: float = 0.60 + baseline_fallback_min_solidity: float = 0.85 + baseline_fallback_min_max_intensity: float = 6000.0 + baseline_fallback_max_centroid_offset_px: float = 40.0 + max_component_offset_px: float = 205.0 + final_max_centroid_offset_px: float = 60.0 + discovery_seed_merge_distance_px: float = 20.0 + duplicate_centroid_distance_px: float = 50.0 + duplicate_mask_overlap_fraction: float = 0.25 + max_components_per_coarse_candidate: int = 12 + + def __post_init__(self) -> None: + if not ( + 0 + < self.discovery_annulus_inner_px + < self.discovery_annulus_outer_px + < self.discovery_window_radius_px + ): + raise ValueError("rescue discovery annulus must fit inside its window") + if not 0.0 < self.annulus_floor_percentile <= 100.0: + raise ValueError("rescue annulus percentile must be in (0, 100]") + if not 0.0 < self.compact_relaxed_annulus_floor_percentile <= 100.0: + raise ValueError("rescue compact-relaxed percentile must be in (0, 100]") + if not 0.0 < self.brightness_reference_percentile <= 100.0: + raise ValueError("rescue brightness percentile must be in (0, 100]") + if self.discovery_min_diameter_um >= self.discovery_max_diameter_um: + raise ValueError("rescue discovery diameter bounds must satisfy min < max") + if self.final_min_diameter_um >= self.final_max_diameter_um: + raise ValueError("rescue final diameter bounds must satisfy min < max") + if ( + self.bright_irregular_min_diameter_um + >= self.bright_irregular_max_diameter_um + ): + raise ValueError("bright-irregular diameter bounds must satisfy min < max") + if self.bright_fragment_min_diameter_um >= self.bright_fragment_max_diameter_um: + raise ValueError("bright-fragment diameter bounds must satisfy min < max") + if ( + self.baseline_fallback_min_diameter_um + >= self.baseline_fallback_max_diameter_um + ): + raise ValueError("baseline-fallback diameter bounds must satisfy min < max") + if not 0.0 <= self.final_score_threshold <= 1.0: + raise ValueError("rescue score threshold must be in [0, 1]") + if not 0.0 <= self.duplicate_mask_overlap_fraction <= 1.0: + raise ValueError("rescue mask-overlap threshold must be in [0, 1]") + for name, value in ( + ("final circularity", self.final_min_circularity), + ("final solidity", self.final_min_solidity), + ("low-intensity eccentricity", self.low_intensity_shape_max_eccentricity), + ("bright-irregular circularity", self.bright_irregular_min_circularity), + ("bright-irregular solidity", self.bright_irregular_min_solidity), + ("bright-fragment circularity", self.bright_fragment_min_circularity), + ("bright-fragment solidity", self.bright_fragment_min_solidity), + ("baseline-fallback circularity", self.baseline_fallback_min_circularity), + ("baseline-fallback solidity", self.baseline_fallback_min_solidity), + ): + if not 0.0 <= value <= 1.0: + raise ValueError(f"rescue {name} must be in [0, 1]") + _positive("rescue max components", self.max_components_per_coarse_candidate) + + +@dataclass(frozen=True) +class OocyteDetectionConfig: + profile_name: str = "donor13_v6" + schema_version: int = 1 + pixel_size_um: float = 0.5 + coarse: CoarseDetectionConfig = CoarseDetectionConfig() + local: LocalSegmentationConfig = LocalSegmentationConfig() + scoring: ScoringConfig = ScoringConfig() + acceptance: AcceptanceConfig = AcceptanceConfig() + deduplication: DeduplicationConfig = DeduplicationConfig() + comparison: ComparisonConfig = ComparisonConfig() + runtime: RuntimeConfig = RuntimeConfig() + experimental: ExperimentalConfig = ExperimentalConfig() + secondary_rescue: Optional[SecondaryRescueConfig] = None + + def __post_init__(self) -> None: + if not self.profile_name: + raise ValueError("profile_name must not be empty") + _positive("schema_version", self.schema_version) + _positive("pixel_size_um", self.pixel_size_um) + if self.experimental.border_rescue_enabled: + raise ValueError("donor13_v6 does not support experimental border rescue") + + def to_dict(self) -> Dict[str, Any]: + payload = asdict(self) + # Keep the frozen donor13_v6 canonical JSON and fingerprint unchanged. + if self.secondary_rescue is None: + payload.pop("secondary_rescue") + return payload + + def canonical_json(self) -> str: + return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) + + def fingerprint(self) -> str: + return hashlib.sha256(self.canonical_json().encode("utf-8")).hexdigest() + + +DONOR13_V6 = OocyteDetectionConfig() +DONOR13_V6_RESCUE_V1 = OocyteDetectionConfig( + profile_name="donor13_v6_rescue_v1", + schema_version=2, + secondary_rescue=SecondaryRescueConfig(), +) + +_PROFILES = { + DONOR13_V6.profile_name: DONOR13_V6, + DONOR13_V6_RESCUE_V1.profile_name: DONOR13_V6_RESCUE_V1, +} + + +def available_profiles() -> Tuple[str, ...]: + return tuple(sorted(_PROFILES)) + + +def get_profile(name: str) -> OocyteDetectionConfig: + try: + return _PROFILES[name] + except KeyError as exc: + choices = ", ".join(available_profiles()) + raise ValueError(f"unknown oocyte detector profile {name!r}; choose from: {choices}") from exc diff --git a/aegle/oocyte/delta.py b/aegle/oocyte/delta.py new file mode 100644 index 0000000..63c3717 --- /dev/null +++ b/aegle/oocyte/delta.py @@ -0,0 +1,238 @@ +"""Incremental secondary-rescue artifacts built from a completed v6 run.""" + +from __future__ import annotations + +import json +import logging +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, List + +import pandas as pd +import tifffile +import zarr + +from .config import OOCYTE_IMPLEMENTATION_VERSION, get_profile +from .detection import _atomic_write_csv, _atomic_write_json, candidate_score +from .io import extract_cyx_channel_patch, load_candidate_mask, save_candidate_mask +from .models import ScoredCandidateMask, SegmentationMetrics +from .rescue import run_secondary_rescue + + +LOGGER = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class RescueDeltaBatchResult: + out_dir: Path + summary: pd.DataFrame + artifact_paths: Dict[str, Path] + + +def _load_baseline_masks( + sample_dir: Path, + candidates: pd.DataFrame, +) -> Dict[str, ScoredCandidateMask]: + masks = {} + for row in candidates.to_dict("records"): + candidate_id = str(row["detector_component_id"]) + persisted = load_candidate_mask(sample_dir / str(row["mask_path"])) + masks[candidate_id] = ScoredCandidateMask( + mask=persisted.mask, + bbox=persisted.bbox, + image_shape_yx=persisted.image_shape_yx, + metrics=SegmentationMetrics(**persisted.metadata["metrics"]), + ) + return masks + + +def generate_rescue_delta_batch( + baseline_batch_dir: Path, + *, + out_dir: Path, + profile_name: str = "donor13_v6_rescue_v1", + sample_ids: Iterable[str] | None = None, +) -> RescueDeltaBatchResult: + """Run only the secondary pass and persist a rescue-only review batch.""" + + baseline_root = Path(baseline_batch_dir).resolve() + destination = Path(out_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + profile = get_profile(profile_name) + if profile.secondary_rescue is None: + raise ValueError(f"profile {profile_name!r} has no secondary rescue pass") + baseline_summary_path = baseline_root / "batch_summary.csv" + if not baseline_summary_path.is_file(): + raise FileNotFoundError( + f"baseline batch summary not found: {baseline_summary_path}" + ) + baseline_summary = pd.read_csv(baseline_summary_path) + requested = None if sample_ids is None else {str(value) for value in sample_ids} + if requested is not None: + known = set(baseline_summary["sample_id"].astype(str)) + missing = sorted(requested - known) + if missing: + raise ValueError(f"sample IDs not present in baseline batch: {missing}") + + summary_rows: List[Dict[str, object]] = [] + for summary_row in baseline_summary.to_dict("records"): + sample_id = str(summary_row["sample_id"]) + if requested is not None and sample_id not in requested: + continue + if str(summary_row["status"]) != "complete": + summary_rows.append( + { + "sample_id": sample_id, + "status": "skipped_incomplete_baseline", + "rescue_candidate_count": 0, + } + ) + continue + LOGGER.info("Secondary rescue starting sample %s", sample_id) + started = time.perf_counter() + baseline_sample_dir = baseline_root / sample_id + sample_dir = destination / sample_id + masks_dir = sample_dir / "masks" + masks_dir.mkdir(parents=True, exist_ok=True) + manifest = json.loads( + (baseline_sample_dir / "run_manifest.json").read_text() + ) + image_path = Path(manifest["source_image"]) + channel_index = int(manifest["resolved_channel_index"]) + coarse = pd.read_csv(baseline_sample_dir / "coarse_candidates.csv") + baseline_candidates = pd.read_csv( + baseline_sample_dir / "candidates.csv" + ) + baseline_masks = _load_baseline_masks( + baseline_sample_dir, + baseline_candidates, + ) + with tifffile.TiffFile(image_path) as tif: + series = tif.series[0] + if series.axes != "CYX": + raise ValueError( + f"secondary rescue requires CYX axes, got {series.axes!r}" + ) + array = zarr.open(series.aszarr(), mode="r") + image_shape = (int(array.shape[1]), int(array.shape[2])) + + def patch_reader(center_x: int, center_y: int, radius: int): + return extract_cyx_channel_patch( + array, + channel_index, + (center_x, center_y), + radius, + ) + + result = run_secondary_rescue( + patch_reader=patch_reader, + image_shape_yx=image_shape, + coarse_candidates=coarse, + baseline_candidates=baseline_candidates, + baseline_masks=baseline_masks, + config=profile, + score_candidate=candidate_score, + ) + + rescue_candidates = result.candidates[ + result.candidates.get("segmentation_pass", pd.Series(dtype=str)) + == "secondary_rescue" + ].copy() + mask_paths = [] + for candidate_id in rescue_candidates[ + "detector_component_id" + ].astype(str): + relative_path = Path("masks") / f"{candidate_id}.npz" + candidate_mask = result.candidate_masks[candidate_id] + save_candidate_mask( + sample_dir / relative_path, + mask=candidate_mask.mask, + bbox=candidate_mask.bbox, + image_shape_yx=candidate_mask.image_shape_yx, + sample_id=sample_id, + candidate_id=candidate_id, + profile_name=profile.profile_name, + profile_fingerprint=profile.fingerprint(), + metrics=candidate_mask.metrics, + implementation_version=OOCYTE_IMPLEMENTATION_VERSION, + ) + mask_paths.append(str(relative_path)) + rescue_candidates["mask_path"] = mask_paths + _atomic_write_csv(rescue_candidates, sample_dir / "candidates.csv") + _atomic_write_csv(result.diagnostics, sample_dir / "rescue_diagnostics.csv") + run_seconds = float(time.perf_counter() - started) + sample_summary = { + "schema_version": 1, + "sample_id": sample_id, + "status": "complete", + "baseline_batch_dir": str(baseline_root), + "baseline_profile_name": str(manifest.get("resolved_config", {}).get("profile_name", "")), + "profile_name": profile.profile_name, + "profile_fingerprint": profile.fingerprint(), + "baseline_accepted_candidate_count": int( + baseline_candidates["accepted"].astype(bool).sum() + ), + "rescue_candidate_count": int(len(rescue_candidates)), + "rescue_diagnostic_count": int(len(result.diagnostics)), + "runtime_seconds": run_seconds, + } + _atomic_write_json(sample_summary, sample_dir / "summary.json") + _atomic_write_json( + { + "schema_version": 1, + "sample_id": sample_id, + "source_image": str(image_path), + "resolved_channel_index": channel_index, + "baseline_sample_dir": str(baseline_sample_dir), + "resolved_config": profile.to_dict(), + "profile_fingerprint": profile.fingerprint(), + "implementation_version": OOCYTE_IMPLEMENTATION_VERSION, + }, + sample_dir / "run_manifest.json", + ) + summary_rows.append( + { + "sample_id": sample_id, + "status": "complete", + "baseline_accepted_candidate_count": int( + baseline_candidates["accepted"].astype(bool).sum() + ), + "rescue_candidate_count": int(len(rescue_candidates)), + "rescue_diagnostic_count": int(len(result.diagnostics)), + "runtime_seconds": run_seconds, + "sample_dir": str(sample_dir), + } + ) + LOGGER.info( + "Secondary rescue completed sample %s: %s candidates in %.1fs", + sample_id, + len(rescue_candidates), + run_seconds, + ) + + summary = pd.DataFrame(summary_rows) + summary_csv = destination / "batch_summary.csv" + summary_json = destination / "batch_summary.json" + _atomic_write_csv(summary, summary_csv) + _atomic_write_json( + { + "schema_version": 1, + "baseline_batch_dir": str(baseline_root), + "profile_name": profile.profile_name, + "profile_fingerprint": profile.fingerprint(), + "samples": summary_rows, + }, + summary_json, + ) + return RescueDeltaBatchResult( + out_dir=destination, + summary=summary, + artifact_paths={ + "batch_summary_csv": summary_csv, + "batch_summary_json": summary_json, + }, + ) + + +__all__ = ["RescueDeltaBatchResult", "generate_rescue_delta_batch"] diff --git a/aegle/oocyte/detection.py b/aegle/oocyte/detection.py new file mode 100644 index 0000000..c14c2eb --- /dev/null +++ b/aegle/oocyte/detection.py @@ -0,0 +1,1622 @@ +"""Whole-slide raw-UCHL1 proposal generation and candidate detection.""" + +from __future__ import annotations + +import json +import os +import tempfile +import time +from dataclasses import dataclass +from dataclasses import replace as dataclass_replace +from pathlib import Path +from typing import Any, Callable, Dict, List, Tuple + +import numpy as np +import pandas as pd +import tifffile +import zarr +from scipy import ndimage as ndi +from skimage import feature, filters, measure, morphology + +from .config import ( + OOCYTE_IMPLEMENTATION_VERSION, + CoarseDetectionConfig, + OocyteDetectionConfig, +) +from .export import LabelExportResult, export_whole_slide_labels +from .io import ( + extract_cyx_channel_patch, + extract_padded_patch, + find_channel_index, + save_candidate_mask, +) +from .models import ExtractedPatch, LocalSegmentationResult, ScoredCandidateMask +from .qc import render_spatial_overview +from .rescue import run_secondary_rescue +from .segmentation import _segment_oocyte_patch + + +REFINED_CANDIDATE_COLUMNS = ( + "detector_component_id", + "source_detector_component_id", + "coarse_component_label", + "coarse_seed_kind", + "coarse_seed_rank", + "coarse_area_ds", + "coarse_equivalent_diameter_um", + "coarse_center_x", + "coarse_center_y", + "coarse_mean_ds", + "coarse_max_ds", + "coarse_eccentricity", + "coarse_solidity", + "coarse_bbox_x0", + "coarse_bbox_y0", + "coarse_bbox_x1", + "coarse_bbox_y1", + "coarse_blob_sigma_ds", + "coarse_blob_score", + "seed_center_x", + "seed_center_y", + "local_context_mode", + "evaluation_window_radius_px", + "evaluation_annulus_inner_px", + "evaluation_annulus_outer_px", + "center_x", + "center_y", + "component_centroid_x", + "component_centroid_y", + "component_centroid_shift_px", + "bbox_x0", + "bbox_y0", + "bbox_x1", + "bbox_y1", + "threshold_method", + "threshold", + "selection_mode", + "local_area_px", + "local_equivalent_diameter_um", + "local_major_axis_um", + "local_minor_axis_um", + "local_eccentricity", + "local_solidity", + "local_circularity", + "local_centroid_offset_px", + "local_mean_intensity", + "local_max_intensity", + "annulus_p95", + "annulus_p99", + "mean_to_annulus_p99_ratio", + "detector_score", + "accepted_strict", + "accepted_rescue", + "acceptance_mode", + "accepted", + "seed_source", + "seed_rank", + "seed_shift_px", + "local_reseed_iteration", + "local_instance_rank", + "local_instance_count_from_coarse", +) + + +@dataclass(frozen=True) +class CoarseDetectionResult: + image_shape_yx: Tuple[int, int] + downsampled: np.ndarray + mask: np.ndarray + contrast: np.ndarray + thresholds: Dict[str, Any] + candidates: pd.DataFrame + + +@dataclass(frozen=True) +class RefinedDetectionResult: + candidates: pd.DataFrame + candidate_masks: Dict[str, ScoredCandidateMask] + rescue_diagnostics: pd.DataFrame | None = None + + +@dataclass(frozen=True) +class OocyteDetectionResult: + sample_id: str + image_path: Path + image_shape_yx: Tuple[int, int] + channel_index: int + profile_name: str + profile_fingerprint: str + implementation_version: str + coarse_candidates: pd.DataFrame + candidates: pd.DataFrame + thresholds: Dict[str, Any] + runtime_seconds: Dict[str, float] + artifact_paths: Dict[str, Path] + + +@dataclass(frozen=True) +class _SegmentationEvaluation: + row: Dict[str, Any] + patch: ExtractedPatch + segmentation: LocalSegmentationResult + + +@dataclass(frozen=True) +class _EvaluatedCandidate: + row: Dict[str, Any] + evaluation: _SegmentationEvaluation + + +def _reduce_strip(strip: np.ndarray, factor: int) -> np.ndarray: + pad_h = (-strip.shape[0]) % factor + pad_w = (-strip.shape[1]) % factor + if pad_h or pad_w: + strip = np.pad(strip, ((0, pad_h), (0, pad_w)), mode="edge") + return strip.reshape( + strip.shape[0] // factor, + factor, + strip.shape[1] // factor, + factor, + ).mean(axis=(1, 3)) + + +def build_downsampled_mean_map_from_array( + channel_image: np.ndarray, + coarse: CoarseDetectionConfig, +) -> np.ndarray: + """Build the detector mean map from a two-dimensional channel image.""" + + image = np.asarray(channel_image) + if image.ndim != 2: + raise ValueError("channel image must be two-dimensional") + image_h, image_w = image.shape + factor = coarse.downsample_factor + downsampled = np.zeros( + ( + (image_h + factor - 1) // factor, + (image_w + factor - 1) // factor, + ), + dtype=np.float32, + ) + out_y = 0 + for y0 in range(0, image_h, coarse.strip_height): + y1 = min(image_h, y0 + coarse.strip_height) + reduced = _reduce_strip( + np.asarray(image[y0:y1, :], dtype=np.float32), + factor, + ) + downsampled[out_y : out_y + reduced.shape[0], : reduced.shape[1]] = reduced + out_y += reduced.shape[0] + return downsampled + + +def _build_downsampled_mean_map_from_zarr( + array: zarr.Array, + channel_index: int, + coarse: CoarseDetectionConfig, +) -> np.ndarray: + image_h = int(array.shape[1]) + image_w = int(array.shape[2]) + factor = coarse.downsample_factor + downsampled = np.zeros( + ( + (image_h + factor - 1) // factor, + (image_w + factor - 1) // factor, + ), + dtype=np.float32, + ) + out_y = 0 + for y0 in range(0, image_h, coarse.strip_height): + y1 = min(image_h, y0 + coarse.strip_height) + strip = np.asarray(array[channel_index, y0:y1, :], dtype=np.float32) + reduced = _reduce_strip(strip, factor) + downsampled[out_y : out_y + reduced.shape[0], : reduced.shape[1]] = reduced + out_y += reduced.shape[0] + return downsampled + + +def _deduplicate_coarse_rows( + rows: List[Dict[str, Any]], + merge_distance_px: float, +) -> List[Dict[str, Any]]: + if not rows: + return rows + seed_kind_priority = { + "peak": 3, + "blob_log": 2, + "local_peak": 2, + "centroid_fallback": 1, + "global_peak": 0, + } + ranked = sorted( + rows, + key=lambda row: ( + float(row["coarse_max_ds"]), + seed_kind_priority.get(str(row["coarse_seed_kind"]), 0), + float(row.get("coarse_area_ds", 0)), + ), + reverse=True, + ) + kept_rows: List[Dict[str, Any]] = [] + for row in ranked: + duplicate = any( + np.hypot( + float(row["coarse_center_x"]) - float(kept["coarse_center_x"]), + float(row["coarse_center_y"]) - float(kept["coarse_center_y"]), + ) + <= merge_distance_px + for kept in kept_rows + ) + if not duplicate: + kept_rows.append(row) + return kept_rows + + +def detect_coarse_candidates( + downsampled: np.ndarray, + config: OocyteDetectionConfig, +) -> CoarseDetectionResult: + """Generate v6 coarse proposals from a downsampled UCHL1 mean map.""" + + image = np.asarray(downsampled, dtype=np.float32) + if image.ndim != 2 or image.size == 0: + raise ValueError("downsampled UCHL1 map must be a non-empty 2D array") + coarse = config.coarse + factor = coarse.downsample_factor + log_image = np.log1p(image) + smooth = ndi.gaussian_filter(log_image, sigma=coarse.gaussian_sigma_small) + background = ndi.gaussian_filter(log_image, sigma=coarse.gaussian_sigma_large) + contrast = smooth - background + + intensity_threshold = max( + float(np.percentile(image, coarse.intensity_percentile)), + float(np.percentile(image, coarse.intensity_floor_percentile)) + * coarse.intensity_floor_multiplier, + ) + positive_contrast = contrast[contrast > 0] + triangle_contrast = ( + float(filters.threshold_triangle(positive_contrast)) + if positive_contrast.size + else 0.0 + ) + contrast_threshold = max( + float(np.percentile(contrast, coarse.contrast_percentile)), + triangle_contrast, + ) + initial_mask = (image >= intensity_threshold) & (contrast >= contrast_threshold) + + def seed_rows_for_region(region, labeled: np.ndarray) -> List[Dict[str, Any]]: + diameter_um = float( + region.equivalent_diameter_area * factor * config.pixel_size_um + ) + if diameter_um < coarse.min_diameter_um or diameter_um > coarse.max_diameter_um: + return [] + + y0, x0, y1, x1 = region.bbox + local_intensity = image[y0:y1, x0:x1] + local_labels = labeled[y0:y1, x0:x1] == region.label + peak_threshold = float( + np.percentile(local_intensity[local_labels], coarse.peak_percentile) + ) + peak_coords = feature.peak_local_max( + local_intensity, + labels=local_labels.astype(np.uint8), + min_distance=coarse.peak_min_distance_ds, + threshold_abs=peak_threshold, + num_peaks=coarse.max_peaks_per_component, + exclude_border=False, + ) + seeds = [] + if peak_coords.size: + peak_rows = [] + for peak_y, peak_x in peak_coords: + if local_labels[int(peak_y), int(peak_x)]: + peak_rows.append( + ( + float(local_intensity[int(peak_y), int(peak_x)]), + int(peak_y), + int(peak_x), + ) + ) + peak_rows.sort(reverse=True) + for peak_rank, (_, peak_y, peak_x) in enumerate( + peak_rows[: coarse.max_peaks_per_component], + start=1, + ): + seeds.append((peak_y + y0, peak_x + x0, "peak", peak_rank)) + else: + seeds.append( + ( + int(round(region.centroid[0])), + int(round(region.centroid[1])), + "centroid_fallback", + 1, + ) + ) + + rows = [] + for seed_y, seed_x, seed_kind, seed_rank in seeds: + rows.append( + { + "coarse_component_label": int(region.label), + "coarse_seed_kind": seed_kind, + "coarse_seed_rank": int(seed_rank), + "coarse_area_ds": int(region.area), + "coarse_equivalent_diameter_um": diameter_um, + "coarse_center_x": int(round(seed_x * factor + factor / 2)), + "coarse_center_y": int(round(seed_y * factor + factor / 2)), + "coarse_mean_ds": float(region.mean_intensity), + "coarse_max_ds": float(region.max_intensity), + "coarse_eccentricity": float(region.eccentricity), + "coarse_solidity": float(region.solidity), + "coarse_bbox_x0": int(region.bbox[1] * factor), + "coarse_bbox_y0": int(region.bbox[0] * factor), + "coarse_bbox_x1": int(region.bbox[3] * factor), + "coarse_bbox_y1": int(region.bbox[2] * factor), + } + ) + return rows + + def regions_from_mask( + mask: np.ndarray, + *, + dilate_first: bool, + min_area_ds: int, + ): + working = mask.copy() + if dilate_first: + working = ndi.binary_dilation(working, structure=morphology.disk(1)) + if coarse.closing_radius_ds > 0: + working = ndi.binary_closing( + working, + structure=morphology.disk(coarse.closing_radius_ds), + ) + working = ndi.binary_fill_holes(working) + working = morphology.remove_small_objects(working, min_size=min_area_ds) + labeled = measure.label(working) + regions_by_label = {} + rows = [] + for region in measure.regionprops(labeled, intensity_image=image): + regions_by_label[int(region.label)] = region + rows.extend(seed_rows_for_region(region, labeled)) + return working, labeled, regions_by_label, rows + + cleaned_mask, labeled, regions_by_label, rows = regions_from_mask( + initial_mask, + dilate_first=False, + min_area_ds=coarse.min_component_area_ds, + ) + cleanup_mode = "standard" + if len(rows) < coarse.fallback_min_candidate_count: + cleaned_mask, labeled, regions_by_label, rows = regions_from_mask( + initial_mask, + dilate_first=True, + min_area_ds=max(8, coarse.min_component_area_ds // 2), + ) + cleanup_mode = "fallback_dilate" + + global_peaks = feature.peak_local_max( + image, + labels=(contrast >= contrast_threshold).astype(np.uint8), + min_distance=coarse.global_peak_min_distance_ds, + threshold_abs=intensity_threshold, + num_peaks=coarse.global_peak_max_count, + exclude_border=False, + ) + for global_rank, (peak_y, peak_x) in enumerate(global_peaks, start=1): + label = int(labeled[int(peak_y), int(peak_x)]) + region = regions_by_label.get(label) + center_x = int(round(peak_x * factor + factor / 2)) + center_y = int(round(peak_y * factor + factor / 2)) + if region is None: + rows.append( + { + "coarse_component_label": label, + "coarse_seed_kind": "global_peak", + "coarse_seed_rank": int(global_rank), + "coarse_area_ds": 1, + "coarse_equivalent_diameter_um": config.pixel_size_um * factor, + "coarse_center_x": center_x, + "coarse_center_y": center_y, + "coarse_mean_ds": float(image[int(peak_y), int(peak_x)]), + "coarse_max_ds": float(image[int(peak_y), int(peak_x)]), + "coarse_eccentricity": 0.0, + "coarse_solidity": 1.0, + "coarse_bbox_x0": int(max(0, peak_x - 1) * factor), + "coarse_bbox_y0": int(max(0, peak_y - 1) * factor), + "coarse_bbox_x1": int((peak_x + 2) * factor), + "coarse_bbox_y1": int((peak_y + 2) * factor), + } + ) + else: + rows.append( + { + "coarse_component_label": int(region.label), + "coarse_seed_kind": "global_peak", + "coarse_seed_rank": int(global_rank), + "coarse_area_ds": int(region.area), + "coarse_equivalent_diameter_um": float( + region.equivalent_diameter_area + * factor + * config.pixel_size_um + ), + "coarse_center_x": center_x, + "coarse_center_y": center_y, + "coarse_mean_ds": float(region.mean_intensity), + "coarse_max_ds": float(image[int(peak_y), int(peak_x)]), + "coarse_eccentricity": float(region.eccentricity), + "coarse_solidity": float(region.solidity), + "coarse_bbox_x0": int(region.bbox[1] * factor), + "coarse_bbox_y0": int(region.bbox[0] * factor), + "coarse_bbox_x1": int(region.bbox[3] * factor), + "coarse_bbox_y1": int(region.bbox[2] * factor), + } + ) + + positive_image = np.clip(contrast, 0, None) + relaxed_intensity = max( + float(np.percentile(image, coarse.blob_intensity_percentile)), + intensity_threshold * coarse.blob_intensity_relax_multiplier, + ) + relaxed_contrast = max( + float(np.percentile(contrast, coarse.blob_contrast_percentile)), + contrast_threshold * coarse.blob_contrast_relax_multiplier, + ) + blob_input = ndi.gaussian_filter( + positive_image, + sigma=coarse.blob_response_sigma_ds, + ) + blob_input_max = float(blob_input.max()) + if blob_input_max > 0 and coarse.blob_max_new_candidates > 0: + blob_rows = [] + blobs = feature.blob_log( + blob_input / blob_input_max, + min_sigma=coarse.blob_min_sigma_ds, + max_sigma=coarse.blob_max_sigma_ds, + num_sigma=coarse.blob_num_sigma, + threshold=coarse.blob_threshold, + overlap=coarse.blob_overlap, + exclude_border=False, + ) + for blob_rank, (blob_y, blob_x, sigma_ds) in enumerate(blobs, start=1): + yi = int(round(blob_y)) + xi = int(round(blob_x)) + if yi < 0 or yi >= image.shape[0] or xi < 0 or xi >= image.shape[1]: + continue + intensity_value = float(image[yi, xi]) + contrast_value = float(contrast[yi, xi]) + if intensity_value < relaxed_intensity or contrast_value < relaxed_contrast: + continue + estimated_diameter_um = float( + np.sqrt(2.0) * float(sigma_ds) * factor * config.pixel_size_um * 2.0 + ) + if ( + estimated_diameter_um < coarse.min_diameter_um + or estimated_diameter_um > coarse.max_diameter_um + ): + continue + label = int(labeled[yi, xi]) + region = regions_by_label.get(label) + radius_ds = max(2.0, 3.0 * float(sigma_ds)) + blob_rows.append( + { + "coarse_component_label": label, + "coarse_seed_kind": "blob_log", + "coarse_seed_rank": int(blob_rank), + "coarse_area_ds": int(region.area) + if region is not None + else int(np.pi * radius_ds * radius_ds), + "coarse_equivalent_diameter_um": float( + region.equivalent_diameter_area + * factor + * config.pixel_size_um + ) + if region is not None + else estimated_diameter_um, + "coarse_center_x": int(round(blob_x * factor + factor / 2)), + "coarse_center_y": int(round(blob_y * factor + factor / 2)), + "coarse_mean_ds": float(region.mean_intensity) + if region is not None + else intensity_value, + "coarse_max_ds": intensity_value, + "coarse_eccentricity": float(region.eccentricity) + if region is not None + else 0.0, + "coarse_solidity": float(region.solidity) + if region is not None + else 1.0, + "coarse_bbox_x0": int( + max(0, round((blob_x - radius_ds) * factor)) + ), + "coarse_bbox_y0": int( + max(0, round((blob_y - radius_ds) * factor)) + ), + "coarse_bbox_x1": int(round((blob_x + radius_ds) * factor)), + "coarse_bbox_y1": int(round((blob_y + radius_ds) * factor)), + "coarse_blob_sigma_ds": float(sigma_ds), + "coarse_blob_score": float( + blob_input[yi, xi] * max(contrast_value, 0.0) + ), + } + ) + blob_rows.sort( + key=lambda row: ( + float(row.get("coarse_blob_score", 0.0)), + float(row["coarse_max_ds"]), + float(row["coarse_equivalent_diameter_um"]), + ), + reverse=True, + ) + existing_centers = [ + (float(row["coarse_center_x"]), float(row["coarse_center_y"])) + for row in rows + ] + kept_blob_rows = [] + kept_blob_centers = [] + for row in blob_rows: + center = ( + float(row["coarse_center_x"]), + float(row["coarse_center_y"]), + ) + duplicate = any( + np.hypot(center[0] - x, center[1] - y) + <= coarse.seed_merge_distance_px + for x, y in existing_centers + kept_blob_centers + ) + if duplicate: + continue + kept_blob_rows.append(row) + kept_blob_centers.append(center) + if len(kept_blob_rows) >= coarse.blob_max_new_candidates: + break + rows.extend(kept_blob_rows) + + rows = _deduplicate_coarse_rows(rows, coarse.seed_merge_distance_px) + candidates = pd.DataFrame(rows) + if not candidates.empty: + candidates = candidates.sort_values( + ["coarse_max_ds", "coarse_area_ds"], + ascending=[False, False], + ).reset_index(drop=True) + candidates.insert( + 0, + "detector_component_id", + [f"det_{index:04d}" for index in range(len(candidates))], + ) + thresholds = { + "intensity_threshold": intensity_threshold, + "contrast_threshold": contrast_threshold, + "triangle_contrast_threshold": triangle_contrast, + "cleanup_mode": cleanup_mode, + "coarse_blob_relaxed_intensity_floor": relaxed_intensity, + "coarse_blob_relaxed_contrast_floor": relaxed_contrast, + } + return CoarseDetectionResult( + image_shape_yx=(image.shape[0] * factor, image.shape[1] * factor), + downsampled=image, + mask=np.asarray(cleaned_mask, dtype=np.bool_), + contrast=np.asarray(contrast, dtype=np.float32), + thresholds=thresholds, + candidates=candidates, + ) + + +def scan_coarse_candidates( + image_path: Path, + channel_index: int, + config: OocyteDetectionConfig, +) -> CoarseDetectionResult: + """Read one OME-TIFF channel in strips and generate coarse proposals.""" + + path = Path(image_path) + with tifffile.TiffFile(path) as tif: + series = tif.series[0] + if series.axes != "CYX": + raise ValueError( + f"whole-slide coarse detection currently requires CYX axes, got {series.axes!r}" + ) + array = zarr.open(series.aszarr(), mode="r") + if not 0 <= channel_index < int(array.shape[0]): + raise IndexError(f"channel index {channel_index} outside image channel range") + image_shape = (int(array.shape[1]), int(array.shape[2])) + downsampled = _build_downsampled_mean_map_from_zarr( + array, + channel_index, + config.coarse, + ) + result = detect_coarse_candidates(downsampled, config) + return CoarseDetectionResult( + image_shape_yx=image_shape, + downsampled=result.downsampled, + mask=result.mask, + contrast=result.contrast, + thresholds=result.thresholds, + candidates=result.candidates, + ) + + +def _clip01(value: float) -> float: + return float(min(max(value, 0.0), 1.0)) + + +def _range_score( + value: float, + core_lo: float, + core_hi: float, + soft_lo: float, + soft_hi: float, +) -> float: + if value < soft_lo or value > soft_hi: + return 0.0 + if core_lo <= value <= core_hi: + return 1.0 + if value < core_lo: + return _clip01((value - soft_lo) / max(core_lo - soft_lo, 1e-6)) + return _clip01((soft_hi - value) / max(soft_hi - core_hi, 1e-6)) + + +def candidate_score( + *, + equivalent_diameter_um: float, + circularity: float, + solidity: float, + brightness_ratio: float, + offset_px: float, + config: OocyteDetectionConfig, +) -> float: + """Score one locally segmented candidate using the frozen profile weights.""" + + scoring = config.scoring + size_score = _range_score( + equivalent_diameter_um, + scoring.size_core_range_um[0], + scoring.size_core_range_um[1], + scoring.size_soft_range_um[0], + scoring.size_soft_range_um[1], + ) + circularity_score = _clip01( + (circularity - scoring.circularity_floor) / scoring.circularity_span + ) + solidity_score = _clip01( + (solidity - scoring.solidity_floor) / scoring.solidity_span + ) + shape_score = ( + scoring.shape_circularity_weight * circularity_score + + scoring.shape_solidity_weight * solidity_score + ) + brightness_score = _clip01( + (brightness_ratio - scoring.brightness_ratio_floor) + / scoring.brightness_ratio_span + ) + centered_score = _clip01(1.0 - offset_px / scoring.centered_offset_limit_px) + return float( + scoring.brightness_weight * brightness_score + + scoring.shape_weight * shape_score + + scoring.size_weight * size_score + + scoring.centered_weight * centered_score + ) + + +def _seed_candidates_from_patch( + patch: np.ndarray, + center_x: int, + center_y: int, + config: OocyteDetectionConfig, +) -> List[Dict[str, Any]]: + local = config.local + patch_center_y = patch.shape[0] // 2 + patch_center_x = patch.shape[1] // 2 + seeds: List[Dict[str, Any]] = [ + { + "seed_center_x": int(center_x), + "seed_center_y": int(center_y), + "seed_source": "coarse_seed", + "seed_rank": 0, + "seed_shift_px": 0.0, + } + ] + + yy, xx = np.ogrid[: patch.shape[0], : patch.shape[1]] + distance = np.sqrt((yy - patch_center_y) ** 2 + (xx - patch_center_x) ** 2) + search_mask = distance <= local.peak_search_radius_px + + def append_peak_seeds( + peak_image: np.ndarray, + *, + source_name: str, + min_distance_px: int, + percentile: float, + max_count: int, + ) -> None: + if max_count <= 0: + return + search_values = peak_image[search_mask] + if search_values.size == 0: + return + peak_threshold = float(np.percentile(search_values, percentile)) + peak_coords = feature.peak_local_max( + peak_image, + labels=search_mask.astype(np.uint8), + min_distance=min_distance_px, + threshold_abs=peak_threshold, + num_peaks=max_count, + exclude_border=False, + ) + peak_rows = [ + (float(peak_image[int(peak_y), int(peak_x)]), int(peak_y), int(peak_x)) + for peak_y, peak_x in peak_coords + ] + peak_rows.sort(reverse=True) + for peak_rank, (_, peak_y, peak_x) in enumerate(peak_rows, start=1): + shift_x = int(peak_x - patch_center_x) + shift_y = int(peak_y - patch_center_y) + shift_px = float(np.hypot(shift_x, shift_y)) + if shift_px <= local.seed_merge_radius_px: + continue + candidate = { + "seed_center_x": int(center_x + shift_x), + "seed_center_y": int(center_y + shift_y), + "seed_source": source_name, + "seed_rank": int(peak_rank), + "seed_shift_px": shift_px, + } + duplicate = any( + np.hypot( + candidate["seed_center_x"] - seed["seed_center_x"], + candidate["seed_center_y"] - seed["seed_center_y"], + ) + <= local.seed_merge_radius_px + for seed in seeds + ) + if not duplicate: + seeds.append(candidate) + + smooth = ndi.gaussian_filter( + patch.astype(np.float32), + sigma=max(local.peak_gaussian_sigma, 0.5), + ) + append_peak_seeds( + smooth, + source_name="local_peak", + min_distance_px=local.peak_min_distance_px, + percentile=local.peak_percentile, + max_count=local.max_peak_seeds_per_candidate, + ) + if local.max_broad_peak_seeds_per_candidate > 0: + broad_smooth = ndi.gaussian_filter( + patch.astype(np.float32), + sigma=max(local.broad_peak_gaussian_sigma, 1.0), + ) + append_peak_seeds( + broad_smooth, + source_name="local_broad_peak", + min_distance_px=local.broad_peak_min_distance_px, + percentile=local.broad_peak_percentile, + max_count=local.max_broad_peak_seeds_per_candidate, + ) + + if local.offset_ring_step_px > 0 and local.max_offset_ring_seeds > 0: + ring_offsets = [ + (-1, -1), + (0, -1), + (1, -1), + (-1, 0), + (1, 0), + (-1, 1), + (0, 1), + (1, 1), + ] + for ring_rank, (dx_sign, dy_sign) in enumerate( + ring_offsets[: local.max_offset_ring_seeds], + start=1, + ): + shift_x = int(dx_sign * local.offset_ring_step_px) + shift_y = int(dy_sign * local.offset_ring_step_px) + candidate = { + "seed_center_x": int(center_x + shift_x), + "seed_center_y": int(center_y + shift_y), + "seed_source": "local_offset_ring", + "seed_rank": int(ring_rank), + "seed_shift_px": float(np.hypot(shift_x, shift_y)), + } + duplicate = any( + np.hypot( + candidate["seed_center_x"] - seed["seed_center_x"], + candidate["seed_center_y"] - seed["seed_center_y"], + ) + <= local.seed_merge_radius_px + for seed in seeds + ) + if not duplicate: + seeds.append(candidate) + return seeds + + +def _candidate_rank(candidate: _EvaluatedCandidate) -> Tuple[float, float, float, float]: + row = candidate.row + return ( + float(row["detector_score"]), + float(row["local_circularity"]), + float(row["mean_to_annulus_p99_ratio"]), + -float(row["seed_shift_px"]), + ) + + +def _keep_distinct_candidate_rows( + candidates: List[_EvaluatedCandidate], + *, + merge_distance_px: float, + max_count: int, +) -> List[_EvaluatedCandidate]: + ranked = sorted(candidates, key=_candidate_rank, reverse=True) + kept: List[_EvaluatedCandidate] = [] + for candidate in ranked: + row = candidate.row + duplicate = any( + np.hypot( + float(row["center_x"]) - float(existing.row["center_x"]), + float(row["center_y"]) - float(existing.row["center_y"]), + ) + <= merge_distance_px + for existing in kept + ) + if not duplicate: + kept.append(candidate) + if len(kept) >= max_count: + break + return kept + + +def _deduplicate_refined_candidates( + candidates: List[_EvaluatedCandidate], + merge_distance_px: float, +) -> List[_EvaluatedCandidate]: + ranked = sorted( + candidates, + key=lambda candidate: ( + float(candidate.row["detector_score"]), + float(candidate.row["local_circularity"]), + float(candidate.row["local_equivalent_diameter_um"]), + ), + reverse=True, + ) + kept: List[_EvaluatedCandidate] = [] + for candidate in ranked: + row = candidate.row + duplicate = any( + np.hypot( + float(row["center_x"]) - float(existing.row["center_x"]), + float(row["center_y"]) - float(existing.row["center_y"]), + ) + <= merge_distance_px + for existing in kept + ) + if not duplicate: + kept.append(candidate) + + renumbered = [] + for index, candidate in enumerate(kept): + source_id = str(candidate.row["detector_component_id"]) + row = { + "detector_component_id": f"det_{index:04d}", + "source_detector_component_id": source_id, + **{ + key: value + for key, value in candidate.row.items() + if key != "detector_component_id" + }, + } + renumbered.append(_EvaluatedCandidate(row=row, evaluation=candidate.evaluation)) + return renumbered + + +def _refine_candidates( + patch_reader: Callable[[int, int, int], ExtractedPatch], + image_shape_yx: Tuple[int, int], + coarse_candidates: pd.DataFrame, + config: OocyteDetectionConfig, +) -> RefinedDetectionResult: + local = config.local + default_context = { + "context_mode": "default", + "window_radius_px": int(local.window_radius_px), + "annulus_inner_px": int(local.annulus_inner_px), + "annulus_outer_px": int(local.annulus_outer_px), + } + compact_context = { + "context_mode": "compact", + "window_radius_px": int(local.compact_window_radius_px), + "annulus_inner_px": int(local.compact_annulus_inner_px), + "annulus_outer_px": int(local.compact_annulus_outer_px), + } + patch_cache: Dict[Tuple[int, int, int], ExtractedPatch] = {} + annulus_mask_cache: Dict[Tuple[int, int, int, int], np.ndarray] = {} + evaluation_cache: Dict[ + Tuple[int, int, int, int, int], + _SegmentationEvaluation | None, + ] = {} + + def cached_patch(seed_x: int, seed_y: int, radius: int) -> ExtractedPatch: + key = (int(seed_x), int(seed_y), int(radius)) + cached = patch_cache.get(key) + if cached is None: + cached = patch_reader(*key) + patch_cache[key] = cached + return cached + + def annulus_mask( + shape: Tuple[int, int], + annulus_inner_px: int, + annulus_outer_px: int, + ) -> np.ndarray: + key = ( + int(shape[0]), + int(shape[1]), + int(annulus_inner_px), + int(annulus_outer_px), + ) + cached = annulus_mask_cache.get(key) + if cached is None: + center_y = shape[0] // 2 + center_x = shape[1] // 2 + yy, xx = np.ogrid[: shape[0], : shape[1]] + distance = np.sqrt((yy - center_y) ** 2 + (xx - center_x) ** 2) + cached = (distance >= annulus_inner_px) & (distance <= annulus_outer_px) + annulus_mask_cache[key] = cached + return cached + + def evaluate_seed_context( + seed_x: int, + seed_y: int, + context: Dict[str, Any], + ) -> _SegmentationEvaluation | None: + cache_key = ( + int(seed_x), + int(seed_y), + int(context["window_radius_px"]), + int(context["annulus_inner_px"]), + int(context["annulus_outer_px"]), + ) + if cache_key in evaluation_cache: + return evaluation_cache[cache_key] + try: + patch = cached_patch( + seed_x, + seed_y, + int(context["window_radius_px"]), + ) + smooth, segmentation = _segment_oocyte_patch( + patch.image, + config, + annulus_inner_px=int(context["annulus_inner_px"]), + annulus_outer_px=int(context["annulus_outer_px"]), + ) + except (IndexError, ValueError): + evaluation_cache[cache_key] = None + return None + + metrics = segmentation.metrics + mask = annulus_mask( + smooth.shape, + int(context["annulus_inner_px"]), + int(context["annulus_outer_px"]), + ) + annulus = smooth[mask] + annulus_p99 = float(np.percentile(annulus, 99)) + annulus_p95 = float(np.percentile(annulus, 95)) + brightness_ratio = float(metrics.mean_intensity / max(annulus_p99, 1.0)) + score = candidate_score( + equivalent_diameter_um=metrics.equivalent_diameter_um, + circularity=metrics.circularity, + solidity=metrics.solidity, + brightness_ratio=brightness_ratio, + offset_px=metrics.centroid_offset_px, + config=config, + ) + acceptance = config.acceptance + strict_accept = bool( + score >= acceptance.score_threshold + and metrics.equivalent_diameter_um >= acceptance.strict_min_diameter_um + ) + rescue_accept = bool( + not strict_accept + and acceptance.rescue_min_diameter_um + <= metrics.equivalent_diameter_um + <= acceptance.rescue_max_diameter_um + and metrics.circularity >= acceptance.rescue_min_circularity + and metrics.solidity >= acceptance.rescue_min_solidity + and brightness_ratio >= acceptance.rescue_min_brightness_ratio + and metrics.centroid_offset_px <= acceptance.rescue_max_offset_px + ) + center_y_patch = smooth.shape[0] // 2 + center_x_patch = smooth.shape[1] // 2 + component_centroid_x = float( + seed_x + metrics.centroid_x_px - center_x_patch + ) + component_centroid_y = float( + seed_y + metrics.centroid_y_px - center_y_patch + ) + row = { + "seed_center_x": int(seed_x), + "seed_center_y": int(seed_y), + "local_context_mode": str(context["context_mode"]), + "evaluation_window_radius_px": int(context["window_radius_px"]), + "evaluation_annulus_inner_px": int(context["annulus_inner_px"]), + "evaluation_annulus_outer_px": int(context["annulus_outer_px"]), + "center_x": int(seed_x), + "center_y": int(seed_y), + "component_centroid_x": component_centroid_x, + "component_centroid_y": component_centroid_y, + "component_centroid_shift_px": float( + np.hypot( + component_centroid_x - seed_x, + component_centroid_y - seed_y, + ) + ), + "bbox_x0": patch.bbox.x0, + "bbox_y0": patch.bbox.y0, + "bbox_x1": patch.bbox.x1, + "bbox_y1": patch.bbox.y1, + "threshold_method": metrics.threshold_method, + "threshold": float(metrics.threshold), + "selection_mode": metrics.selection_mode, + "local_area_px": int(metrics.area_px), + "local_equivalent_diameter_um": float(metrics.equivalent_diameter_um), + "local_major_axis_um": float(metrics.major_axis_um), + "local_minor_axis_um": float(metrics.minor_axis_um), + "local_eccentricity": float(metrics.eccentricity), + "local_solidity": float(metrics.solidity), + "local_circularity": float(metrics.circularity), + "local_centroid_offset_px": float(metrics.centroid_offset_px), + "local_mean_intensity": float(metrics.mean_intensity), + "local_max_intensity": float(metrics.max_intensity), + "annulus_p95": annulus_p95, + "annulus_p99": annulus_p99, + "mean_to_annulus_p99_ratio": brightness_ratio, + "detector_score": float(score), + "accepted_strict": strict_accept, + "accepted_rescue": rescue_accept, + "acceptance_mode": ( + "strict" if strict_accept else ("rescue" if rescue_accept else "rejected") + ), + "accepted": bool(strict_accept or rescue_accept), + } + evaluation = _SegmentationEvaluation( + row=row, + patch=patch, + segmentation=segmentation, + ) + evaluation_cache[cache_key] = evaluation + return evaluation + + def evaluate_candidate( + record: Dict[str, Any], + seed_meta: Dict[str, Any], + seed_x: int, + seed_y: int, + *, + context: Dict[str, Any], + reseed_iteration: int, + ) -> _EvaluatedCandidate | None: + evaluation = evaluate_seed_context(seed_x, seed_y, context) + if evaluation is None: + return None + coarse_shift_px = float( + np.hypot( + seed_x - float(record["coarse_center_x"]), + seed_y - float(record["coarse_center_y"]), + ) + ) + source = str(seed_meta["seed_source"]) + row = { + **record, + **evaluation.row, + "seed_source": ( + source if reseed_iteration == 0 else f"{source}_centroid_reseed" + ), + "seed_rank": int(seed_meta["seed_rank"]), + "seed_shift_px": coarse_shift_px, + "local_reseed_iteration": int(reseed_iteration), + } + return _EvaluatedCandidate(row=row, evaluation=evaluation) + + def evaluate_seed_variants( + record: Dict[str, Any], + seed: Dict[str, Any], + *, + contexts_to_try: List[Dict[str, Any]], + ) -> List[_EvaluatedCandidate]: + candidate_rows: List[_EvaluatedCandidate] = [] + context_index = 0 + while context_index < len(contexts_to_try): + context = contexts_to_try[context_index] + seed_x = int(seed["seed_center_x"]) + seed_y = int(seed["seed_center_y"]) + best_context_row = None + for reseed_iteration in range(local.max_centroid_reseed_iterations + 1): + candidate = evaluate_candidate( + record, + seed, + seed_x, + seed_y, + context=context, + reseed_iteration=reseed_iteration, + ) + if candidate is None: + break + if best_context_row is None or _candidate_rank(candidate) > _candidate_rank( + best_context_row + ): + best_context_row = candidate + if reseed_iteration >= local.max_centroid_reseed_iterations: + break + centroid_shift_px = float( + candidate.row["component_centroid_shift_px"] + ) + if centroid_shift_px < local.centroid_reseed_min_shift_px: + break + if centroid_shift_px > local.centroid_reseed_max_shift_px: + break + next_seed_x = int(round(candidate.row["component_centroid_x"])) + next_seed_y = int(round(candidate.row["component_centroid_y"])) + if np.hypot(next_seed_x - seed_x, next_seed_y - seed_y) <= 1.0: + break + seed_x = next_seed_x + seed_y = next_seed_y + if best_context_row is not None: + candidate_rows.append(best_context_row) + should_retry_compact = bool( + compact_context["window_radius_px"] + < default_context["window_radius_px"] + and context["context_mode"] == "default" + and str(seed["seed_source"]) == "coarse_seed" + and ( + not bool(best_context_row.row["accepted"]) + or best_context_row.row["selection_mode"] != "center_component" + or float( + best_context_row.row["mean_to_annulus_p99_ratio"] + ) + < local.compact_retry_max_brightness_ratio + ) + ) + if should_retry_compact: + contexts_to_try.append(compact_context) + context_index += 1 + return candidate_rows + + rows: List[_EvaluatedCandidate] = [] + for record in coarse_candidates.to_dict("records"): + center_x = int(record["coarse_center_x"]) + center_y = int(record["coarse_center_y"]) + try: + initial_patch = cached_patch( + center_x, + center_y, + local.window_radius_px, + ) + except (IndexError, ValueError): + continue + candidate_rows: List[_EvaluatedCandidate] = [] + for seed in _seed_candidates_from_patch( + initial_patch.image, + center_x, + center_y, + config, + ): + contexts_to_try = [default_context] + if ( + compact_context["window_radius_px"] + < default_context["window_radius_px"] + and str(seed["seed_source"]) != "coarse_seed" + ): + contexts_to_try.append(compact_context) + candidate_rows.extend( + evaluate_seed_variants( + record, + seed, + contexts_to_try=contexts_to_try, + ) + ) + kept_rows = _keep_distinct_candidate_rows( + candidate_rows, + merge_distance_px=local.instance_merge_distance_px, + max_count=local.max_instances_per_coarse_candidate, + ) + for local_instance_rank, candidate in enumerate(kept_rows, start=1): + row = { + **candidate.row, + "local_instance_rank": int(local_instance_rank), + "local_instance_count_from_coarse": int(len(kept_rows)), + } + rows.append(_EvaluatedCandidate(row=row, evaluation=candidate.evaluation)) + # Final rows retain their own masks; discard all rejected evaluations before + # moving to the next proposal so memory does not grow with every seed tested. + patch_cache.clear() + evaluation_cache.clear() + + rows.sort( + key=lambda candidate: ( + float(candidate.row["detector_score"]), + float(candidate.row["coarse_max_ds"]), + ), + reverse=True, + ) + rows = _deduplicate_refined_candidates( + rows, + config.deduplication.refined_center_distance_px, + ) + candidate_masks = {} + for candidate in rows: + candidate_id = str(candidate.row["detector_component_id"]) + evaluation = candidate.evaluation + cropped_mask = evaluation.patch.crop_to_image_bounds( + evaluation.segmentation.mask + ) + candidate_masks[candidate_id] = ScoredCandidateMask( + mask=np.asarray(cropped_mask, dtype=np.bool_).copy(), + bbox=evaluation.patch.bbox, + image_shape_yx=image_shape_yx, + metrics=evaluation.segmentation.metrics, + ) + return RefinedDetectionResult( + candidates=pd.DataFrame([candidate.row for candidate in rows]).reindex( + columns=REFINED_CANDIDATE_COLUMNS + ), + candidate_masks=candidate_masks, + ) + + +def refine_candidates_from_array( + channel_image: np.ndarray, + coarse_candidates: pd.DataFrame, + config: OocyteDetectionConfig, +) -> RefinedDetectionResult: + """Refine coarse candidates against an in-memory raw UCHL1 image.""" + + image = np.asarray(channel_image) + if image.ndim != 2: + raise ValueError("channel image must be two-dimensional") + + def patch_reader(center_x: int, center_y: int, radius: int) -> ExtractedPatch: + return extract_padded_patch(image, (center_x, center_y), radius) + + baseline = _refine_candidates( + patch_reader, + (int(image.shape[0]), int(image.shape[1])), + coarse_candidates, + config, + ) + if config.secondary_rescue is None: + return baseline + rescued = run_secondary_rescue( + patch_reader=patch_reader, + image_shape_yx=(int(image.shape[0]), int(image.shape[1])), + coarse_candidates=coarse_candidates, + baseline_candidates=baseline.candidates, + baseline_masks=baseline.candidate_masks, + config=config, + score_candidate=candidate_score, + ) + return RefinedDetectionResult( + candidates=rescued.candidates, + candidate_masks=rescued.candidate_masks, + rescue_diagnostics=rescued.diagnostics, + ) + + +def scan_refined_candidates( + image_path: Path, + channel_index: int, + coarse_candidates: pd.DataFrame, + config: OocyteDetectionConfig, +) -> RefinedDetectionResult: + """Refine coarse candidates from one raw OME-TIFF channel.""" + + path = Path(image_path) + with tifffile.TiffFile(path) as tif: + series = tif.series[0] + if series.axes != "CYX": + raise ValueError( + f"whole-slide refinement currently requires CYX axes, got {series.axes!r}" + ) + array = zarr.open(series.aszarr(), mode="r") + if not 0 <= channel_index < int(array.shape[0]): + raise IndexError(f"channel index {channel_index} outside image channel range") + image_shape = (int(array.shape[1]), int(array.shape[2])) + + def patch_reader(center_x: int, center_y: int, radius: int) -> ExtractedPatch: + return extract_cyx_channel_patch( + array, + channel_index, + (center_x, center_y), + radius, + ) + + baseline = _refine_candidates( + patch_reader, + image_shape, + coarse_candidates, + config, + ) + if config.secondary_rescue is None: + return baseline + rescued = run_secondary_rescue( + patch_reader=patch_reader, + image_shape_yx=image_shape, + coarse_candidates=coarse_candidates, + baseline_candidates=baseline.candidates, + baseline_masks=baseline.candidate_masks, + config=config, + score_candidate=candidate_score, + ) + return RefinedDetectionResult( + candidates=rescued.candidates, + candidate_masks=rescued.candidate_masks, + rescue_diagnostics=rescued.diagnostics, + ) + + +def _json_default(value: Any) -> Any: + if isinstance(value, Path): + return str(value) + if isinstance(value, np.generic): + return value.item() + raise TypeError(f"object of type {type(value).__name__} is not JSON serializable") + + +def _atomic_write_json(payload: Dict[str, Any], destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".json", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + json.dump( + payload, + handle, + indent=2, + sort_keys=True, + default=_json_default, + ) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _atomic_write_csv(table: pd.DataFrame, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".csv", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + table.to_csv(handle, index=False) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def detect_oocytes( + image_path: Path, + *, + sample_id: str, + out_dir: Path, + config: OocyteDetectionConfig, + channel_name: str | None = "UCHL1", + channel_index: int | None = None, + antibodies_path: Path | None = None, + pixel_size_um: float | None = None, +) -> OocyteDetectionResult: + """Run standalone raw-UCHL1 detection and persist one sample deliverable.""" + + if not sample_id.strip(): + raise ValueError("sample_id must not be empty") + source_path = Path(image_path).resolve() + if not source_path.is_file(): + raise FileNotFoundError(f"raw OME-TIFF not found: {source_path}") + resolved_config = config + if pixel_size_um is not None: + if pixel_size_um <= 0: + raise ValueError("pixel_size_um must be positive") + if not np.isclose(pixel_size_um, config.pixel_size_um): + resolved_config = dataclass_replace( + config, + pixel_size_um=float(pixel_size_um), + ) + if channel_index is None: + if antibodies_path is None: + raise ValueError( + "provide channel_index or antibodies_path for UCHL1 channel resolution" + ) + channel_index = find_channel_index( + Path(antibodies_path), + channel_name or "UCHL1", + ) + + sample_dir = Path(out_dir) + sample_dir.mkdir(parents=True, exist_ok=True) + masks_dir = sample_dir / "masks" + masks_dir.mkdir(parents=True, exist_ok=True) + fingerprint = resolved_config.fingerprint() + total_start = time.perf_counter() + + stage_start = time.perf_counter() + coarse = scan_coarse_candidates(source_path, channel_index, resolved_config) + coarse_seconds = time.perf_counter() - stage_start + + stage_start = time.perf_counter() + refined = scan_refined_candidates( + source_path, + channel_index, + coarse.candidates, + resolved_config, + ) + refinement_seconds = time.perf_counter() - stage_start + + stage_start = time.perf_counter() + candidates = refined.candidates.copy() + mask_paths = [] + for candidate_id in candidates.get( + "detector_component_id", + pd.Series(dtype=str), + ).astype(str): + relative_path = Path("masks") / f"{candidate_id}.npz" + candidate_mask = refined.candidate_masks[candidate_id] + save_candidate_mask( + sample_dir / relative_path, + mask=candidate_mask.mask, + bbox=candidate_mask.bbox, + image_shape_yx=candidate_mask.image_shape_yx, + sample_id=sample_id, + candidate_id=candidate_id, + profile_name=resolved_config.profile_name, + profile_fingerprint=fingerprint, + metrics=candidate_mask.metrics, + implementation_version=OOCYTE_IMPLEMENTATION_VERSION, + ) + mask_paths.append(str(relative_path)) + candidates["mask_path"] = mask_paths + persistence_seconds = time.perf_counter() - stage_start + + coarse_path = sample_dir / "coarse_candidates.csv" + candidates_path = sample_dir / "candidates.csv" + labels_path = sample_dir / "oocyte_labels.ome.tiff" + labels_mapping_path = sample_dir / "oocyte_labels.csv" + summary_path = sample_dir / "summary.json" + runtime_path = sample_dir / "runtime.json" + manifest_path = sample_dir / "run_manifest.json" + overview_path = sample_dir / "overview.png" + duplicate_suspects_path = sample_dir / "accepted_duplicate_suspects.csv" + rescue_diagnostics_path = sample_dir / "rescue_diagnostics.csv" + _atomic_write_csv(coarse.candidates, coarse_path) + _atomic_write_csv(candidates, candidates_path) + if refined.rescue_diagnostics is not None: + _atomic_write_csv(refined.rescue_diagnostics, rescue_diagnostics_path) + + stage_start = time.perf_counter() + label_export: LabelExportResult = export_whole_slide_labels( + candidates, + sample_dir=sample_dir, + image_shape_yx=coarse.image_shape_yx, + image_path=labels_path, + mapping_path=labels_mapping_path, + ) + export_seconds = time.perf_counter() - stage_start + stage_start = time.perf_counter() + spatial_qc = render_spatial_overview( + coarse.downsampled, + candidates, + downsample_factor=resolved_config.coarse.downsample_factor, + pixel_size_um=resolved_config.pixel_size_um, + sample_id=sample_id, + out_path=overview_path, + ) + _atomic_write_csv(spatial_qc.duplicate_suspects, duplicate_suspects_path) + spatial_qc_seconds = time.perf_counter() - stage_start + total_seconds = time.perf_counter() - total_start + runtime = { + "coarse_detection": float(coarse_seconds), + "local_refinement": float(refinement_seconds), + "mask_persistence": float(persistence_seconds), + "label_export": float(export_seconds), + "spatial_qc": float(spatial_qc_seconds), + "total": float(total_seconds), + } + artifact_paths = { + "coarse_candidates": coarse_path, + "candidates": candidates_path, + "masks": masks_dir, + "labels": labels_path, + "label_mapping": labels_mapping_path, + "summary": summary_path, + "runtime": runtime_path, + "run_manifest": manifest_path, + "overview": overview_path, + "duplicate_suspects": duplicate_suspects_path, + } + if refined.rescue_diagnostics is not None: + artifact_paths["rescue_diagnostics"] = rescue_diagnostics_path + accepted_count = int(candidates["accepted"].sum()) if not candidates.empty else 0 + rescue_candidate_count = int( + ( + candidates.get("segmentation_pass", pd.Series(dtype=str)) + == "secondary_rescue" + ).sum() + ) + summary = { + "schema_version": 1, + "sample_id": sample_id, + "status": "complete", + "image_shape_yx": list(coarse.image_shape_yx), + "channel_index": int(channel_index), + "profile_name": resolved_config.profile_name, + "profile_fingerprint": fingerprint, + "implementation_version": OOCYTE_IMPLEMENTATION_VERSION, + "coarse_candidate_count": int(len(coarse.candidates)), + "refined_candidate_count": int(len(candidates)), + "accepted_candidate_count": accepted_count, + "secondary_rescue_candidate_count": rescue_candidate_count, + "secondary_rescue_evaluation_count": int( + 0 + if refined.rescue_diagnostics is None + else len(refined.rescue_diagnostics) + ), + "label_count": int(label_export.label_count), + "assigned_label_pixel_count": int(label_export.assigned_pixel_count), + "overlap_label_pixel_count": int(label_export.overlap_pixel_count), + "accepted_duplicate_suspect_count": int( + len(spatial_qc.duplicate_suspects) + ), + "thresholds": coarse.thresholds, + "runtime_seconds": runtime, + "artifact_paths": artifact_paths, + } + manifest = { + "schema_version": 1, + "sample_id": sample_id, + "source_image": str(source_path), + "source_image_size_bytes": int(source_path.stat().st_size), + "antibodies_path": None + if antibodies_path is None + else str(Path(antibodies_path).resolve()), + "requested_channel_name": channel_name, + "resolved_channel_index": int(channel_index), + "resolved_config": resolved_config.to_dict(), + "profile_fingerprint": fingerprint, + "implementation_version": OOCYTE_IMPLEMENTATION_VERSION, + "outputs": artifact_paths, + } + _atomic_write_json(runtime, runtime_path) + _atomic_write_json(summary, summary_path) + _atomic_write_json(manifest, manifest_path) + return OocyteDetectionResult( + sample_id=sample_id, + image_path=source_path, + image_shape_yx=coarse.image_shape_yx, + channel_index=int(channel_index), + profile_name=resolved_config.profile_name, + profile_fingerprint=fingerprint, + implementation_version=OOCYTE_IMPLEMENTATION_VERSION, + coarse_candidates=coarse.candidates, + candidates=candidates, + thresholds=coarse.thresholds, + runtime_seconds=runtime, + artifact_paths=artifact_paths, + ) diff --git a/aegle/oocyte/export.py b/aegle/oocyte/export.py new file mode 100644 index 0000000..175a2b2 --- /dev/null +++ b/aegle/oocyte/export.py @@ -0,0 +1,199 @@ +"""Derived whole-slide label exports built from persisted candidate masks.""" + +from __future__ import annotations + +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Tuple + +import numpy as np +import pandas as pd +import tifffile + +from .io import load_candidate_mask + + +@dataclass(frozen=True) +class LabelExportResult: + image_path: Path + mapping_path: Path + label_count: int + assigned_pixel_count: int + overlap_pixel_count: int + + +def _atomic_write_csv(table: pd.DataFrame, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".csv", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + table.to_csv(handle, index=False) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def export_whole_slide_labels( + candidates: pd.DataFrame, + *, + sample_dir: Path, + image_shape_yx: Tuple[int, int], + image_path: Path, + mapping_path: Path, + tile_shape_yx: Tuple[int, int] = (512, 512), +) -> LabelExportResult: + """Compose accepted masks into one sparse whole-slide label OME-TIFF.""" + + required = { + "detector_component_id", + "accepted", + "detector_score", + "acceptance_mode", + "center_x", + "center_y", + "mask_path", + } + missing = required.difference(candidates.columns) + if missing: + raise ValueError(f"candidate table missing label-export columns: {sorted(missing)}") + image_h, image_w = (int(image_shape_yx[0]), int(image_shape_yx[1])) + if image_h <= 0 or image_w <= 0: + raise ValueError("whole-slide label dimensions must be positive") + if any(value <= 0 or value % 16 for value in tile_shape_yx): + raise ValueError("TIFF tile dimensions must be positive multiples of 16") + + accepted = candidates[candidates["accepted"].astype(bool)].copy() + accepted = accepted.sort_values( + ["detector_score", "detector_component_id"], + ascending=[False, True], + kind="stable", + ).reset_index(drop=True) + if len(accepted) > np.iinfo(np.uint16).max: + raise ValueError("uint16 label export supports at most 65535 accepted candidates") + + destination = Path(image_path) + destination.parent.mkdir(parents=True, exist_ok=True) + raw_path = None + temporary_tiff = None + mapping_rows = [] + total_assigned = 0 + total_overlap = 0 + try: + with tempfile.NamedTemporaryFile( + suffix=".labels.raw", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + raw_path = Path(handle.name) + handle.truncate(image_h * image_w * np.dtype(np.uint16).itemsize) + labels = np.memmap( + raw_path, + mode="r+", + dtype=np.uint16, + shape=(image_h, image_w), + ) + labels[:] = 0 + + sample_root = Path(sample_dir) + for label_value, record in enumerate(accepted.to_dict("records"), start=1): + candidate_mask_path = Path(str(record["mask_path"])) + if not candidate_mask_path.is_absolute(): + candidate_mask_path = sample_root / candidate_mask_path + persisted = load_candidate_mask(candidate_mask_path) + if persisted.image_shape_yx != (image_h, image_w): + raise ValueError( + f"candidate mask image shape mismatch: {candidate_mask_path}" + ) + bbox = persisted.bbox + region = labels[bbox.y0 : bbox.y1, bbox.x0 : bbox.x1] + foreground = persisted.mask + overlap = foreground & (region != 0) + writable = foreground & (region == 0) + overlap_count = int(overlap.sum()) + assigned_count = int(writable.sum()) + region[writable] = label_value + total_overlap += overlap_count + total_assigned += assigned_count + mapping_rows.append( + { + "label": int(label_value), + "detector_component_id": str(record["detector_component_id"]), + "detector_score": float(record["detector_score"]), + "acceptance_mode": str(record["acceptance_mode"]), + "center_x": int(record["center_x"]), + "center_y": int(record["center_y"]), + "bbox_x0": bbox.x0, + "bbox_y0": bbox.y0, + "bbox_x1": bbox.x1, + "bbox_y1": bbox.y1, + "mask_path": str(record["mask_path"]), + "assigned_pixel_count": assigned_count, + "overlap_pixel_count": overlap_count, + } + ) + labels.flush() + + with tempfile.NamedTemporaryFile( + suffix=".ome.tiff", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + temporary_tiff = Path(handle.name) + tifffile.imwrite( + temporary_tiff, + labels, + dtype=np.uint16, + bigtiff=True, + ome=True, + metadata={"axes": "YX"}, + photometric="minisblack", + tile=tile_shape_yx, + compression="zlib", + ) + del labels + temporary_tiff.replace(destination) + temporary_tiff = None + finally: + if temporary_tiff is not None and temporary_tiff.exists(): + temporary_tiff.unlink() + if raw_path is not None and raw_path.exists(): + raw_path.unlink() + + mapping_columns = [ + "label", + "detector_component_id", + "detector_score", + "acceptance_mode", + "center_x", + "center_y", + "bbox_x0", + "bbox_y0", + "bbox_x1", + "bbox_y1", + "mask_path", + "assigned_pixel_count", + "overlap_pixel_count", + ] + mapping = pd.DataFrame(mapping_rows, columns=mapping_columns) + _atomic_write_csv(mapping, Path(mapping_path)) + return LabelExportResult( + image_path=destination, + mapping_path=Path(mapping_path), + label_count=len(mapping), + assigned_pixel_count=total_assigned, + overlap_pixel_count=total_overlap, + ) diff --git a/aegle/oocyte/io.py b/aegle/oocyte/io.py new file mode 100644 index 0000000..90a6016 --- /dev/null +++ b/aegle/oocyte/io.py @@ -0,0 +1,276 @@ +"""Raw-image patch access and exact candidate-mask persistence.""" + +from __future__ import annotations + +import csv +import json +import os +import re +import tempfile +from pathlib import Path +from typing import Any, Dict, Tuple + +import numpy as np +import tifffile +import zarr + +from .models import BoundingBox, ExtractedPatch, PersistedMask, SegmentationMetrics + + +def find_channel_index(antibodies_path: Path, channel_name: str = "UCHL1") -> int: + """Resolve a marker name to its image channel index from a TSV table.""" + + path = Path(antibodies_path) + with path.open(newline="") as handle: + reader = csv.DictReader(handle, delimiter="\t") + if not reader.fieldnames: + raise ValueError(f"antibody table has no header: {path}") + name_column = next( + ( + name + for name in ("antibody_name", "marker_name", "name") + if name in reader.fieldnames + ), + None, + ) + if name_column is None: + raise ValueError(f"antibody table has no marker-name column: {path}") + + target = channel_name.strip().casefold() + for row_index, row in enumerate(reader): + if str(row.get(name_column, "")).strip().casefold() != target: + continue + channel_id = str(row.get("channel_id", "")) + match = re.search(r":(\d+)$", channel_id) + return int(match.group(1)) if match else row_index + raise ValueError(f"channel {channel_name!r} not found in antibody table: {path}") + + +def _patch_geometry( + image_shape_yx: Tuple[int, int], + center_xy: Tuple[int, int], + radius: int, +) -> Tuple[BoundingBox, Tuple[int, int, int, int]]: + image_h, image_w = image_shape_yx + center_x, center_y = center_xy + if radius < 1: + raise ValueError("patch radius must be positive") + + requested_x0 = center_x - radius + requested_x1 = center_x + radius + 1 + requested_y0 = center_y - radius + requested_y1 = center_y + radius + 1 + if ( + requested_x1 <= 0 + or requested_y1 <= 0 + or requested_x0 >= image_w + or requested_y0 >= image_h + ): + raise ValueError("requested patch does not intersect the source image") + x0 = max(0, requested_x0) + x1 = min(image_w, requested_x1) + y0 = max(0, requested_y0) + y1 = min(image_h, requested_y1) + bbox = BoundingBox(x0=x0, y0=y0, x1=x1, y1=y1) + padding = ( + y0 - requested_y0, + requested_y1 - y1, + x0 - requested_x0, + requested_x1 - x1, + ) + return bbox, padding + + +def extract_padded_patch( + image: np.ndarray, + center_xy: Tuple[int, int], + radius: int, +) -> ExtractedPatch: + """Extract an edge-padded fixed-size patch from a two-dimensional image.""" + + array = np.asarray(image) + if array.ndim != 2: + raise ValueError("source image must be two-dimensional") + image_shape = (int(array.shape[0]), int(array.shape[1])) + bbox, padding = _patch_geometry(image_shape, center_xy, radius) + patch = np.asarray(array[bbox.y0 : bbox.y1, bbox.x0 : bbox.x1]) + top, bottom, left, right = padding + if any(padding): + patch = np.pad(patch, ((top, bottom), (left, right)), mode="edge") + return ExtractedPatch( + image=patch, + bbox=bbox, + image_shape_yx=image_shape, + padding_tblr=padding, + ) + + +def extract_cyx_channel_patch( + array: Any, + channel_index: int, + center_xy: Tuple[int, int], + radius: int, +) -> ExtractedPatch: + """Extract a padded patch from an array-like object with C/Y/X axes.""" + + shape = tuple(int(value) for value in array.shape) + if len(shape) != 3: + raise ValueError("channel source must use C/Y/X axes") + if not 0 <= channel_index < shape[0]: + raise IndexError(f"channel index {channel_index} outside image channel range") + image_shape = (shape[1], shape[2]) + bbox, padding = _patch_geometry(image_shape, center_xy, radius) + patch = np.asarray( + array[channel_index, bbox.y0 : bbox.y1, bbox.x0 : bbox.x1] + ) + top, bottom, left, right = padding + if any(padding): + patch = np.pad(patch, ((top, bottom), (left, right)), mode="edge") + return ExtractedPatch( + image=patch, + bbox=bbox, + image_shape_yx=image_shape, + padding_tblr=padding, + ) + + +def read_ome_channel_patch( + image_path: Path, + channel_index: int, + center_xy: Tuple[int, int], + radius: int, +) -> ExtractedPatch: + """Read one cropped channel patch without loading a whole OME-TIFF plane.""" + + path = Path(image_path) + with tifffile.TiffFile(path) as tif: + series = tif.series[0] + axes = series.axes + shape = tuple(int(value) for value in series.shape) + if "C" not in axes or "Y" not in axes or "X" not in axes: + raise ValueError(f"expected C/Y/X axes in OME-TIFF, got {axes!r}") + + channel_axis = axes.index("C") + y_axis = axes.index("Y") + x_axis = axes.index("X") + if not 0 <= channel_index < shape[channel_axis]: + raise IndexError( + f"channel index {channel_index} outside [0, {shape[channel_axis]})" + ) + image_shape = (shape[y_axis], shape[x_axis]) + bbox, padding = _patch_geometry(image_shape, center_xy, radius) + + store = series.aszarr() + array = zarr.open(store, mode="r") + indexer = [] + for axis, axis_size in zip(axes, shape): + if axis == "C": + indexer.append(channel_index) + elif axis == "Y": + indexer.append(slice(bbox.y0, bbox.y1)) + elif axis == "X": + indexer.append(slice(bbox.x0, bbox.x1)) + elif axis_size == 1: + indexer.append(0) + else: + raise ValueError( + f"unsupported non-singleton OME axis {axis!r} with size {axis_size}" + ) + patch = np.asarray(array[tuple(indexer)]) + + if patch.ndim != 2: + raise ValueError(f"channel patch did not resolve to two dimensions: {patch.shape}") + top, bottom, left, right = padding + if any(padding): + patch = np.pad(patch, ((top, bottom), (left, right)), mode="edge") + return ExtractedPatch( + image=patch, + bbox=bbox, + image_shape_yx=image_shape, + padding_tblr=padding, + ) + + +def save_candidate_mask( + path: Path, + *, + mask: np.ndarray, + bbox: BoundingBox, + image_shape_yx: Tuple[int, int], + sample_id: str, + candidate_id: str, + profile_name: str, + profile_fingerprint: str, + metrics: SegmentationMetrics, + implementation_version: str | None = None, +) -> Path: + """Atomically persist the exact cropped mask used for candidate scoring.""" + + mask_array = np.asarray(mask, dtype=np.bool_) + if mask_array.shape != bbox.shape_yx: + raise ValueError("mask shape must match its image-space bounding box") + image_h, image_w = image_shape_yx + if bbox.x1 > image_w or bbox.y1 > image_h: + raise ValueError("mask bounding box exceeds source image dimensions") + if not sample_id or not candidate_id or not profile_name or not profile_fingerprint: + raise ValueError("mask identity and profile fields must not be empty") + + metadata: Dict[str, Any] = { + "schema_version": 1, + "sample_id": sample_id, + "candidate_id": candidate_id, + "profile_name": profile_name, + "profile_fingerprint": profile_fingerprint, + "implementation_version": implementation_version, + "metrics": metrics.to_dict(), + } + metadata_json = json.dumps(metadata, sort_keys=True, separators=(",", ":")) + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + suffix=".npz", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + np.savez_compressed( + handle, + mask=mask_array, + bbox_xyxy=np.asarray(bbox.as_tuple(), dtype=np.int64), + image_shape_yx=np.asarray(image_shape_yx, dtype=np.int64), + metadata_json=np.asarray(metadata_json), + ) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + return destination + + +def load_candidate_mask(path: Path) -> PersistedMask: + """Load and validate a cropped mask archive without pickle support.""" + + with np.load(Path(path), allow_pickle=False) as archive: + required = {"mask", "bbox_xyxy", "image_shape_yx", "metadata_json"} + missing = required.difference(archive.files) + if missing: + raise ValueError(f"candidate mask archive missing fields: {sorted(missing)}") + mask = np.asarray(archive["mask"], dtype=np.bool_) + bbox_values = tuple(int(value) for value in archive["bbox_xyxy"].tolist()) + image_shape = tuple(int(value) for value in archive["image_shape_yx"].tolist()) + metadata = json.loads(str(archive["metadata_json"].item())) + + if len(bbox_values) != 4 or len(image_shape) != 2: + raise ValueError("candidate mask archive has invalid geometry") + return PersistedMask( + mask=mask, + bbox=BoundingBox(*bbox_values), + image_shape_yx=image_shape, + metadata=metadata, + ) diff --git a/aegle/oocyte/manual_seed_finalize.py b/aegle/oocyte/manual_seed_finalize.py new file mode 100644 index 0000000..f5b1223 --- /dev/null +++ b/aegle/oocyte/manual_seed_finalize.py @@ -0,0 +1,617 @@ +"""Finalize reviewed manual-seed masks into versioned label artifacts.""" + +from __future__ import annotations + +import json +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping, Sequence, Tuple + +import numpy as np +import pandas as pd + +from .export import LabelExportResult, export_whole_slide_labels +from .io import load_candidate_mask +from .models import BoundingBox, PersistedMask +from .recall_review import ( + _atomic_write_text, + _file_sha256, + _json_safe, + _load_sample, + _mask_path, + _read_json, +) +from .recall_overlay import overlay_dir_from_identity + + +MANUAL_SEED_PROFILE_NAME = "manual_seed_review_v1" +ACCEPTED_CHOICES = { + "accept_manual_conservative": "conservative", + "accept_manual_expanded": "expanded", +} +ALLOWED_CHOICES = set(ACCEPTED_CHOICES) | {"neither", "duplicate", "unsure"} + + +@dataclass(frozen=True) +class ManualSeedFinalizeResult: + out_dir: Path + decisions_path: Path + candidates_path: Path + overlap_audit_path: Path + delta_labels: LabelExportResult + combined_labels: LabelExportResult | None + combined_candidates_path: Path | None + manifest_path: Path + accepted_count: int + boundary_warning_count: int + + +def _atomic_write_csv(table: pd.DataFrame, path: Path) -> None: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + newline="", + prefix=f".{destination.name}.", + suffix=".tmp", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + table.to_csv(handle, index=False) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _boundary_warning(notes: str) -> bool: + normalized = notes.strip().casefold() + return bool( + normalized + and any( + term in normalized + for term in ( + "boundary", + "misses part", + "undersegment", + "under-segment", + "oversegment", + "over-segment", + ) + ) + ) + + +def _tight_mask(mask: PersistedMask) -> PersistedMask: + ys, xs = np.nonzero(mask.mask) + if not len(xs): + raise ValueError("reviewed mask contains no foreground pixels") + local_x0, local_x1 = int(xs.min()), int(xs.max()) + 1 + local_y0, local_y1 = int(ys.min()), int(ys.max()) + 1 + bbox = BoundingBox( + mask.bbox.x0 + local_x0, + mask.bbox.y0 + local_y0, + mask.bbox.x0 + local_x1, + mask.bbox.y0 + local_y1, + ) + return PersistedMask( + mask=np.asarray( + mask.mask[local_y0:local_y1, local_x0:local_x1], + dtype=np.bool_, + ), + bbox=bbox, + image_shape_yx=mask.image_shape_yx, + metadata=mask.metadata, + ) + + +def _write_reviewed_mask( + path: Path, + *, + mask: PersistedMask, + metadata: Mapping[str, Any], +) -> Path: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + prefix=f".{destination.name}.", + suffix=".npz", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + np.savez_compressed( + handle, + mask=mask.mask, + bbox_xyxy=np.asarray(mask.bbox.as_tuple(), dtype=np.int64), + image_shape_yx=np.asarray(mask.image_shape_yx, dtype=np.int64), + metadata_json=np.asarray( + json.dumps( + _json_safe(dict(metadata)), + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + ), + ) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + return destination + + +def _overlap_metrics(left: PersistedMask, right: PersistedMask) -> Tuple[int, float, float]: + x0 = max(left.bbox.x0, right.bbox.x0) + y0 = max(left.bbox.y0, right.bbox.y0) + x1 = min(left.bbox.x1, right.bbox.x1) + y1 = min(left.bbox.y1, right.bbox.y1) + if x0 >= x1 or y0 >= y1: + return 0, 0.0, 0.0 + left_region = left.mask[ + y0 - left.bbox.y0 : y1 - left.bbox.y0, + x0 - left.bbox.x0 : x1 - left.bbox.x0, + ] + right_region = right.mask[ + y0 - right.bbox.y0 : y1 - right.bbox.y0, + x0 - right.bbox.x0 : x1 - right.bbox.x0, + ] + overlap = int(np.logical_and(left_region, right_region).sum()) + return ( + overlap, + overlap / max(int(left.mask.sum()), 1), + overlap / max(int(right.mask.sum()), 1), + ) + + +def _overlap_audit( + manual_masks: Sequence[Tuple[str, PersistedMask]], + production_masks: Sequence[Tuple[str, PersistedMask]], +) -> pd.DataFrame: + rows = [] + + def add_overlap( + left_scope: str, + left_id: str, + left: PersistedMask, + right_scope: str, + right_id: str, + right: PersistedMask, + ) -> None: + pixels, left_fraction, right_fraction = _overlap_metrics(left, right) + if not pixels: + return + rows.append( + { + "left_scope": left_scope, + "left_id": left_id, + "right_scope": right_scope, + "right_id": right_id, + "overlap_pixel_count": pixels, + "left_overlap_fraction": left_fraction, + "right_overlap_fraction": right_fraction, + "smaller_mask_overlap_fraction": max(left_fraction, right_fraction), + } + ) + + for left_index, (left_id, left) in enumerate(manual_masks): + for right_id, right in manual_masks[left_index + 1 :]: + add_overlap("manual", left_id, left, "manual", right_id, right) + for right_id, right in production_masks: + add_overlap("manual", left_id, left, "production", right_id, right) + columns = [ + "left_scope", + "left_id", + "right_scope", + "right_id", + "overlap_pixel_count", + "left_overlap_fraction", + "right_overlap_fraction", + "smaller_mask_overlap_fraction", + ] + return pd.DataFrame(rows, columns=columns) + + +def _validate_review_identity(sample_identity: Mapping[str, Any], payload: Mapping[str, Any]) -> None: + if payload.get("schema_version") != 1: + raise ValueError("manual-seed review schema_version must be 1") + if payload.get("review_type") != "manual_seed_mask_review": + raise ValueError("review_type must be 'manual_seed_mask_review'") + identity = payload.get("identity") + if not isinstance(identity, Mapping): + raise ValueError("manual-seed review is missing its identity object") + for field, expected in sample_identity.items(): + if identity.get(field) != expected: + raise ValueError(f"manual-seed review identity mismatch for {field}") + if not identity.get("analysis_sha256"): + raise ValueError("manual-seed review identity is missing analysis_sha256") + + +def finalize_manual_seed_review( + sample_dir: Path, + review_json: Path, + out_dir: Path, + *, + analysis_dir: Path | None = None, + write_combined_labels: bool = True, + tile_shape_yx: Tuple[int, int] = (512, 512), + max_smaller_overlap_fraction: float = 0.25, +) -> ManualSeedFinalizeResult: + """Validate a completed boundary review and write immutable label artifacts.""" + + review_path = Path(review_json).resolve() + payload = _read_json(review_path) + review_identity = payload.get("identity") + if not isinstance(review_identity, Mapping): + raise ValueError("manual-seed review is missing its identity object") + sample = _load_sample( + sample_dir, + overlay_dir=overlay_dir_from_identity(review_identity), + ) + _validate_review_identity(sample.review_identity, payload) + identity = payload["identity"] + if analysis_dir is None: + analysis_path = Path(str(identity.get("analysis_table", ""))).resolve() + else: + analysis_path = Path(analysis_dir).resolve() / "recall_failure_analysis.csv" + if not analysis_path.is_file(): + raise FileNotFoundError(f"recall analysis table does not exist: {analysis_path}") + analysis_sha256 = _file_sha256(analysis_path) + if analysis_sha256 != str(identity["analysis_sha256"]): + raise ValueError("recall analysis SHA-256 does not match the review identity") + + review_rows = payload.get("rows") + if not isinstance(review_rows, list): + raise ValueError("manual-seed review rows must be a list") + analysis = pd.read_csv(analysis_path) + if analysis["annotation_id"].duplicated().any(): + raise ValueError("recall analysis contains duplicate annotation_id values") + analysis_by_id = analysis.set_index("annotation_id", drop=False) + review_ids = [str(row.get("annotation_id", "")) for row in review_rows] + if len(review_ids) != len(set(review_ids)): + raise ValueError("manual-seed review contains duplicate annotation_id values") + if set(review_ids) != set(str(value) for value in analysis["annotation_id"]): + raise ValueError("manual-seed review rows do not match the recall analysis") + + review_sha256 = _file_sha256(review_path) + selections = [] + decisions = [] + choice_counts: Dict[str, int] = {} + analysis_root = analysis_path.parent.resolve() + for review_index, raw_row in enumerate(review_rows, start=1): + if not isinstance(raw_row, Mapping): + raise ValueError(f"manual-seed review row {review_index} must be an object") + annotation_id = str(raw_row["annotation_id"]) + source = analysis_by_id.loc[annotation_id] + if not ( + np.isclose(float(raw_row["x"]), float(source["x"]), atol=0.01) + and np.isclose(float(raw_row["y"]), float(source["y"]), atol=0.01) + ): + raise ValueError(f"manual-seed review coordinates changed for {annotation_id}") + choice = str(raw_row.get("manual_mask_choice", "")).strip() + if choice not in ALLOWED_CHOICES: + raise ValueError(f"invalid or missing manual mask choice for {annotation_id}") + choice_counts[choice] = choice_counts.get(choice, 0) + 1 + notes = str(raw_row.get("manual_notes", "")).strip() + warning = _boundary_warning(notes) + decision: Dict[str, Any] = { + "review_index": review_index, + "annotation_id": annotation_id, + "x": float(source["x"]), + "y": float(source["y"]), + "failure_class": str(source["failure_class"]), + "manual_mask_choice": choice, + "accepted": choice in ACCEPTED_CHOICES, + "selected_variant": ACCEPTED_CHOICES.get(choice, ""), + "manual_notes": notes, + "boundary_warning": warning, + "source_mask_path": "", + "source_mask_sha256": "", + "reviewed_mask_path": "", + "reviewed_mask_sha256": "", + } + if choice in ACCEPTED_CHOICES: + variant = ACCEPTED_CHOICES[choice] + source_column = f"manual_{variant}_mask_path" + source_mask_path = Path(str(source[source_column])).resolve() + if not source_mask_path.is_file(): + raise FileNotFoundError(f"selected provisional mask is missing: {source_mask_path}") + if not source_mask_path.is_relative_to(analysis_root): + raise ValueError( + f"selected provisional mask is outside the analysis directory: {source_mask_path}" + ) + provisional = load_candidate_mask(source_mask_path) + if provisional.image_shape_yx != sample.image_shape_yx: + raise ValueError(f"selected mask image shape mismatch: {source_mask_path}") + if str(provisional.metadata.get("annotation_id", "")) != annotation_id: + raise ValueError(f"selected mask annotation mismatch: {source_mask_path}") + candidate_id = f"manual_seed_{review_index:03d}" + selections.append( + { + "review_index": review_index, + "candidate_id": candidate_id, + "annotation_id": annotation_id, + "variant": variant, + "choice": choice, + "notes": notes, + "boundary_warning": warning, + "source": source, + "source_mask_path": source_mask_path, + "source_mask_sha256": _file_sha256(source_mask_path), + "mask": _tight_mask(provisional), + } + ) + decision["source_mask_path"] = str(source_mask_path) + decision["source_mask_sha256"] = selections[-1]["source_mask_sha256"] + decisions.append(decision) + + if not selections: + raise ValueError("manual-seed review accepted no masks to finalize") + + production_masks = [] + for record in sample.candidates.to_dict("records"): + candidate_id = str(record["detector_component_id"]) + persisted = load_candidate_mask(_mask_path(sample.sample_dir, record)) + if persisted.image_shape_yx != sample.image_shape_yx: + raise ValueError(f"production mask image shape mismatch: {candidate_id}") + production_masks.append((candidate_id, persisted)) + audit = _overlap_audit( + [(str(item["candidate_id"]), item["mask"]) for item in selections], + production_masks, + ) + blocking = audit[ + audit["smaller_mask_overlap_fraction"] >= max_smaller_overlap_fraction + ] + if not blocking.empty: + pairs = ", ".join( + f"{row.left_id}/{row.right_id}={row.smaller_mask_overlap_fraction:.3f}" + for row in blocking.itertuples() + ) + raise ValueError(f"reviewed masks have blocking overlap: {pairs}") + + destination = Path(out_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + manifest_path = destination / "manual_seed_finalize_manifest.json" + if manifest_path.exists(): + manifest_path.unlink() + masks_dir = destination / "reviewed_masks" + masks_dir.mkdir(parents=True, exist_ok=True) + expected_mask_names = {f"{item['candidate_id']}.npz" for item in selections} + for existing in masks_dir.glob("*.npz"): + if existing.name not in expected_mask_names: + existing.unlink() + + candidate_rows = [] + decision_by_id = {str(row["annotation_id"]): row for row in decisions} + for item in selections: + candidate_id = str(item["candidate_id"]) + reviewed_path = masks_dir / f"{candidate_id}.npz" + source = item["source"] + variant = str(item["variant"]) + metrics = dict(item["mask"].metadata.get("metrics", {})) + metadata = { + **dict(item["mask"].metadata), + "schema_version": 1, + "sample_id": sample.sample_id, + "candidate_id": candidate_id, + "profile_name": MANUAL_SEED_PROFILE_NAME, + "base_profile_name": sample.profile_name, + "base_profile_fingerprint": sample.profile_fingerprint, + "implementation_version": sample.implementation_version, + "reviewed_manual_seed": True, + "provisional_only": False, + "source_annotation_id": item["annotation_id"], + "manual_mask_choice": item["choice"], + "manual_notes": item["notes"], + "boundary_warning": item["boundary_warning"], + "review_json_sha256": review_sha256, + "review_exported_at": payload.get("exported_at"), + "analysis_sha256": analysis_sha256, + "source_mask_path": str(item["source_mask_path"]), + "source_mask_sha256": item["source_mask_sha256"], + } + _write_reviewed_mask(reviewed_path, mask=item["mask"], metadata=metadata) + reviewed_sha256 = _file_sha256(reviewed_path) + decision = decision_by_id[str(item["annotation_id"])] + decision["reviewed_mask_path"] = str(reviewed_path) + decision["reviewed_mask_sha256"] = reviewed_sha256 + ys, xs = np.nonzero(item["mask"].mask) + centroid_x = item["mask"].bbox.x0 + float(xs.mean()) + centroid_y = item["mask"].bbox.y0 + float(ys.mean()) + percentile = source[f"manual_{variant}_percentile"] + candidate_rows.append( + { + "detector_component_id": candidate_id, + "source_annotation_id": item["annotation_id"], + "display_id": f"#R{int(item['review_index']):03d}", + "html_id": f"manual-seed-{int(item['review_index']):03d}", + "accepted": True, + "accepted_strict": False, + "accepted_rescue": False, + "detector_score": 1.0, + "acceptance_mode": f"manual_seed_reviewed_{variant}", + "detection_pass": MANUAL_SEED_PROFILE_NAME, + "segmentation_pass": f"manual_{variant}", + "center_x": int(round(centroid_x)), + "center_y": int(round(centroid_y)), + "component_centroid_x": centroid_x, + "component_centroid_y": centroid_y, + "bbox_x0": item["mask"].bbox.x0, + "bbox_y0": item["mask"].bbox.y0, + "bbox_x1": item["mask"].bbox.x1, + "bbox_y1": item["mask"].bbox.y1, + "local_area_px": int(item["mask"].mask.sum()), + "local_equivalent_diameter_um": metrics.get("equivalent_diameter_um"), + "local_major_axis_um": metrics.get("major_axis_um"), + "local_minor_axis_um": metrics.get("minor_axis_um"), + "local_eccentricity": metrics.get("eccentricity"), + "local_solidity": metrics.get("solidity"), + "local_circularity": metrics.get("circularity"), + "local_centroid_offset_px": metrics.get("centroid_offset_px"), + "local_mean_intensity": metrics.get("mean_intensity"), + "local_max_intensity": metrics.get("max_intensity"), + "threshold_method": metrics.get("threshold_method"), + "threshold": metrics.get("threshold"), + "selection_mode": metrics.get("selection_mode"), + "score_background_percentile": percentile, + "failure_class": source["failure_class"], + "manual_review_index": int(item["review_index"]), + "manual_mask_choice": item["choice"], + "manual_notes": item["notes"], + "boundary_warning": bool(item["boundary_warning"]), + "quality_class": ( + "reviewed_boundary_warning" + if item["boundary_warning"] + else "reviewed_manual_seed" + ), + "mask_path": str(reviewed_path.relative_to(destination)), + "mask_source_dir": str(destination), + "source_provisional_mask_path": str(item["source_mask_path"]), + "source_provisional_mask_sha256": item["source_mask_sha256"], + "reviewed_mask_sha256": reviewed_sha256, + "review_json_sha256": review_sha256, + "duplicate_suppressed": False, + } + ) + + decisions_path = destination / "manual_seed_review_decisions.csv" + candidates_path = destination / "manual_seed_accepted_candidates.csv" + overlap_audit_path = destination / "mask_overlap_audit.csv" + _atomic_write_csv(pd.DataFrame(decisions), decisions_path) + candidates = pd.DataFrame(candidate_rows) + _atomic_write_csv(candidates, candidates_path) + _atomic_write_csv(audit, overlap_audit_path) + + delta_labels = export_whole_slide_labels( + candidates, + sample_dir=destination, + image_shape_yx=sample.image_shape_yx, + image_path=destination / "oocyte_labels_manual_seed_delta_v1.ome.tiff", + mapping_path=destination / "oocyte_labels_manual_seed_delta_v1_mapping.csv", + tile_shape_yx=tile_shape_yx, + ) + + combined_labels = None + combined_candidates_path = None + if write_combined_labels: + production = sample.candidates.copy() + production["mask_path"] = [ + str(_mask_path(sample.sample_dir, record).resolve()) + for record in sample.candidates.to_dict("records") + ] + production["mask_source_dir"] = "" + combined = pd.concat([production, candidates], ignore_index=True, sort=False) + combined_candidates_path = ( + destination / "oocyte_candidates_rescue_v1_plus_manual_seed_v1.csv" + ) + _atomic_write_csv(combined, combined_candidates_path) + combined_labels = export_whole_slide_labels( + combined, + sample_dir=destination, + image_shape_yx=sample.image_shape_yx, + image_path=( + destination + / "oocyte_labels_rescue_v1_plus_manual_seed_v1.ome.tiff" + ), + mapping_path=( + destination + / "oocyte_labels_rescue_v1_plus_manual_seed_v1_mapping.csv" + ), + tile_shape_yx=tile_shape_yx, + ) + + artifact_paths = [ + decisions_path, + candidates_path, + overlap_audit_path, + delta_labels.image_path, + delta_labels.mapping_path, + *(Path(row["reviewed_mask_path"]) for row in decisions if row["reviewed_mask_path"]), + ] + if combined_labels is not None and combined_candidates_path is not None: + artifact_paths.extend( + [ + combined_candidates_path, + combined_labels.image_path, + combined_labels.mapping_path, + ] + ) + manifest = { + "schema_version": 1, + "delivery_name": "reviewed_manual_seed_delta_v1", + "sample": sample.review_identity, + "review_identity": dict(identity), + "review_json": str(review_path), + "review_json_sha256": review_sha256, + "review_exported_at": payload.get("exported_at"), + "analysis_table": str(analysis_path), + "analysis_sha256": analysis_sha256, + "choice_counts": choice_counts, + "reviewed_row_count": len(review_rows), + "accepted_manual_mask_count": len(candidates), + "boundary_warning_count": int(candidates["boundary_warning"].sum()), + "production_candidate_count": int(sample.candidates["accepted"].astype(bool).sum()), + "combined_label_count": ( + None if combined_labels is None else combined_labels.label_count + ), + "manual_overlap_audit_row_count": len(audit), + "max_smaller_overlap_fraction": ( + 0.0 if audit.empty else float(audit["smaller_mask_overlap_fraction"].max()) + ), + "label_export": { + "delta_label_count": delta_labels.label_count, + "delta_assigned_pixel_count": delta_labels.assigned_pixel_count, + "delta_overlap_pixel_count": delta_labels.overlap_pixel_count, + "combined_assigned_pixel_count": ( + None if combined_labels is None else combined_labels.assigned_pixel_count + ), + "combined_overlap_pixel_count": ( + None if combined_labels is None else combined_labels.overlap_pixel_count + ), + }, + "production_outputs_modified": False, + "artifacts": { + str(path.relative_to(destination)): { + "path": str(path), + "sha256": _file_sha256(path), + "size_bytes": path.stat().st_size, + } + for path in artifact_paths + }, + } + _atomic_write_text( + manifest_path, + json.dumps(_json_safe(manifest), indent=2, sort_keys=True, allow_nan=False), + ) + return ManualSeedFinalizeResult( + out_dir=destination, + decisions_path=decisions_path, + candidates_path=candidates_path, + overlap_audit_path=overlap_audit_path, + delta_labels=delta_labels, + combined_labels=combined_labels, + combined_candidates_path=combined_candidates_path, + manifest_path=manifest_path, + accepted_count=len(candidates), + boundary_warning_count=int(candidates["boundary_warning"].sum()), + ) + + +__all__ = [ + "MANUAL_SEED_PROFILE_NAME", + "ManualSeedFinalizeResult", + "finalize_manual_seed_review", +] diff --git a/aegle/oocyte/manual_seed_review.py b/aegle/oocyte/manual_seed_review.py new file mode 100644 index 0000000..61e7a94 --- /dev/null +++ b/aegle/oocyte/manual_seed_review.py @@ -0,0 +1,258 @@ +"""Static side-by-side review for human-seeded provisional oocyte masks.""" + +from __future__ import annotations + +import html +import io +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping + +import numpy as np +import pandas as pd +from PIL import Image, ImageDraw +from scipy import ndimage as ndi + +from .models import BoundingBox, PersistedMask +from .recall_review import ( + RecallReviewRuntime, + _atomic_write_text, + _file_sha256, + _identity_contains_required, + _json_safe, + _load_sample, + _read_json, +) +from .recall_overlay import overlay_dir_from_identity + + +@dataclass(frozen=True) +class ManualSeedReviewResult: + page_path: Path + assets_dir: Path + card_count: int + + +def _load_provisional(path: Any) -> PersistedMask | None: + if path is None or pd.isna(path) or not str(path).strip(): + return None + source = Path(str(path)) + if not source.is_file(): + return None + with np.load(source, allow_pickle=False) as archive: + mask = np.asarray(archive["mask"], dtype=np.bool_) + bbox = BoundingBox(*(int(value) for value in archive["bbox_xyxy"].tolist())) + image_shape = tuple(int(value) for value in archive["image_shape_yx"].tolist()) + metadata = json.loads(str(archive["metadata_json"].item())) + return PersistedMask( + mask=mask, + bbox=bbox, + image_shape_yx=(image_shape[0], image_shape[1]), + metadata=metadata, + ) + + +def _draw_mask( + image: Image.Image, + placed: np.ndarray, + *, + color: tuple[int, int, int], +) -> Image.Image: + rgba = np.asarray(image.convert("RGBA")).copy() + if placed.any(): + rgb = rgba[..., :3] + foreground = rgb[placed].astype(np.float32) + rgb[placed] = np.asarray( + 0.78 * foreground + 0.22 * np.asarray(color, dtype=np.float32), + dtype=np.uint8, + ) + boundary = np.logical_xor(placed, ndi.binary_erosion(placed)) + boundary = ndi.binary_dilation(boundary, iterations=1) + rgb[boundary] = np.asarray(color, dtype=np.uint8) + rgba[..., 3][boundary] = 255 + return Image.fromarray(rgba) + + +def _crosshair(image: Image.Image, x: int, y: int) -> None: + draw = ImageDraw.Draw(image) + color = (255, 87, 57, 255) + draw.ellipse((x - 11, y - 11, x + 11, y + 11), outline=color, width=3) + draw.line((x - 16, y, x + 16, y), fill=color, width=3) + draw.line((x, y - 16, x, y + 16), fill=color, width=3) + + +def _panel_label(image: Image.Image, label: str, *, color: tuple[int, int, int]) -> Image.Image: + output = Image.new("RGBA", (image.width, image.height + 34), (23, 31, 28, 255)) + output.paste(image.convert("RGBA"), (0, 34)) + draw = ImageDraw.Draw(output) + draw.rectangle((0, 0, image.width, 34), fill=(23, 31, 28, 255)) + draw.text((12, 9), label, fill=(*color, 255)) + return output + + +def _render_card( + runtime: RecallReviewRuntime, + row: Mapping[str, Any], + destination: Path, + *, + radius: int, +) -> None: + center = (int(round(float(row["x"]))), int(round(float(row["y"])))) + patch = runtime.source.read_patch(center, radius) + raw = Image.open(io.BytesIO(runtime.render_patch(center, radius, "local"))).convert("RGBA") + existing = Image.open(io.BytesIO(runtime.render_overlay(center, radius))).convert("RGBA") + base = Image.alpha_composite(raw, existing) + requested_x0 = center[0] - radius + requested_y0 = center[1] - radius + click_x = int(round(float(row["x"]))) - requested_x0 + click_y = int(round(float(row["y"]))) - requested_y0 + + existing_panel = base.copy() + _crosshair(existing_panel, click_x, click_y) + existing_panel = _panel_label( + existing_panel, + "RAW + EXISTING MASKS + MANUAL CENTER", + color=(0, 255, 242), + ) + + panels = [existing_panel] + for prefix, color, label in ( + ("manual_conservative", (87, 255, 127), "CONSERVATIVE PROVISIONAL MASK"), + ("manual_expanded", (255, 211, 72), "EXPANDED PROVISIONAL MASK"), + ): + provisional = _load_provisional(row.get(f"{prefix}_mask_path")) + panel = base.copy() + if provisional is not None: + placed = runtime._place_mask(provisional, patch) + panel = _draw_mask(panel, placed, color=color) + _crosshair(panel, click_x, click_y) + suffix = "" if provisional is not None else " (UNAVAILABLE)" + panels.append(_panel_label(panel, label + suffix, color=color)) + + gutter = 5 + canvas = Image.new( + "RGBA", + (sum(panel.width for panel in panels) + gutter * (len(panels) - 1), panels[0].height), + (245, 238, 224, 255), + ) + x_offset = 0 + for panel in panels: + canvas.paste(panel, (x_offset, 0)) + x_offset += panel.width + gutter + destination.parent.mkdir(parents=True, exist_ok=True) + canvas.convert("RGB").save(destination, format="WEBP", quality=88, method=6) + + +def _metric(value: Any, digits: int = 2) -> str: + try: + number = float(value) + except (TypeError, ValueError): + return "n/a" + if not np.isfinite(number): + return "n/a" + return f"{number:.{digits}f}" + + +def _card(row: Mapping[str, Any], index: int) -> str: + annotation_id = html.escape(str(row["annotation_id"]), quote=True) + card_id = f"seed-{index:03d}" + failure_class = html.escape(str(row["failure_class"])) + note_value = "" if pd.isna(row.get("notes")) else str(row.get("notes", "")) + return f'''
+ Conservative and expanded provisional masks for {annotation_id} +

#{index:03d}

{failure_class}
+
{annotation_id} · x {_metric(row['x'],1)} · y {_metric(row['y'],1)}
+
conservative P{_metric(row.get('manual_conservative_percentile'),0)} · d {_metric(row.get('manual_conservative_equivalent_diameter_um'),1)} umconservative circ {_metric(row.get('manual_conservative_circularity'))}conservative solid {_metric(row.get('manual_conservative_solidity'))}expanded P{_metric(row.get('manual_expanded_percentile'),0)} · d {_metric(row.get('manual_expanded_equivalent_diameter_um'),1)} umexpanded circ {_metric(row.get('manual_expanded_circularity'))}expanded solid {_metric(row.get('manual_expanded_solidity'))}
+
+ +
''' + + +_CSS = r''' +:root{--ink:#172522;--panel:#fffaf0;--paper:#eee6d8;--teal:#087b78;--line:#d1c6b3;--amber:#d97b24;--red:#a94336;--green:#458d68}*{box-sizing:border-box}body{margin:0;color:var(--ink);background:radial-gradient(circle at 12% 0,#fff9ec 0,transparent 28%),linear-gradient(135deg,#e9dfce,#f7f1e5 65%,#e0ece5);font-family:"Iowan Old Style","Palatino Linotype",Palatino,serif}.shell{width:min(1680px,calc(100% - 24px));margin:auto}.hero{margin:16px 0;padding:24px 28px;border:1px solid var(--line);border-radius:22px;background:linear-gradient(115deg,#fffaf0,#e4f3ec);box-shadow:0 15px 35px rgba(30,45,38,.12)}.hero h1{font-size:clamp(2rem,4.5vw,4.4rem);line-height:.95;margin:.2em 0}.eyebrow,.mono{font-family:"IBM Plex Mono","Aptos Mono","Courier New",monospace}.eyebrow{text-transform:uppercase;letter-spacing:.14em;color:var(--teal);font-size:.75rem;font-weight:700}.hero p{max-width:1000px;line-height:1.5}.stats{display:flex;gap:10px;flex-wrap:wrap}.stat{padding:10px 14px;border:1px solid var(--line);border-radius:12px;background:#fffaf0}.toolbar{position:sticky;top:6px;z-index:5;display:flex;gap:8px;align-items:center;flex-wrap:wrap;padding:10px 12px;margin:14px 0;border:1px solid var(--line);border-radius:14px;background:rgba(255,250,240,.95);backdrop-filter:blur(8px)}button,.button,input{font:inherit;border:1px solid #b9ae9d;border-radius:10px;padding:8px 11px;background:#fffaf0;color:var(--ink)}button{cursor:pointer}.button{text-decoration:none}.grow{flex:1}.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(650px,1fr));gap:14px;margin-bottom:50px}.card{background:var(--panel);border:1px solid var(--line);border-radius:17px;overflow:hidden;box-shadow:0 9px 24px rgba(32,45,39,.08)}.card.hidden{display:none}.card img{width:100%;display:block;background:#171f1c}.body{padding:13px}.title{display:flex;align-items:center;justify-content:space-between}.title h2{margin:0}.title span{font-family:monospace;color:var(--amber)}.metrics{display:grid;grid-template-columns:repeat(3,1fr);gap:6px;margin:10px 0;font:12px monospace}.metrics span{background:#f0e7d8;padding:6px;border-radius:7px}.actions{display:grid;grid-template-columns:repeat(5,1fr);gap:5px}.actions button.selected[data-choice=accept_manual_conservative]{background:#d8f0dd;border-color:var(--green)}.actions button.selected[data-choice=accept_manual_expanded]{background:#f6e6a9;border-color:#bd9022}.actions button.selected[data-choice=neither]{background:#f1d6cf;border-color:var(--red)}.actions button.selected[data-choice=duplicate]{background:#dae5ee;border-color:#557b98}.actions button.selected[data-choice=unsure]{background:#eee3ca;border-color:#9f834a}.notes{width:100%;margin-top:8px}.footer{color:#68726c;padding:10px 0 50px}@media(max-width:720px){.shell{width:min(100% - 10px,1680px)}.hero{padding:18px}.cards{grid-template-columns:1fr}.metrics{grid-template-columns:1fr 1fr}.actions{grid-template-columns:1fr 1fr}.toolbar{position:static}} +''' + + +_JS = r''' +const DATA=JSON.parse(document.getElementById('seed-data').textContent),KEY='aegle-oocyte-manual-seed-review:'+DATA.identity.sample_id+':'+DATA.identity.analysis_sha256;let state=JSON.parse(localStorage.getItem(KEY)||'{}'),filter='all';const cards=[...document.querySelectorAll('.card')];function save(){localStorage.setItem(KEY,JSON.stringify(state));paintProgress()}function paint(card){const id=card.dataset.id,s=state[id]||{};card.dataset.review=s.choice||'unreviewed';card.querySelectorAll('[data-choice]').forEach(b=>b.classList.toggle('selected',b.dataset.choice===s.choice));card.querySelector('.notes').value=s.notes??card.querySelector('.notes').value}function paintProgress(){const reviewed=Object.values(state).filter(s=>s.choice).length;document.getElementById('progress').textContent=reviewed+' / '+DATA.rows.length+' masks reviewed'}function apply(){cards.forEach(c=>c.classList.toggle('hidden',filter==='unreviewed'&&c.dataset.review!=='unreviewed'))}cards.forEach(paint);document.querySelectorAll('[data-choice]').forEach(b=>b.onclick=()=>{const card=b.closest('.card'),id=card.dataset.id;state[id]={...(state[id]||{}),choice:b.dataset.choice};paint(card);save();apply()});document.querySelectorAll('.notes').forEach(n=>n.onchange=()=>{const card=n.closest('.card'),id=card.dataset.id;state[id]={...(state[id]||{}),notes:n.value};save()});document.querySelectorAll('[data-filter]').forEach(b=>b.onclick=()=>{filter=b.dataset.filter;apply()});function exportData(type){const rows=DATA.rows.map(r=>({...r,manual_mask_choice:(state[r.annotation_id]||{}).choice||'',manual_notes:(state[r.annotation_id]||{}).notes||''})),payload={schema_version:1,review_type:'manual_seed_mask_review',identity:DATA.identity,exported_at:new Date().toISOString(),rows};let blob,name;if(type==='json'){blob=new Blob([JSON.stringify(payload,null,2)],{type:'application/json'});name=DATA.identity.sample_id+'_manual_seed_mask_review.json'}else{const keys=Object.keys(rows[0]||{}),esc=v=>'"'+String(v??'').replaceAll('"','""')+'"';blob=new Blob([[keys.join(','),...rows.map(r=>keys.map(k=>esc(r[k])).join(','))].join('\n')],{type:'text/csv'});name=DATA.identity.sample_id+'_manual_seed_mask_review.csv'}const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download=name;a.click();setTimeout(()=>URL.revokeObjectURL(a.href),0)}document.getElementById('export-json').onclick=()=>exportData('json');document.getElementById('export-csv').onclick=()=>exportData('csv');paintProgress();apply(); +''' + + +def generate_manual_seed_review( + sample_dir: Path, + analysis_dir: Path, + *, + patch_radius_px: int = 220, +) -> ManualSeedReviewResult: + """Render each manual click with conservative and expanded mask overlays.""" + + root = Path(analysis_dir).resolve() + analysis_summary_path = root / "summary.json" + overlay_dir = None + expected_identity = None + if analysis_summary_path.is_file(): + analysis_summary = _read_json(analysis_summary_path) + expected_identity = analysis_summary.get("sample") + if isinstance(expected_identity, Mapping): + overlay_dir = overlay_dir_from_identity(expected_identity) + sample = _load_sample(sample_dir, overlay_dir=overlay_dir) + if expected_identity is not None and not _identity_contains_required( + expected_identity, + sample.review_identity, + ): + raise ValueError("recall analysis identity does not match current sample overlay") + table_path = root / "recall_failure_analysis.csv" + if not table_path.is_file(): + raise FileNotFoundError(f"recall analysis table does not exist: {table_path}") + table = pd.read_csv(table_path) + required = { + "annotation_id", + "x", + "y", + "manual_conservative_mask_path", + "manual_expanded_mask_path", + } + missing = required.difference(table.columns) + if missing: + raise ValueError(f"recall analysis table missing columns: {sorted(missing)}") + assets_dir = root / "review_assets" + assets_dir.mkdir(parents=True, exist_ok=True) + expected = {f"seed-{index:03d}.webp" for index in range(1, len(table) + 1)} + for existing in assets_dir.glob("*.webp"): + if existing.name not in expected: + existing.unlink() + with RecallReviewRuntime(sample.sample_dir, overlay_dir=overlay_dir) as runtime: + for index, row in enumerate(table.to_dict("records"), start=1): + destination = assets_dir / f"seed-{index:03d}.webp" + _render_card(runtime, row, destination, radius=patch_radius_px) + + rows = [_json_safe(row) for row in table.to_dict("records")] + identity: Dict[str, Any] = { + **( + dict(expected_identity) + if isinstance(expected_identity, Mapping) + else sample.review_identity + ), + "analysis_sha256": _file_sha256(table_path), + "analysis_table": str(table_path), + "patch_radius_px": patch_radius_px, + } + payload = {"identity": identity, "rows": rows} + cards = "".join(_card(row, index) for index, row in enumerate(table.to_dict("records"), start=1)) + failure_counts = table["failure_class"].value_counts().to_dict() if not table.empty else {} + stats = "".join( + f'{int(count)} {html.escape(str(name))}' + for name, count in failure_counts.items() + ) + page = f'''{html.escape(sample.sample_id)} manual-seed mask review
Aegle / human-seeded recall delta

{html.escape(sample.sample_id)} provisional mask review

Each row compares the same raw UCHL1 patch and manual center. Cyan is an existing accepted mask, green is the highest-percentile click-targeted conservative result, and yellow is a bounded expansion of the same component. Choose one mask only when it isolates the intended oocyte; choose Neither for a false click or two bad segmentations, and Duplicate when this click repeats another card.

{len(table)} manual clicks{stats}
{cards}
Selections remain browser-local until exported. No provisional mask is part of the production label image.
''' + page_path = root / "manual_seed_review.html" + _atomic_write_text(page_path, page) + return ManualSeedReviewResult( + page_path=page_path, + assets_dir=assets_dir, + card_count=len(table), + ) + + +__all__ = ["ManualSeedReviewResult", "generate_manual_seed_review"] diff --git a/aegle/oocyte/models.py b/aegle/oocyte/models.py new file mode 100644 index 0000000..f12f679 --- /dev/null +++ b/aegle/oocyte/models.py @@ -0,0 +1,148 @@ +"""Typed values shared by standalone oocyte modules.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any, Dict, Tuple + +import numpy as np + + +@dataclass(frozen=True) +class BoundingBox: + """Half-open image-space bounding box in X/Y order.""" + + x0: int + y0: int + x1: int + y1: int + + def __post_init__(self) -> None: + if self.x0 < 0 or self.y0 < 0: + raise ValueError("bounding box coordinates must be non-negative") + if self.x1 <= self.x0 or self.y1 <= self.y0: + raise ValueError("bounding box must have positive width and height") + + @property + def width(self) -> int: + return self.x1 - self.x0 + + @property + def height(self) -> int: + return self.y1 - self.y0 + + @property + def shape_yx(self) -> Tuple[int, int]: + return (self.height, self.width) + + def as_tuple(self) -> Tuple[int, int, int, int]: + return (self.x0, self.y0, self.x1, self.y1) + + +@dataclass(frozen=True) +class ExtractedPatch: + """A fixed-size image patch plus its clipped full-image geometry.""" + + image: np.ndarray + bbox: BoundingBox + image_shape_yx: Tuple[int, int] + padding_tblr: Tuple[int, int, int, int] + + def __post_init__(self) -> None: + if self.image.ndim != 2: + raise ValueError("extracted patch must be two-dimensional") + top, bottom, left, right = self.padding_tblr + if min(top, bottom, left, right) < 0: + raise ValueError("patch padding must be non-negative") + expected_shape = ( + self.bbox.height + top + bottom, + self.bbox.width + left + right, + ) + if self.image.shape != expected_shape: + raise ValueError( + f"patch shape {self.image.shape} does not match geometry {expected_shape}" + ) + + def crop_to_image_bounds(self, array: np.ndarray) -> np.ndarray: + """Remove edge padding from a patch-aligned array.""" + + if array.shape[:2] != self.image.shape: + raise ValueError("array must share the extracted patch Y/X shape") + top, bottom, left, right = self.padding_tblr + y1 = array.shape[0] - bottom if bottom else array.shape[0] + x1 = array.shape[1] - right if right else array.shape[1] + return array[top:y1, left:x1] + + +@dataclass(frozen=True) +class SegmentationMetrics: + threshold_method: str + base_threshold: float + annulus_floor: float + threshold: float + selection_mode: str + area_px: int + equivalent_diameter_um: float + major_axis_um: float + minor_axis_um: float + eccentricity: float + solidity: float + circularity: float + centroid_y_px: float + centroid_x_px: float + centroid_offset_px: float + mean_intensity: float + max_intensity: float + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class LocalSegmentationResult: + mask: np.ndarray + metrics: SegmentationMetrics + + def __post_init__(self) -> None: + if self.mask.ndim != 2 or self.mask.dtype != np.bool_: + raise ValueError("segmentation mask must be a two-dimensional boolean array") + if not self.mask.any(): + raise ValueError("segmentation mask must contain at least one foreground pixel") + if int(self.mask.sum()) != self.metrics.area_px: + raise ValueError("segmentation mask area does not match metrics") + + +@dataclass(frozen=True) +class ScoredCandidateMask: + """Image-bounded mask and metrics retained for one refined candidate.""" + + mask: np.ndarray + bbox: BoundingBox + image_shape_yx: Tuple[int, int] + metrics: SegmentationMetrics + + def __post_init__(self) -> None: + if self.mask.ndim != 2 or self.mask.dtype != np.bool_: + raise ValueError("candidate mask must be a two-dimensional boolean array") + if self.mask.shape != self.bbox.shape_yx: + raise ValueError("candidate mask shape must match its image-space bounding box") + image_h, image_w = self.image_shape_yx + if self.bbox.x1 > image_w or self.bbox.y1 > image_h: + raise ValueError("candidate mask bounding box exceeds the source image") + + +@dataclass(frozen=True) +class PersistedMask: + mask: np.ndarray + bbox: BoundingBox + image_shape_yx: Tuple[int, int] + metadata: Dict[str, Any] + + def __post_init__(self) -> None: + if self.mask.ndim != 2 or self.mask.dtype != np.bool_: + raise ValueError("persisted mask must be a two-dimensional boolean array") + if self.mask.shape != self.bbox.shape_yx: + raise ValueError("persisted mask shape must match its image-space bounding box") + image_h, image_w = self.image_shape_yx + if self.bbox.x1 > image_w or self.bbox.y1 > image_h: + raise ValueError("persisted mask bounding box exceeds the source image") diff --git a/aegle/oocyte/precision_boundary_finalize.py b/aegle/oocyte/precision_boundary_finalize.py new file mode 100644 index 0000000..ed59625 --- /dev/null +++ b/aegle/oocyte/precision_boundary_finalize.py @@ -0,0 +1,894 @@ +"""Finalize reviewed Precision decisions into an immutable intermediate label set.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping, Sequence, Tuple + +import numpy as np +import pandas as pd + +from .export import LabelExportResult, export_whole_slide_labels +from .io import load_candidate_mask +from .manual_seed_finalize import ( + _atomic_write_csv, + _overlap_metrics, + _tight_mask, + _write_reviewed_mask, +) +from .models import PersistedMask +from .precision_boundary_review import ( + _note_tokens, + _review_key, + _validate_precision_review, +) +from .recall_review import ( + _atomic_write_text, + _file_sha256, + _json_safe, + _load_sample, + _mask_path, + _read_json, +) + + +PRECISION_RESOLVED_PROFILE_NAME = "precision_resolved_v1" +PRECISION_BOUNDARY_CHOICES = { + "keep_current", + "use_conservative", + "use_expanded", + "needs_manual", + "exclude", + "unsure", +} +_ACCEPTED_BOUNDARY_CHOICES = { + "keep_current", + "use_conservative", + "use_expanded", +} +_PROPOSAL_VARIANTS = { + "use_conservative": "conservative", + "use_expanded": "expanded", +} +_PROPOSAL_METRICS = ( + "area_px", + "equivalent_diameter_um", + "circularity", + "solidity", + "centroid_offset_px", +) + + +@dataclass(frozen=True) +class PrecisionBoundaryFinalizeResult: + out_dir: Path + decisions_path: Path + candidates_path: Path + manual_queue_path: Path + overlap_audit_path: Path + labels: LabelExportResult + manifest_path: Path + resolved_count: int + unresolved_manual_count: int + excluded_count: int + + +def _clean_text(value: Any) -> str: + if value is None or (not isinstance(value, str) and pd.isna(value)): + return "" + return str(value).strip() + + +def _bool_value(value: Any, *, field: str) -> bool: + if value is None or (not isinstance(value, str) and pd.isna(value)): + return False + if isinstance(value, (bool, np.bool_)): + return bool(value) + normalized = str(value).strip().casefold() + if normalized in {"true", "1", "yes"}: + return True + if normalized in {"false", "0", "no", ""}: + return False + raise ValueError(f"invalid boolean value for {field}: {value!r}") + + +def _safe_mask_name(review_key: str) -> str: + value = re.sub(r"[^A-Za-z0-9_.-]+", "__", review_key).strip("._-") + if not value: + raise ValueError(f"review key cannot form a mask filename: {review_key!r}") + return f"{value}.npz" + + +def _atomic_copy(source: Path, destination: Path) -> Path: + source_path = Path(source) + target = Path(destination) + target.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with source_path.open("rb") as source_handle, tempfile.NamedTemporaryFile( + mode="wb", + prefix=f".{target.name}.", + suffix=".tmp", + dir=target.parent, + delete=False, + ) as target_handle: + temporary_path = Path(target_handle.name) + shutil.copyfileobj(source_handle, target_handle) + target_handle.flush() + os.fsync(target_handle.fileno()) + temporary_path.replace(target) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + return target + + +def _validate_boundary_identity( + sample_identity: Mapping[str, Any], + payload: Mapping[str, Any], +) -> Mapping[str, Any]: + if payload.get("schema_version") != 1: + raise ValueError("Precision boundary review schema_version must be 1") + if payload.get("review_type") != "oocyte_precision_boundary_review": + raise ValueError( + "review_type must be 'oocyte_precision_boundary_review'" + ) + identity = payload.get("identity") + if not isinstance(identity, Mapping): + raise ValueError("Precision boundary review identity is missing") + for field, expected in sample_identity.items(): + if identity.get(field) != expected: + raise ValueError(f"Precision boundary identity mismatch for {field}") + for field in ( + "precision_review_json", + "precision_review_json_sha256", + "boundary_candidate_table", + "boundary_candidate_table_sha256", + ): + if not identity.get(field): + raise ValueError(f"Precision boundary identity is missing {field}") + return identity + + +def _validate_boundary_rows( + payload: Mapping[str, Any], + candidates: pd.DataFrame, +) -> Tuple[Sequence[Mapping[str, Any]], Dict[str, Dict[str, Any]]]: + required = { + "boundary_index", + "review_key", + "display_id", + "detector_component_id", + "detection_pass", + "x", + "y", + "current_mask_path", + "conservative_available", + "expanded_available", + } + missing = required.difference(candidates.columns) + if missing: + raise ValueError( + f"Precision boundary candidate table is missing columns: {sorted(missing)}" + ) + if candidates["review_key"].astype(str).duplicated().any(): + raise ValueError("Precision boundary candidate keys are not unique") + rows = payload.get("rows") + if not isinstance(rows, list): + raise ValueError("Precision boundary review rows must be a list") + by_key = { + str(row["review_key"]): row for row in candidates.to_dict("records") + } + review_by_key: Dict[str, Dict[str, Any]] = {} + for row_number, raw_row in enumerate(rows, start=1): + if not isinstance(raw_row, Mapping): + raise ValueError( + f"Precision boundary review row {row_number} must be an object" + ) + key = str(raw_row.get("review_key", "")) + if key in review_by_key: + raise ValueError(f"Precision boundary review has duplicate key {key}") + candidate = by_key.get(key) + if candidate is None: + raise ValueError(f"Precision boundary review contains unknown key {key}") + for field in ( + "boundary_index", + "detector_component_id", + "detection_pass", + ): + if str(raw_row.get(field, "")) != str(candidate[field]): + raise ValueError( + f"Precision boundary row metadata changed for {key}: {field}" + ) + for field in ("x", "y"): + if not np.isclose( + float(raw_row.get(field, np.nan)), + float(candidate[field]), + atol=0.01, + ): + raise ValueError( + f"Precision boundary coordinates changed for {key}: {field}" + ) + choice = str(raw_row.get("boundary_review_choice", "")).strip() + if choice not in PRECISION_BOUNDARY_CHOICES: + raise ValueError( + f"invalid or missing Precision boundary choice for {key}" + ) + if choice == "unsure": + raise ValueError(f"Precision boundary review remains unsure for {key}") + review_by_key[key] = { + "choice": choice, + "notes": _clean_text(raw_row.get("boundary_review_notes")), + "candidate": candidate, + } + if set(review_by_key) != set(by_key): + raise ValueError( + "Precision boundary review rows do not match its candidate table" + ) + return rows, review_by_key + + +def _validate_current_mask( + path: Path, + *, + sample_id: str, + candidate_id: str, + image_shape_yx: Tuple[int, int], +) -> PersistedMask: + if not path.is_file(): + raise FileNotFoundError(f"current candidate mask is missing: {path}") + persisted = load_candidate_mask(path) + if persisted.image_shape_yx != image_shape_yx: + raise ValueError(f"current candidate mask image shape mismatch: {path}") + if str(persisted.metadata.get("sample_id", "")) != sample_id: + raise ValueError(f"current candidate mask sample mismatch: {path}") + if str(persisted.metadata.get("candidate_id", "")) != candidate_id: + raise ValueError(f"current candidate mask ID mismatch: {path}") + if not persisted.mask.any(): + raise ValueError(f"current candidate mask is empty: {path}") + return persisted + + +def _validate_proposal_mask( + candidate: Mapping[str, Any], + *, + key: str, + variant: str, + boundary_root: Path, + image_shape_yx: Tuple[int, int], +) -> Tuple[Path, PersistedMask]: + available_field = f"{variant}_available" + if not _bool_value(candidate.get(available_field), field=available_field): + raise ValueError(f"selected {variant} proposal is unavailable for {key}") + path_value = _clean_text(candidate.get(f"{variant}_mask_path")) + if not path_value: + raise ValueError(f"selected {variant} proposal has no path for {key}") + path = Path(path_value).resolve() + if not path.is_relative_to(boundary_root): + raise ValueError(f"selected proposal is outside its review pack: {path}") + if not path.is_file(): + raise FileNotFoundError(f"selected proposal is missing: {path}") + persisted = load_candidate_mask(path) + if persisted.image_shape_yx != image_shape_yx: + raise ValueError(f"selected proposal image shape mismatch: {path}") + if str(persisted.metadata.get("annotation_id", "")) != key: + raise ValueError(f"selected proposal annotation mismatch: {path}") + metrics = persisted.metadata.get("metrics") + if not isinstance(metrics, Mapping): + raise ValueError(f"selected proposal has no metrics: {path}") + if int(persisted.mask.sum()) != int(metrics.get("area_px", -1)): + raise ValueError(f"selected proposal mask area does not match metadata: {path}") + for name in _PROPOSAL_METRICS: + table_value = float(candidate.get(f"{variant}_{name}", np.nan)) + metadata_value = float(metrics.get(name, np.nan)) + if not np.isfinite(table_value) or not np.isfinite(metadata_value): + raise ValueError(f"selected proposal metric {name} is missing: {path}") + if not np.isclose(table_value, metadata_value, rtol=1e-7, atol=1e-7): + raise ValueError( + f"selected proposal metric {name} does not match table: {path}" + ) + return path, persisted + + +def _resolved_overlap_audit( + masks: Sequence[Tuple[str, PersistedMask]], +) -> pd.DataFrame: + rows = [] + for left_index, (left_key, left) in enumerate(masks): + for right_key, right in masks[left_index + 1 :]: + pixels, left_fraction, right_fraction = _overlap_metrics(left, right) + if not pixels: + continue + rows.append( + { + "left_review_key": left_key, + "right_review_key": right_key, + "overlap_pixel_count": pixels, + "left_overlap_fraction": left_fraction, + "right_overlap_fraction": right_fraction, + "smaller_mask_overlap_fraction": max( + left_fraction, right_fraction + ), + } + ) + return pd.DataFrame( + rows, + columns=[ + "left_review_key", + "right_review_key", + "overlap_pixel_count", + "left_overlap_fraction", + "right_overlap_fraction", + "smaller_mask_overlap_fraction", + ], + ) + + +def _candidate_row( + source: Mapping[str, Any], + *, + review_key: str, + mask: PersistedMask, + reviewed_path: Path, + reviewed_sha256: str, + destination: Path, + final_source: str, + precision_status: str, + precision_notes: str, + boundary_choice: str, + boundary_notes: str, + source_mask_path: Path, + source_mask_sha256: str, + precision_review_sha256: str, + boundary_review_sha256: str, +) -> Dict[str, Any]: + row = dict(source) + source_metrics = mask.metadata.get("metrics", {}) + metrics = source_metrics if isinstance(source_metrics, Mapping) else {} + ys, xs = np.nonzero(mask.mask) + centroid_x = mask.bbox.x0 + float(xs.mean()) + centroid_y = mask.bbox.y0 + float(ys.mean()) + source_acceptance_mode = _clean_text(row.get("acceptance_mode")) + source_segmentation_pass = _clean_text(row.get("segmentation_pass")) + row.update( + { + "review_key": review_key, + "resolved_oocyte_id": review_key, + "source_center_x": source.get("center_x"), + "source_center_y": source.get("center_y"), + "source_acceptance_mode": source_acceptance_mode, + "source_segmentation_pass": source_segmentation_pass, + "accepted": True, + "center_x": int(round(centroid_x)), + "center_y": int(round(centroid_y)), + "component_centroid_x": centroid_x, + "component_centroid_y": centroid_y, + "bbox_x0": mask.bbox.x0, + "bbox_y0": mask.bbox.y0, + "bbox_x1": mask.bbox.x1, + "bbox_y1": mask.bbox.y1, + "local_area_px": int(mask.mask.sum()), + "local_equivalent_diameter_um": metrics.get( + "equivalent_diameter_um" + ), + "local_major_axis_um": metrics.get("major_axis_um"), + "local_minor_axis_um": metrics.get("minor_axis_um"), + "local_eccentricity": metrics.get("eccentricity"), + "local_solidity": metrics.get("solidity"), + "local_circularity": metrics.get("circularity"), + "local_centroid_offset_px": metrics.get("centroid_offset_px"), + "local_mean_intensity": metrics.get("mean_intensity"), + "local_max_intensity": metrics.get("max_intensity"), + "threshold_method": metrics.get("threshold_method"), + "threshold": metrics.get("threshold"), + "selection_mode": metrics.get("selection_mode"), + "acceptance_mode": final_source, + "segmentation_pass": final_source, + "quality_class": "precision_reviewed_boundary", + "precision_review_status": precision_status, + "precision_review_notes": precision_notes, + "precision_boundary_choice": boundary_choice, + "precision_boundary_notes": boundary_notes, + "precision_resolution_source": final_source, + "mask_path": str(reviewed_path.relative_to(destination)), + "mask_source_dir": str(destination), + "source_mask_path": str(source_mask_path), + "source_mask_sha256": source_mask_sha256, + "reviewed_mask_sha256": reviewed_sha256, + "precision_review_json_sha256": precision_review_sha256, + "precision_boundary_review_json_sha256": boundary_review_sha256, + "duplicate_suppressed": False, + } + ) + return row + + +def finalize_precision_boundary_review( + sample_dir: Path, + precision_review_json: Path, + boundary_review_json: Path, + out_dir: Path, + *, + tile_shape_yx: Tuple[int, int] = (512, 512), + max_smaller_overlap_fraction: float = 0.25, +) -> PrecisionBoundaryFinalizeResult: + """Apply completed Precision and boundary decisions without claiming Recall.""" + + sample = _load_sample(sample_dir) + precision_path = Path(precision_review_json).resolve() + boundary_review_path = Path(boundary_review_json).resolve() + precision_payload = _read_json(precision_path) + precision_rows = _validate_precision_review( + sample.review_identity, + sample.candidates, + precision_payload, + ) + for row in precision_rows: + if str(row.get("manual_status", "")) == "unsure": + raise ValueError( + f"Precision review remains unsure for {row.get('review_key', '')}" + ) + precision_sha256 = _file_sha256(precision_path) + + boundary_payload = _read_json(boundary_review_path) + boundary_identity = _validate_boundary_identity( + sample.review_identity, + boundary_payload, + ) + identity_precision_path = Path( + str(boundary_identity["precision_review_json"]) + ).resolve() + if identity_precision_path != precision_path: + raise ValueError("Precision review path does not match boundary identity") + if precision_sha256 != str(boundary_identity["precision_review_json_sha256"]): + raise ValueError("Precision review SHA-256 does not match boundary identity") + + boundary_table_path = Path( + str(boundary_identity["boundary_candidate_table"]) + ).resolve() + if not boundary_table_path.is_file(): + raise FileNotFoundError( + f"Precision boundary candidate table is missing: {boundary_table_path}" + ) + boundary_table_sha256 = _file_sha256(boundary_table_path) + if boundary_table_sha256 != str( + boundary_identity["boundary_candidate_table_sha256"] + ): + raise ValueError("Precision boundary candidate table SHA-256 mismatch") + boundary_root = boundary_table_path.parent.resolve() + boundary_candidates = pd.read_csv(boundary_table_path) + _, boundary_by_key = _validate_boundary_rows( + boundary_payload, + boundary_candidates, + ) + boundary_review_sha256 = _file_sha256(boundary_review_path) + + candidate_records = sample.candidates.to_dict("records") + candidate_by_key = {_review_key(row): row for row in candidate_records} + if len(candidate_by_key) != len(candidate_records): + raise ValueError("sample candidate review keys are not unique") + component_ids = [str(row["detector_component_id"]) for row in candidate_records] + if len(component_ids) != len(set(component_ids)): + raise ValueError("sample detector_component_id values are not unique") + expected_boundary_keys = { + str(row["review_key"]) + for row in precision_rows + if str(row.get("manual_status")) == "reject" + and "true_oocyte" in _note_tokens(row.get("manual_notes")) + } + if set(boundary_by_key) != expected_boundary_keys: + raise ValueError( + "Precision boundary candidate set does not match true-oocyte rejects" + ) + + precision_by_key = { + str(row["review_key"]): row for row in precision_rows + } + selections = [] + decisions = [] + manual_queue = [] + choice_counts: Dict[str, int] = {} + false_positive_count = 0 + for review_index, precision_row in enumerate(precision_rows, start=1): + key = str(precision_row["review_key"]) + source = candidate_by_key[key] + status = str(precision_row["manual_status"]) + precision_notes = _clean_text(precision_row.get("manual_notes")) + boundary = boundary_by_key.get(key) + boundary_choice = "" if boundary is None else str(boundary["choice"]) + boundary_notes = "" if boundary is None else str(boundary["notes"]) + if boundary_choice: + choice_counts[boundary_choice] = choice_counts.get(boundary_choice, 0) + 1 + + accepted = False + resolution_state = "excluded" + final_source = "precision_reject" + selected_variant = "" + selected_path: Path | None = None + selected_mask: PersistedMask | None = None + if status == "accept": + accepted = True + resolution_state = "resolved" + final_source = "precision_accept_current" + selected_path = _mask_path(sample.sample_dir, source).resolve() + selected_mask = _validate_current_mask( + selected_path, + sample_id=sample.sample_id, + candidate_id=str(source["detector_component_id"]), + image_shape_yx=sample.image_shape_yx, + ) + elif boundary_choice in _ACCEPTED_BOUNDARY_CHOICES: + accepted = True + resolution_state = "resolved" + if boundary_choice == "keep_current": + final_source = "precision_boundary_keep_current" + selected_path = _mask_path(sample.sample_dir, source).resolve() + table_current_path = Path( + str(boundary["candidate"]["current_mask_path"]) + ).resolve() + if table_current_path != selected_path: + raise ValueError( + f"current boundary mask path changed for {key}" + ) + selected_mask = _validate_current_mask( + selected_path, + sample_id=sample.sample_id, + candidate_id=str(source["detector_component_id"]), + image_shape_yx=sample.image_shape_yx, + ) + else: + selected_variant = _PROPOSAL_VARIANTS[boundary_choice] + final_source = f"precision_boundary_{selected_variant}" + selected_path, selected_mask = _validate_proposal_mask( + boundary["candidate"], + key=key, + variant=selected_variant, + boundary_root=boundary_root, + image_shape_yx=sample.image_shape_yx, + ) + elif boundary_choice == "needs_manual": + resolution_state = "manual_boundary_required" + final_source = "precision_boundary_needs_manual" + manual_queue.append( + { + "review_index": review_index, + "boundary_index": int(boundary["candidate"]["boundary_index"]), + "review_key": key, + "display_id": str(source.get("display_id", "")), + "detector_component_id": str(source["detector_component_id"]), + "detection_pass": str(source["detection_pass"]), + "center_x": float(source["center_x"]), + "center_y": float(source["center_y"]), + "precision_notes": precision_notes, + "boundary_review_notes": boundary_notes, + "current_mask_path": str( + Path(str(boundary["candidate"]["current_mask_path"])).resolve() + ), + "required_action": "draw_and_review_manual_boundary", + } + ) + elif boundary_choice == "exclude": + final_source = "precision_boundary_excluded" + elif status == "reject": + false_positive_count += 1 + else: + raise ValueError(f"unsupported Precision decision for {key}") + + source_sha256 = "" + if accepted: + if selected_path is None or selected_mask is None: + raise AssertionError(f"accepted selection has no mask: {key}") + source_sha256 = _file_sha256(selected_path) + selections.append( + { + "review_index": review_index, + "review_key": key, + "source": source, + "precision_status": status, + "precision_notes": precision_notes, + "boundary_choice": boundary_choice, + "boundary_notes": boundary_notes, + "selected_variant": selected_variant, + "final_source": final_source, + "source_path": selected_path, + "source_sha256": source_sha256, + "mask": _tight_mask(selected_mask), + } + ) + decisions.append( + { + "review_index": review_index, + "review_key": key, + "display_id": str(source.get("display_id", "")), + "detector_component_id": str(source["detector_component_id"]), + "detection_pass": str(source["detection_pass"]), + "center_x": float(source["center_x"]), + "center_y": float(source["center_y"]), + "precision_status": status, + "precision_notes": precision_notes, + "boundary_review_choice": boundary_choice, + "boundary_review_notes": boundary_notes, + "final_accepted": accepted, + "resolution_state": resolution_state, + "final_source": final_source, + "selected_variant": selected_variant, + "source_mask_path": "" if selected_path is None else str(selected_path), + "source_mask_sha256": source_sha256, + "reviewed_mask_path": "", + "reviewed_mask_sha256": "", + } + ) + + if set(precision_by_key) != set(candidate_by_key): + raise AssertionError("validated Precision review unexpectedly changed candidate set") + audit = _resolved_overlap_audit( + [(str(item["review_key"]), item["mask"]) for item in selections] + ) + blocking = audit[ + audit["smaller_mask_overlap_fraction"] >= max_smaller_overlap_fraction + ] + if not blocking.empty: + pairs = ", ".join( + f"{row.left_review_key}/{row.right_review_key}=" + f"{row.smaller_mask_overlap_fraction:.3f}" + for row in blocking.itertuples() + ) + raise ValueError(f"Precision-resolved masks have blocking overlap: {pairs}") + + destination = Path(out_dir).resolve() + manifest_path = destination / "precision_resolved_manifest.json" + if manifest_path.exists(): + raise FileExistsError( + f"immutable Precision delivery already exists: {manifest_path}" + ) + destination.mkdir(parents=True, exist_ok=True) + masks_dir = destination / "reviewed_masks" + masks_dir.mkdir(parents=True, exist_ok=True) + expected_mask_names = { + _safe_mask_name(str(item["review_key"])) for item in selections + } + for existing in masks_dir.glob("*.npz"): + if existing.name not in expected_mask_names: + existing.unlink() + + input_dir = destination / "review_inputs" + input_copies = { + "automatic_candidates": ( + sample.sample_dir / "html_candidates.csv", + input_dir / "automatic_candidates.csv", + ), + "precision_review": ( + precision_path, + input_dir / "precision_review.json", + ), + "boundary_candidates": ( + boundary_table_path, + input_dir / "precision_boundary_candidates.csv", + ), + "boundary_review": ( + boundary_review_path, + input_dir / "precision_boundary_review.json", + ), + } + for source_path, copy_path in input_copies.values(): + _atomic_copy(source_path, copy_path) + if _file_sha256(source_path) != _file_sha256(copy_path): + raise ValueError(f"review-input copy SHA-256 mismatch: {copy_path}") + + decision_by_key = {str(row["review_key"]): row for row in decisions} + candidate_rows = [] + for item in selections: + key = str(item["review_key"]) + reviewed_path = masks_dir / _safe_mask_name(key) + metadata = { + **dict(item["mask"].metadata), + "schema_version": 1, + "sample_id": sample.sample_id, + "candidate_id": str(item["source"]["detector_component_id"]), + "review_key": key, + "resolved_oocyte_id": key, + "profile_name": PRECISION_RESOLVED_PROFILE_NAME, + "base_profile_name": sample.profile_name, + "base_profile_fingerprint": sample.profile_fingerprint, + "implementation_version": sample.implementation_version, + "precision_resolved": True, + "provisional_only": False, + "precision_resolution_source": item["final_source"], + "precision_review_status": item["precision_status"], + "precision_review_notes": item["precision_notes"], + "precision_boundary_choice": item["boundary_choice"], + "precision_boundary_notes": item["boundary_notes"], + "precision_review_json_sha256": precision_sha256, + "precision_boundary_review_json_sha256": boundary_review_sha256, + "boundary_candidate_table_sha256": boundary_table_sha256, + "source_mask_path": str(item["source_path"]), + "source_mask_sha256": item["source_sha256"], + } + _write_reviewed_mask(reviewed_path, mask=item["mask"], metadata=metadata) + reviewed_sha256 = _file_sha256(reviewed_path) + decision = decision_by_key[key] + decision["reviewed_mask_path"] = str(reviewed_path) + decision["reviewed_mask_sha256"] = reviewed_sha256 + candidate_rows.append( + _candidate_row( + item["source"], + review_key=key, + mask=item["mask"], + reviewed_path=reviewed_path, + reviewed_sha256=reviewed_sha256, + destination=destination, + final_source=str(item["final_source"]), + precision_status=str(item["precision_status"]), + precision_notes=str(item["precision_notes"]), + boundary_choice=str(item["boundary_choice"]), + boundary_notes=str(item["boundary_notes"]), + source_mask_path=Path(item["source_path"]), + source_mask_sha256=str(item["source_sha256"]), + precision_review_sha256=precision_sha256, + boundary_review_sha256=boundary_review_sha256, + ) + ) + + decisions_path = destination / "precision_review_decisions.csv" + candidates_path = destination / "precision_resolved_candidates.csv" + manual_queue_path = destination / "manual_boundary_queue.csv" + overlap_audit_path = destination / "mask_overlap_audit.csv" + decisions_table = pd.DataFrame(decisions) + _atomic_write_csv(decisions_table, decisions_path) + if candidate_rows: + resolved_candidates = pd.DataFrame(candidate_rows) + else: + resolved_candidates = sample.candidates.iloc[:0].copy() + empty_columns = { + "accepted": "bool", + "detector_score": "float64", + "acceptance_mode": "object", + "review_key": "object", + "resolved_oocyte_id": "object", + "precision_review_status": "object", + "precision_review_notes": "object", + "precision_boundary_choice": "object", + "precision_boundary_notes": "object", + "precision_resolution_source": "object", + "reviewed_mask_sha256": "object", + } + for column, dtype in empty_columns.items(): + if column not in resolved_candidates: + resolved_candidates[column] = pd.Series(dtype=dtype) + _atomic_write_csv(resolved_candidates, candidates_path) + manual_columns = [ + "review_index", + "boundary_index", + "review_key", + "display_id", + "detector_component_id", + "detection_pass", + "center_x", + "center_y", + "precision_notes", + "boundary_review_notes", + "current_mask_path", + "required_action", + ] + _atomic_write_csv(pd.DataFrame(manual_queue, columns=manual_columns), manual_queue_path) + _atomic_write_csv(audit, overlap_audit_path) + + labels = export_whole_slide_labels( + resolved_candidates, + sample_dir=destination, + image_shape_yx=sample.image_shape_yx, + image_path=destination / "oocyte_labels_precision_resolved_v1.ome.tiff", + mapping_path=destination / "oocyte_labels_precision_resolved_v1_mapping.csv", + tile_shape_yx=tile_shape_yx, + ) + if labels.overlap_pixel_count: + raise AssertionError("preflight and label-export overlap counts disagree") + expected_pixels = sum(int(item["mask"].mask.sum()) for item in selections) + if labels.assigned_pixel_count != expected_pixels: + raise AssertionError("label-export pixel count does not match resolved masks") + + excluded_count = int((~decisions_table["final_accepted"]).sum()) - len(manual_queue) + artifact_paths = [ + decisions_path, + candidates_path, + manual_queue_path, + overlap_audit_path, + labels.image_path, + labels.mapping_path, + *(copy_path for _, copy_path in input_copies.values()), + *(Path(row["reviewed_mask_path"]) for row in decisions if row["reviewed_mask_path"]), + ] + source_counts = ( + resolved_candidates["precision_resolution_source"] + .value_counts() + .sort_index() + .to_dict() + if len(resolved_candidates) + else {} + ) + manifest = { + "schema_version": 1, + "delivery_name": PRECISION_RESOLVED_PROFILE_NAME, + "delivery_status": "intermediate_precision_only", + "release_ready": False, + "precision_complete": True, + "manual_boundary_complete": len(manual_queue) == 0, + "recall_complete": False, + "sample": sample.review_identity, + "precision_review_json": str(precision_path), + "precision_review_json_sha256": precision_sha256, + "precision_review_exported_at": precision_payload.get("exported_at"), + "precision_boundary_review_json": str(boundary_review_path), + "precision_boundary_review_json_sha256": boundary_review_sha256, + "precision_boundary_review_exported_at": boundary_payload.get("exported_at"), + "boundary_candidate_table": str(boundary_table_path), + "boundary_candidate_table_sha256": boundary_table_sha256, + "automatic_candidate_count": len(sample.candidates), + "resolved_label_count": labels.label_count, + "unresolved_manual_count": len(manual_queue), + "excluded_count": excluded_count, + "false_positive_count": false_positive_count, + "boundary_choice_counts": choice_counts, + "resolution_source_counts": source_counts, + "unresolved_review_keys": [row["review_key"] for row in manual_queue], + "overlap_audit_row_count": len(audit), + "max_smaller_overlap_fraction": ( + 0.0 + if audit.empty + else float(audit["smaller_mask_overlap_fraction"].max()) + ), + "overlap_blocking_threshold": max_smaller_overlap_fraction, + "label_export": { + "label_count": labels.label_count, + "assigned_pixel_count": labels.assigned_pixel_count, + "overlap_pixel_count": labels.overlap_pixel_count, + }, + "production_outputs_modified": False, + "review_pack_outputs_modified": False, + "input_copies": { + name: { + "source_path": str(source_path), + "copied_path": str(copy_path), + "sha256": _file_sha256(copy_path), + } + for name, (source_path, copy_path) in input_copies.items() + }, + "artifacts": { + str(path.relative_to(destination)): { + "path": str(path), + "sha256": _file_sha256(path), + "size_bytes": path.stat().st_size, + } + for path in artifact_paths + }, + } + _atomic_write_text( + manifest_path, + json.dumps(_json_safe(manifest), indent=2, sort_keys=True, allow_nan=False), + ) + return PrecisionBoundaryFinalizeResult( + out_dir=destination, + decisions_path=decisions_path, + candidates_path=candidates_path, + manual_queue_path=manual_queue_path, + overlap_audit_path=overlap_audit_path, + labels=labels, + manifest_path=manifest_path, + resolved_count=labels.label_count, + unresolved_manual_count=len(manual_queue), + excluded_count=excluded_count, + ) + + +__all__ = [ + "PRECISION_BOUNDARY_CHOICES", + "PRECISION_RESOLVED_PROFILE_NAME", + "PrecisionBoundaryFinalizeResult", + "finalize_precision_boundary_review", +] diff --git a/aegle/oocyte/precision_boundary_review.py b/aegle/oocyte/precision_boundary_review.py new file mode 100644 index 0000000..9911ae5 --- /dev/null +++ b/aegle/oocyte/precision_boundary_review.py @@ -0,0 +1,563 @@ +"""Review-gated boundary replacements for accepted raw-UCHL1 candidates.""" + +from __future__ import annotations + +import html +import io +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping, Sequence, Tuple + +import numpy as np +import pandas as pd +from PIL import Image + +from .io import load_candidate_mask +from .manual_seed_finalize import _atomic_write_csv +from .manual_seed_review import _crosshair, _draw_mask, _load_provisional, _panel_label +from .models import LocalSegmentationResult, PersistedMask +from .recall_review import ( + RecallReviewRuntime, + _atomic_save_image, + _atomic_write_text, + _file_sha256, + _json_safe, + _load_sample, + _mask_path, + _read_json, + _save_provisional_mask, +) + + +PRECISION_BOUNDARY_REVIEW_SCHEMA_VERSION = 1 +BOUNDARY_CANDIDATE_COLUMNS = ( + "boundary_index", + "review_key", + "display_id", + "detector_component_id", + "detection_pass", + "x", + "y", + "current_mask_path", + "conservative_available", + "expanded_available", +) +BOUNDARY_RECOVERY_PARAMETERS = { + "annulus_percentiles": [95.0, 90.0, 85.0, 80.0, 75.0, 70.0, 65.0, 60.0], + "min_area_growth_ratio": 1.10, + "standard_max_area_growth_ratio": 4.0, + "shape_max_area_growth_ratio": 6.0, + "min_current_overlap": 0.95, + "max_equivalent_diameter_um": 100.0, + "max_centroid_offset_px": 50.0, + "shape_min_circularity": 0.80, + "shape_min_solidity": 0.90, + "shape_max_centroid_offset_px": 25.0, +} + + +@dataclass(frozen=True) +class PrecisionBoundaryReviewResult: + page_path: Path + candidates_path: Path + assets_dir: Path + masks_dir: Path + card_count: int + proposal_count: int + manual_only_count: int + automatic_review_path: Path | None + + +def _review_key(row: Mapping[str, Any]) -> str: + return ( + f"{str(row.get('detection_pass', 'baseline_v6'))}:" + f"{str(row['detector_component_id'])}" + ) + + +def _note_tokens(value: Any) -> set[str]: + return { + token.strip().casefold() + for token in str(value or "").split(";") + if token.strip() + } + + +def _validate_precision_review( + sample_identity: Mapping[str, Any], + candidates: pd.DataFrame, + payload: Mapping[str, Any], +) -> Sequence[Mapping[str, Any]]: + if payload.get("schema_version") != PRECISION_BOUNDARY_REVIEW_SCHEMA_VERSION: + raise ValueError("precision review schema_version must be 1") + if payload.get("review_type") != "oocyte_precision_review": + raise ValueError("review_type must be 'oocyte_precision_review'") + identity = payload.get("identity") + if not isinstance(identity, Mapping): + raise ValueError("precision review identity is missing") + for field, expected in sample_identity.items(): + if identity.get(field) != expected: + raise ValueError(f"precision review identity mismatch for {field}") + rows = payload.get("rows") + if not isinstance(rows, list): + raise ValueError("precision review rows must be a list") + + current_by_key = { + _review_key(row): row for row in candidates.to_dict("records") + } + if len(current_by_key) != len(candidates): + raise ValueError("current candidate review keys are not unique") + seen = set() + allowed_statuses = {"accept", "reject", "unsure"} + for index, row in enumerate(rows, start=1): + if not isinstance(row, Mapping): + raise ValueError(f"precision review row {index} must be an object") + key = str(row.get("review_key", "")) + if key in seen: + raise ValueError(f"precision review contains duplicate key {key}") + seen.add(key) + current = current_by_key.get(key) + if current is None: + raise ValueError(f"precision review contains unknown candidate {key}") + for field in ("detector_component_id", "detection_pass"): + if str(row.get(field, "")) != str(current[field]): + raise ValueError(f"precision review metadata mismatch for {key}: {field}") + for field in ("center_x", "center_y"): + if float(row.get(field, np.nan)) != float(current[field]): + raise ValueError(f"precision review metadata mismatch for {key}: {field}") + if str(row.get("manual_status", "")) not in allowed_statuses: + raise ValueError(f"precision review status is unresolved for {key}") + if seen != set(current_by_key): + raise ValueError("precision review does not cover the current candidate table") + return rows + + +def _proposal_is_safe( + result: LocalSegmentationResult, + current_local: np.ndarray, +) -> Tuple[bool, Dict[str, float], Sequence[str]]: + current_area = max(int(current_local.sum()), 1) + proposal_area = int(result.metrics.area_px) + intersection = int(np.logical_and(current_local, result.mask).sum()) + union = int(np.logical_or(current_local, result.mask).sum()) + area_ratio = proposal_area / current_area + current_overlap = intersection / current_area + iou = intersection / max(union, 1) + shape_extension = bool( + area_ratio <= BOUNDARY_RECOVERY_PARAMETERS["shape_max_area_growth_ratio"] + and result.metrics.circularity + >= BOUNDARY_RECOVERY_PARAMETERS["shape_min_circularity"] + and result.metrics.solidity + >= BOUNDARY_RECOVERY_PARAMETERS["shape_min_solidity"] + and result.metrics.centroid_offset_px + <= BOUNDARY_RECOVERY_PARAMETERS["shape_max_centroid_offset_px"] + ) + safe = bool( + area_ratio >= BOUNDARY_RECOVERY_PARAMETERS["min_area_growth_ratio"] + and ( + area_ratio + <= BOUNDARY_RECOVERY_PARAMETERS["standard_max_area_growth_ratio"] + or shape_extension + ) + and current_overlap + >= BOUNDARY_RECOVERY_PARAMETERS["min_current_overlap"] + and result.metrics.equivalent_diameter_um + <= BOUNDARY_RECOVERY_PARAMETERS["max_equivalent_diameter_um"] + and result.metrics.centroid_offset_px + <= BOUNDARY_RECOVERY_PARAMETERS["max_centroid_offset_px"] + ) + warnings = [] + if result.metrics.circularity < 0.65: + warnings.append("low_circularity") + if result.metrics.solidity < 0.85: + warnings.append("low_solidity") + if area_ratio > 3.0: + warnings.append("large_area_growth") + if result.metrics.centroid_offset_px > 25.0: + warnings.append("large_centroid_offset") + metrics = { + "area_px": float(proposal_area), + "area_ratio": float(area_ratio), + "current_overlap": float(current_overlap), + "iou": float(iou), + "equivalent_diameter_um": float(result.metrics.equivalent_diameter_um), + "circularity": float(result.metrics.circularity), + "solidity": float(result.metrics.solidity), + "centroid_offset_px": float(result.metrics.centroid_offset_px), + } + return safe, metrics, warnings + + +def _current_metrics(mask: PersistedMask) -> Mapping[str, Any]: + metrics = mask.metadata.get("metrics", {}) + return metrics if isinstance(metrics, Mapping) else {} + + +def _add_proposal_fields( + row: Dict[str, Any], + *, + prefix: str, + result: LocalSegmentationResult | None, + percentile: float | None, + current_local: np.ndarray, + path: Path, + patch: Any, + annotation_id: str, +) -> bool: + if result is None or percentile is None: + row[f"{prefix}_available"] = False + return False + safe, metrics, warnings = _proposal_is_safe(result, current_local) + row[f"{prefix}_available"] = safe + row[f"{prefix}_percentile"] = float(percentile) + for name, value in metrics.items(): + row[f"{prefix}_{name}"] = value + row[f"{prefix}_warnings"] = ";".join(warnings) + if not safe: + return False + _save_provisional_mask( + path, + result=result, + patch=patch, + annotation_id=annotation_id, + percentile=float(percentile), + ) + row[f"{prefix}_mask_path"] = str(path) + return True + + +def _render_card( + runtime: RecallReviewRuntime, + row: Mapping[str, Any], + destination: Path, + *, + radius: int, +) -> None: + center = (int(round(float(row["x"]))), int(round(float(row["y"])))) + patch = runtime.source.read_patch(center, radius) + raw = Image.open(io.BytesIO(runtime.render_patch(center, radius, "local"))).convert( + "RGBA" + ) + existing = Image.open(io.BytesIO(runtime.render_overlay(center, radius))).convert( + "RGBA" + ) + base = Image.alpha_composite(raw, existing) + click_x = center[0] - (center[0] - radius) + click_y = center[1] - (center[1] - radius) + context = base.copy() + _crosshair(context, click_x, click_y) + panels = [ + _panel_label( + context, + f"#{int(row['boundary_index']):03d} RAW + ALL CURRENT MASKS", + color=(0, 255, 242), + ) + ] + + current = load_candidate_mask(Path(str(row["current_mask_path"]))) + current_panel = _draw_mask( + base.copy(), runtime._place_mask(current, patch), color=(255, 211, 72) + ) + _crosshair(current_panel, click_x, click_y) + panels.append( + _panel_label( + current_panel, + f"CURRENT / d {float(row['current_equivalent_diameter_um']):.1f} um", + color=(255, 211, 72), + ) + ) + + for prefix, color, label in ( + ("conservative", (80, 214, 132), "CONSERVATIVE"), + ("expanded", (255, 132, 67), "EXPANDED"), + ): + panel = base.copy() + if bool(row.get(f"{prefix}_available", False)): + persisted = _load_provisional(str(row[f"{prefix}_mask_path"])) + if persisted is None: + raise FileNotFoundError(f"boundary proposal is missing: {prefix}") + panel = _draw_mask( + panel, + runtime._place_mask(persisted, patch), + color=color, + ) + panel_label = ( + f"{label} / P{float(row[f'{prefix}_percentile']):.0f} / " + f"d {float(row[f'{prefix}_equivalent_diameter_um']):.1f} um" + ) + else: + panel_label = f"{label} / NO SAFE PROPOSAL" + _crosshair(panel, click_x, click_y) + panels.append(_panel_label(panel, panel_label, color=color)) + + gutter = 5 + canvas = Image.new( + "RGBA", + ( + sum(panel.width for panel in panels) + gutter * (len(panels) - 1), + panels[0].height, + ), + (245, 238, 224, 255), + ) + x_offset = 0 + for panel in panels: + canvas.paste(panel, (x_offset, 0)) + x_offset += panel.width + gutter + _atomic_save_image(destination, canvas.convert("RGB"), format_name="WEBP") + + +def _metric(value: Any, digits: int = 2) -> str: + try: + number = float(value) + except (TypeError, ValueError): + return "n/a" + return f"{number:.{digits}f}" if np.isfinite(number) else "n/a" + + +def _card_html(row: Mapping[str, Any]) -> str: + key = html.escape(str(row["review_key"]), quote=True) + index = int(row["boundary_index"]) + conservative_disabled = "" if row.get("conservative_available") else " disabled" + expanded_disabled = "" if row.get("expanded_available") else " disabled" + warnings = "; ".join( + token + for token in ( + str(row.get("conservative_warnings", "")), + str(row.get("expanded_warnings", "")), + ) + if token + ) or "none" + return f'''
+ Boundary comparison for {key} +

#{index:03d} / {html.escape(str(row['display_id']))}

{html.escape(str(row['proposal_status']))}
+
{key} · x {int(round(float(row['x'])))} · y {int(round(float(row['y'])))}
+
current d {_metric(row['current_equivalent_diameter_um'],1)} umcurrent circ {_metric(row['current_circularity'])}current solid {_metric(row['current_solidity'])}conservative growth {_metric(row.get('conservative_area_ratio'))}xexpanded growth {_metric(row.get('expanded_area_ratio'))}xwarnings {html.escape(warnings)}
+
+ +
''' + + +_CSS = r''' +:root{--ink:#172522;--panel:#fffaf0;--line:#d1c6b3;--teal:#087b78;--green:#3f9b68;--orange:#d9662b;--yellow:#b58a17;--red:#a94336}*{box-sizing:border-box}body{margin:0;color:var(--ink);background:radial-gradient(circle at 10% 0,#fff8e8 0,transparent 30%),linear-gradient(135deg,#eadfcd,#f8f2e7 66%,#dcebe3);font-family:"Iowan Old Style","Palatino Linotype",Palatino,serif}.shell{width:min(1600px,calc(100% - 24px));margin:auto}.hero{margin:16px 0;padding:24px 28px;border:1px solid var(--line);border-radius:22px;background:linear-gradient(115deg,#fffaf0,#e4f3ec);box-shadow:0 15px 35px rgba(30,45,38,.12)}.hero h1{font-size:clamp(2rem,4.5vw,4.2rem);line-height:.95;margin:.2em 0}.hero p{max-width:1050px;line-height:1.5}.eyebrow,.mono{font-family:"IBM Plex Mono","Aptos Mono","Courier New",monospace}.eyebrow{text-transform:uppercase;letter-spacing:.14em;color:var(--teal);font-size:.75rem;font-weight:700}.toolbar{position:sticky;top:6px;z-index:5;display:flex;gap:8px;align-items:center;flex-wrap:wrap;padding:10px 12px;margin:14px 0;border:1px solid var(--line);border-radius:14px;background:rgba(255,250,240,.95)}button,.button,input{font:inherit;border:1px solid #b9ae9d;border-radius:10px;padding:8px 11px;background:#fffaf0;color:var(--ink)}button{cursor:pointer}button:disabled{opacity:.35;cursor:not-allowed}.button{text-decoration:none}.grow{flex:1}.cards{display:grid;grid-template-columns:1fr;gap:16px;margin-bottom:50px}.card{background:var(--panel);border:1px solid var(--line);border-radius:18px;overflow:hidden;box-shadow:0 9px 24px rgba(32,45,39,.08)}.card.hidden{display:none}.card img{width:100%;display:block;background:#171f1c}.body{padding:14px}.title{display:flex;align-items:center;justify-content:space-between}.title h2{margin:0}.title span{font-family:monospace;color:var(--orange)}.metrics{display:grid;grid-template-columns:repeat(3,1fr);gap:6px;margin:10px 0;font:12px monospace}.metrics span{background:#f0e7d8;padding:7px;border-radius:7px}.actions{display:grid;grid-template-columns:repeat(6,1fr);gap:6px}.actions button.selected[data-choice=keep_current]{background:#f6e6a9;border-color:var(--yellow)}.actions button.selected[data-choice=use_conservative]{background:#d4eedf;border-color:var(--green)}.actions button.selected[data-choice=use_expanded]{background:#ffd7bd;border-color:var(--orange)}.actions button.selected[data-choice=needs_manual],.actions button.selected[data-choice=exclude]{background:#f1d6cf;border-color:var(--red)}.actions button.selected[data-choice=unsure]{background:#e5e1d7;border-color:#847d70}.notes{width:100%;margin-top:8px}.footer{padding:10px 0 50px;color:#68726c}@media(max-width:900px){.actions{grid-template-columns:1fr 1fr 1fr}.metrics{grid-template-columns:1fr 1fr}.toolbar{position:static}} +''' + + +_JS = r''' +const DATA=JSON.parse(document.getElementById('boundary-data').textContent),KEY='aegle-oocyte-precision-boundary:'+DATA.identity.sample_id+':'+DATA.identity.boundary_candidate_table_sha256;let state=JSON.parse(localStorage.getItem(KEY)||'{}'),filter='all';const cards=[...document.querySelectorAll('.card')];function save(){localStorage.setItem(KEY,JSON.stringify(state));progress()}function paint(card){const id=card.dataset.id,s=state[id]||{};card.dataset.review=s.choice||'unreviewed';card.querySelectorAll('[data-choice]').forEach(b=>b.classList.toggle('selected',b.dataset.choice===s.choice));card.querySelector('.notes').value=s.notes??card.querySelector('.notes').value}function progress(){const n=DATA.rows.filter(r=>(state[r.review_key]||{}).choice).length;document.getElementById('progress').textContent=n+' / '+DATA.rows.length+' reviewed'}function apply(){cards.forEach(c=>c.classList.toggle('hidden',filter==='unreviewed'&&c.dataset.review!=='unreviewed'))}cards.forEach(paint);document.querySelectorAll('[data-choice]').forEach(b=>b.onclick=()=>{const card=b.closest('.card'),id=card.dataset.id;state[id]={...(state[id]||{}),choice:b.dataset.choice};paint(card);save();apply()});document.querySelectorAll('.notes').forEach(n=>n.onchange=()=>{const card=n.closest('.card'),id=card.dataset.id;state[id]={...(state[id]||{}),notes:n.value};save()});document.querySelectorAll('[data-filter]').forEach(b=>b.onclick=()=>{filter=b.dataset.filter;apply()});function exportData(type){const rows=DATA.rows.map(r=>({...r,boundary_review_choice:(state[r.review_key]||{}).choice||'',boundary_review_notes:(state[r.review_key]||{}).notes||''})),payload={schema_version:1,review_type:'oocyte_precision_boundary_review',identity:DATA.identity,exported_at:new Date().toISOString(),rows};let blob,name;if(type==='json'){blob=new Blob([JSON.stringify(payload,null,2)],{type:'application/json'});name=DATA.identity.sample_id+'_precision_boundary_review.json'}else{const keys=Object.keys(rows[0]||{}),esc=v=>'"'+String(v??'').replaceAll('"','""')+'"';blob=new Blob([[keys.join(','),...rows.map(r=>keys.map(k=>esc(r[k])).join(','))].join('\n')],{type:'text/csv'});name=DATA.identity.sample_id+'_precision_boundary_review.csv'}const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download=name;a.click();setTimeout(()=>URL.revokeObjectURL(a.href),0)}document.getElementById('export-json').onclick=()=>exportData('json');document.getElementById('export-csv').onclick=()=>exportData('csv');progress();apply(); +''' + + +def generate_precision_boundary_review( + sample_dir: Path, + precision_review_json: Path, + out_dir: Path, + *, + patch_radius_px: int = 220, +) -> PrecisionBoundaryReviewResult: + """Build a review pack for true oocytes rejected only for mask quality.""" + + sample = _load_sample(sample_dir) + precision_path = Path(precision_review_json).resolve() + precision_payload = _read_json(precision_path) + precision_rows = _validate_precision_review( + sample.review_identity, + sample.candidates, + precision_payload, + ) + boundary_rows = [ + row + for row in precision_rows + if str(row.get("manual_status")) == "reject" + and "true_oocyte" in _note_tokens(row.get("manual_notes")) + ] + + candidate_by_key = { + _review_key(row): row for row in sample.candidates.to_dict("records") + } + true_centers = [ + (float(row["center_x"]), float(row["center_y"])) + for row in precision_rows + if str(row.get("manual_status")) == "accept" + or "true_oocyte" in _note_tokens(row.get("manual_notes")) + ] + root = Path(out_dir).resolve() + masks_dir = root / "proposal_masks" + assets_dir = root / "review_assets" + masks_dir.mkdir(parents=True, exist_ok=True) + assets_dir.mkdir(parents=True, exist_ok=True) + output_rows = [] + + with RecallReviewRuntime(sample.sample_dir) as runtime: + for index, review_row in enumerate(boundary_rows, start=1): + key = str(review_row["review_key"]) + candidate = candidate_by_key[key] + x = float(candidate["center_x"]) + y = float(candidate["center_y"]) + other_centers = tuple( + center + for center in true_centers + if float(np.hypot(center[0] - x, center[1] - y)) >= 25.0 + ) + segmentation = runtime.segment_manual_provisionals( + x, + y, + exclude_points_xy=other_centers, + allow_shape_recovery=True, + ) + current = runtime._candidate_mask(candidate) + current_local = runtime._place_mask(current, segmentation.patch) + metrics = _current_metrics(current) + output: Dict[str, Any] = { + "boundary_index": index, + "review_key": key, + "display_id": str(review_row["display_id"]), + "detector_component_id": str(candidate["detector_component_id"]), + "detection_pass": str(candidate["detection_pass"]), + "x": x, + "y": y, + "precision_status": str(review_row["manual_status"]), + "precision_notes": str(review_row.get("manual_notes", "")), + "current_mask_path": str(_mask_path(sample.sample_dir, candidate).resolve()), + "current_area_px": int(current.mask.sum()), + "current_equivalent_diameter_um": float( + metrics.get("equivalent_diameter_um", np.nan) + ), + "current_circularity": float(metrics.get("circularity", np.nan)), + "current_solidity": float(metrics.get("solidity", np.nan)), + "segmentation_error": segmentation.error, + } + conservative_available = _add_proposal_fields( + output, + prefix="conservative", + result=segmentation.conservative, + percentile=segmentation.conservative_percentile, + current_local=current_local, + path=masks_dir / f"boundary-{index:03d}-conservative.npz", + patch=segmentation.patch, + annotation_id=key, + ) + expanded_available = _add_proposal_fields( + output, + prefix="expanded", + result=segmentation.expanded, + percentile=segmentation.expanded_percentile, + current_local=current_local, + path=masks_dir / f"boundary-{index:03d}-expanded.npz", + patch=segmentation.patch, + annotation_id=key, + ) + output["proposal_status"] = ( + "automatic_options" + if conservative_available or expanded_available + else "manual_required" + ) + output_rows.append(output) + + expected_masks = { + Path(str(row[path_key])).name + for row in output_rows + for path_key in ("conservative_mask_path", "expanded_mask_path") + if row.get(path_key) + } + for existing in masks_dir.glob("*.npz"): + if existing.name not in expected_masks: + existing.unlink() + table = pd.DataFrame(output_rows) + if table.empty: + table = pd.DataFrame(columns=BOUNDARY_CANDIDATE_COLUMNS) + candidates_path = root / "precision_boundary_candidates.csv" + _atomic_write_csv(table, candidates_path) + expected_assets = { + f"boundary-{int(row['boundary_index']):03d}.webp" + for row in output_rows + } + for existing in assets_dir.glob("*.webp"): + if existing.name not in expected_assets: + existing.unlink() + for row in output_rows: + _render_card( + runtime, + row, + assets_dir / f"boundary-{int(row['boundary_index']):03d}.webp", + radius=patch_radius_px, + ) + + identity: Dict[str, Any] = { + **sample.review_identity, + "precision_review_json": str(precision_path), + "precision_review_json_sha256": _file_sha256(precision_path), + "boundary_candidate_table": str(candidates_path), + "boundary_candidate_table_sha256": _file_sha256(candidates_path), + "boundary_recovery_parameters": BOUNDARY_RECOVERY_PARAMETERS, + "patch_radius_px": int(patch_radius_px), + } + safe_rows = [_json_safe(row) for row in output_rows] + cards = "".join(_card_html(row) for row in output_rows) + payload = {"identity": identity, "rows": safe_rows} + automatic_review_path = None + if not output_rows: + automatic_review_path = root / "precision_boundary_review_empty.json" + automatic_review = { + "schema_version": PRECISION_BOUNDARY_REVIEW_SCHEMA_VERSION, + "review_type": "oocyte_precision_boundary_review", + "identity": identity, + "exported_at": None, + "rows": [], + "generated_automatically": True, + "reason": "precision review contains no true-oocyte boundary failures", + } + _atomic_write_text( + automatic_review_path, + json.dumps( + _json_safe(automatic_review), + indent=2, + sort_keys=True, + allow_nan=False, + ), + ) + page = f'''{html.escape(sample.sample_id)} Precision boundary review
Aegle / reviewed Precision boundary delta

{html.escape(sample.sample_id)} boundary recovery

These are true oocytes whose current masks were rejected for boundary quality. Cyan shows all current masks in context, yellow is the frozen current candidate, green is the conservative lower-annulus proposal, and orange is the largest bounded expansion. Choose a proposal only when it follows the intended outer oocyte boundary without absorbing a neighbor. This page never modifies detector output.

{cards}
Selections remain browser-local until exported. Final labels change only after an identity-validated boundary review is finalized.
''' + page_path = root / "precision_boundary_review.html" + _atomic_write_text(page_path, page) + proposal_count = sum( + bool(row.get("conservative_available")) + or bool(row.get("expanded_available")) + for row in output_rows + ) + summary = { + "schema_version": 1, + "review_type": "oocyte_precision_boundary_review_pack", + "sample": sample.review_identity, + "precision_review_json": str(precision_path), + "precision_review_json_sha256": _file_sha256(precision_path), + "boundary_card_count": len(output_rows), + "proposal_count": proposal_count, + "manual_only_count": len(output_rows) - proposal_count, + "page": str(page_path), + "candidates": str(candidates_path), + "automatic_review": ( + None if automatic_review_path is None else str(automatic_review_path) + ), + "production_outputs_modified": False, + } + _atomic_write_text( + root / "summary.json", + json.dumps(_json_safe(summary), indent=2, sort_keys=True, allow_nan=False), + ) + return PrecisionBoundaryReviewResult( + page_path=page_path, + candidates_path=candidates_path, + assets_dir=assets_dir, + masks_dir=masks_dir, + card_count=len(output_rows), + proposal_count=proposal_count, + manual_only_count=len(output_rows) - proposal_count, + automatic_review_path=automatic_review_path, + ) + + +__all__ = [ + "BOUNDARY_RECOVERY_PARAMETERS", + "PrecisionBoundaryReviewResult", + "generate_precision_boundary_review", +] diff --git a/aegle/oocyte/precision_manual_boundary_finalize.py b/aegle/oocyte/precision_manual_boundary_finalize.py new file mode 100644 index 0000000..3ca74ab --- /dev/null +++ b/aegle/oocyte/precision_manual_boundary_finalize.py @@ -0,0 +1,854 @@ +"""Finalize reviewed manual polygons into a Precision-resolved v2 delivery.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping, Sequence, Tuple + +import numpy as np +import pandas as pd +from skimage import draw, measure + +from .export import LabelExportResult, export_whole_slide_labels +from .io import load_candidate_mask +from .manual_seed_finalize import ( + _atomic_write_csv, + _tight_mask, + _write_reviewed_mask, +) +from .models import BoundingBox, PersistedMask +from .precision_boundary_finalize import ( + _atomic_copy, + _resolved_overlap_audit, +) +from .precision_boundary_review import _review_key +from .precision_manual_boundary_review import ( + _verify_precision_resolved_delivery, +) +from .recall_review import ( + _atomic_write_text, + _file_sha256, + _json_safe, + _load_sample, + _mask_path, + _read_json, +) + + +PRECISION_RESOLVED_V2_PROFILE_NAME = "precision_resolved_v2" +PRECISION_MANUAL_BOUNDARY_CHOICES = { + "accept_manual_contour", + "exclude", + "unsure", +} +MANUAL_CONTOUR_MIN_DIAMETER_UM = 10.0 +MANUAL_CONTOUR_MAX_DIAMETER_UM = 100.0 + + +@dataclass(frozen=True) +class PrecisionManualBoundaryFinalizeResult: + out_dir: Path + decisions_path: Path + candidates_path: Path + manual_decisions_path: Path + remaining_queue_path: Path + overlap_audit_path: Path + labels: LabelExportResult + manifest_path: Path + resolved_count: int + manual_added_count: int + manual_excluded_count: int + + +def _clean_text(value: Any) -> str: + if value is None or (not isinstance(value, str) and pd.isna(value)): + return "" + return str(value).strip() + + +def _orientation( + a: tuple[float, float], + b: tuple[float, float], + c: tuple[float, float], +) -> float: + return (b[0] - a[0]) * (c[1] - a[1]) - (b[1] - a[1]) * ( + c[0] - a[0] + ) + + +def _on_segment( + a: tuple[float, float], + b: tuple[float, float], + p: tuple[float, float], + *, + tolerance: float = 1e-9, +) -> bool: + return bool( + min(a[0], b[0]) - tolerance <= p[0] <= max(a[0], b[0]) + tolerance + and min(a[1], b[1]) - tolerance + <= p[1] + <= max(a[1], b[1]) + tolerance + and abs(_orientation(a, b, p)) <= tolerance + ) + + +def _segments_intersect( + a: tuple[float, float], + b: tuple[float, float], + c: tuple[float, float], + d: tuple[float, float], +) -> bool: + values = ( + _orientation(a, b, c), + _orientation(a, b, d), + _orientation(c, d, a), + _orientation(c, d, b), + ) + if values[0] * values[1] < 0 and values[2] * values[3] < 0: + return True + return bool( + (abs(values[0]) <= 1e-9 and _on_segment(a, b, c)) + or (abs(values[1]) <= 1e-9 and _on_segment(a, b, d)) + or (abs(values[2]) <= 1e-9 and _on_segment(c, d, a)) + or (abs(values[3]) <= 1e-9 and _on_segment(c, d, b)) + ) + + +def _validate_simple_polygon( + vertices: Sequence[tuple[float, float]], + *, + key: str, +) -> None: + edge_count = len(vertices) + for left_index in range(edge_count): + a = vertices[left_index] + b = vertices[(left_index + 1) % edge_count] + for right_index in range(left_index + 1, edge_count): + if right_index in { + left_index, + (left_index + 1) % edge_count, + (left_index - 1) % edge_count, + }: + continue + if left_index == 0 and right_index == edge_count - 1: + continue + c = vertices[right_index] + d = vertices[(right_index + 1) % edge_count] + if _segments_intersect(a, b, c, d): + raise ValueError(f"manual contour self-intersects for {key}") + + +def _normalize_vertices( + value: Any, + *, + candidate: Mapping[str, Any], + image_shape_yx: Tuple[int, int], + key: str, +) -> list[tuple[float, float]]: + if not isinstance(value, list): + raise ValueError(f"manual contour vertices must be a list for {key}") + vertices = [] + for index, raw_vertex in enumerate(value, start=1): + if not isinstance(raw_vertex, (list, tuple)) or len(raw_vertex) != 2: + raise ValueError(f"manual contour vertex {index} is invalid for {key}") + x, y = float(raw_vertex[0]), float(raw_vertex[1]) + if not np.isfinite(x) or not np.isfinite(y): + raise ValueError(f"manual contour vertex {index} is non-finite for {key}") + point = (x, y) + if not vertices or not np.allclose(point, vertices[-1], atol=1e-6): + vertices.append(point) + if len(vertices) > 1 and np.allclose(vertices[0], vertices[-1], atol=1e-6): + vertices.pop() + if len(vertices) < 3 or len({(round(x, 6), round(y, 6)) for x, y in vertices}) < 3: + raise ValueError(f"manual contour requires three unique vertices for {key}") + origin_x = float(candidate["patch_origin_x"]) + origin_y = float(candidate["patch_origin_y"]) + width = int(candidate["asset_width_px"]) + height = int(candidate["asset_height_px"]) + image_h, image_w = image_shape_yx + for x, y in vertices: + if not ( + origin_x <= x <= origin_x + width - 1 + and origin_y <= y <= origin_y + height - 1 + ): + raise ValueError(f"manual contour leaves its reviewed patch for {key}") + if not (0 <= x < image_w and 0 <= y < image_h): + raise ValueError(f"manual contour leaves the source image for {key}") + _validate_simple_polygon(vertices, key=key) + signed_area = 0.5 * sum( + x0 * y1 - x1 * y0 + for (x0, y0), (x1, y1) in zip(vertices, vertices[1:] + vertices[:1]) + ) + if abs(signed_area) < 1.0: + raise ValueError(f"manual contour has negligible area for {key}") + return vertices + + +def _rasterize_polygon( + vertices: Sequence[tuple[float, float]], + *, + center_xy: tuple[float, float], + image_shape_yx: Tuple[int, int], + pixel_size_um: float, + key: str, +) -> tuple[PersistedMask, Dict[str, float | int | str | None]]: + image_h, image_w = image_shape_yx + xs = np.asarray([point[0] for point in vertices], dtype=np.float64) + ys = np.asarray([point[1] for point in vertices], dtype=np.float64) + x0 = max(0, int(math.floor(float(xs.min())))) + y0 = max(0, int(math.floor(float(ys.min())))) + x1 = min(image_w, int(math.ceil(float(xs.max()))) + 1) + y1 = min(image_h, int(math.ceil(float(ys.max()))) + 1) + bbox = BoundingBox(x0, y0, x1, y1) + rows, columns = draw.polygon(ys - y0, xs - x0, shape=bbox.shape_yx) + mask = np.zeros(bbox.shape_yx, dtype=np.bool_) + mask[rows, columns] = True + if not mask.any(): + raise ValueError(f"manual contour rasterized to an empty mask for {key}") + center_x, center_y = center_xy + local_center_x = int(round(center_x)) - bbox.x0 + local_center_y = int(round(center_y)) - bbox.y0 + if not ( + 0 <= local_center_x < bbox.width + and 0 <= local_center_y < bbox.height + and mask[local_center_y, local_center_x] + ): + raise ValueError(f"manual contour does not contain its reviewed center for {key}") + prop = measure.regionprops(mask.astype(np.uint8))[0] + equivalent_diameter_um = float(prop.equivalent_diameter_area * pixel_size_um) + if not ( + MANUAL_CONTOUR_MIN_DIAMETER_UM + <= equivalent_diameter_um + <= MANUAL_CONTOUR_MAX_DIAMETER_UM + ): + raise ValueError( + f"manual contour diameter {equivalent_diameter_um:.1f} um is outside " + f"[{MANUAL_CONTOUR_MIN_DIAMETER_UM:.1f}, " + f"{MANUAL_CONTOUR_MAX_DIAMETER_UM:.1f}] for {key}" + ) + centroid_x = bbox.x0 + float(prop.centroid[1]) + centroid_y = bbox.y0 + float(prop.centroid[0]) + circularity = ( + 0.0 + if prop.perimeter <= 0 + else float(4.0 * np.pi * prop.area / (prop.perimeter**2)) + ) + metrics: Dict[str, float | int | str | None] = { + "threshold_method": "manual_polygon", + "base_threshold": None, + "annulus_floor": None, + "threshold": None, + "selection_mode": "reviewed_manual_polygon", + "area_px": int(prop.area), + "equivalent_diameter_um": equivalent_diameter_um, + "major_axis_um": float(prop.axis_major_length * pixel_size_um), + "minor_axis_um": float(prop.axis_minor_length * pixel_size_um), + "eccentricity": float(prop.eccentricity), + "solidity": float(prop.solidity), + "circularity": circularity, + "centroid_y_px": float(prop.centroid[0]), + "centroid_x_px": float(prop.centroid[1]), + "centroid_offset_px": float( + np.hypot(centroid_x - center_x, centroid_y - center_y) + ), + "mean_intensity": None, + "max_intensity": None, + } + return ( + PersistedMask( + mask=mask, + bbox=bbox, + image_shape_yx=image_shape_yx, + metadata={"schema_version": 1, "metrics": metrics}, + ), + metrics, + ) + + +def _validate_review_identity( + sample_identity: Mapping[str, Any], + payload: Mapping[str, Any], +) -> Mapping[str, Any]: + if payload.get("schema_version") != 1: + raise ValueError("manual boundary review schema_version must be 1") + if payload.get("review_type") != "oocyte_precision_manual_boundary_review": + raise ValueError( + "review_type must be 'oocyte_precision_manual_boundary_review'" + ) + identity = payload.get("identity") + if not isinstance(identity, Mapping): + raise ValueError("manual boundary review identity is missing") + for field, expected in sample_identity.items(): + if identity.get(field) != expected: + raise ValueError(f"manual boundary identity mismatch for {field}") + for field in ( + "base_precision_resolved_dir", + "base_precision_resolved_manifest", + "base_precision_resolved_manifest_sha256", + "base_resolved_candidates_sha256", + "manual_boundary_queue_sha256", + "manual_boundary_candidate_table", + "manual_boundary_candidate_table_sha256", + ): + if not identity.get(field): + raise ValueError(f"manual boundary identity is missing {field}") + return identity + + +def _validate_review_rows( + payload: Mapping[str, Any], + candidate_table: pd.DataFrame, + *, + image_shape_yx: Tuple[int, int], +) -> Dict[str, Dict[str, Any]]: + if candidate_table["review_key"].astype(str).duplicated().any(): + raise ValueError("manual boundary candidate keys are not unique") + candidates = { + str(row["review_key"]): row for row in candidate_table.to_dict("records") + } + rows = payload.get("rows") + if not isinstance(rows, list): + raise ValueError("manual boundary review rows must be a list") + review_by_key: Dict[str, Dict[str, Any]] = {} + for row_number, raw_row in enumerate(rows, start=1): + if not isinstance(raw_row, Mapping): + raise ValueError(f"manual boundary row {row_number} must be an object") + key = str(raw_row.get("review_key", "")) + if key in review_by_key: + raise ValueError(f"manual boundary review contains duplicate key {key}") + candidate = candidates.get(key) + if candidate is None: + raise ValueError(f"manual boundary review contains unknown key {key}") + for field in ( + "manual_index", + "review_index", + "boundary_index", + "detector_component_id", + "detection_pass", + ): + if str(raw_row.get(field, "")) != str(candidate[field]): + raise ValueError(f"manual boundary metadata changed for {key}: {field}") + for field in ("center_x", "center_y"): + if not np.isclose( + float(raw_row.get(field, np.nan)), + float(candidate[field]), + atol=0.01, + ): + raise ValueError(f"manual boundary coordinates changed for {key}: {field}") + choice = str(raw_row.get("manual_boundary_choice", "")).strip() + if choice not in PRECISION_MANUAL_BOUNDARY_CHOICES: + raise ValueError(f"invalid or missing manual boundary choice for {key}") + if choice == "unsure": + raise ValueError(f"manual boundary review remains unsure for {key}") + vertices: list[tuple[float, float]] = [] + if choice == "accept_manual_contour": + vertices = _normalize_vertices( + raw_row.get("vertices_xy"), + candidate=candidate, + image_shape_yx=image_shape_yx, + key=key, + ) + review_by_key[key] = { + "choice": choice, + "notes": _clean_text(raw_row.get("manual_boundary_notes")), + "vertices": vertices, + "candidate": candidate, + } + if set(review_by_key) != set(candidates): + raise ValueError("manual boundary review rows do not match its candidate table") + return review_by_key + + +def _manual_candidate_row( + source: Mapping[str, Any], + *, + key: str, + mask: PersistedMask, + metrics: Mapping[str, Any], + reviewed_path: Path, + reviewed_sha256: str, + destination: Path, + notes: str, + review_sha256: str, + candidate_table_sha256: str, + current_mask_path: Path, + current_mask_sha256: str, +) -> Dict[str, Any]: + row = dict(source) + ys, xs = np.nonzero(mask.mask) + centroid_x = mask.bbox.x0 + float(xs.mean()) + centroid_y = mask.bbox.y0 + float(ys.mean()) + row.update( + { + "review_key": key, + "resolved_oocyte_id": key, + "source_center_x": source.get("center_x"), + "source_center_y": source.get("center_y"), + "source_acceptance_mode": source.get("acceptance_mode"), + "source_segmentation_pass": source.get("segmentation_pass"), + "accepted": True, + "center_x": int(round(centroid_x)), + "center_y": int(round(centroid_y)), + "component_centroid_x": centroid_x, + "component_centroid_y": centroid_y, + "bbox_x0": mask.bbox.x0, + "bbox_y0": mask.bbox.y0, + "bbox_x1": mask.bbox.x1, + "bbox_y1": mask.bbox.y1, + "local_area_px": int(mask.mask.sum()), + "local_equivalent_diameter_um": metrics.get("equivalent_diameter_um"), + "local_major_axis_um": metrics.get("major_axis_um"), + "local_minor_axis_um": metrics.get("minor_axis_um"), + "local_eccentricity": metrics.get("eccentricity"), + "local_solidity": metrics.get("solidity"), + "local_circularity": metrics.get("circularity"), + "local_centroid_offset_px": metrics.get("centroid_offset_px"), + "local_mean_intensity": None, + "local_max_intensity": None, + "threshold_method": "manual_polygon", + "threshold": None, + "selection_mode": "reviewed_manual_polygon", + "acceptance_mode": "precision_manual_contour", + "segmentation_pass": "precision_manual_contour", + "quality_class": "precision_reviewed_manual_boundary", + "precision_review_status": "reject", + "precision_boundary_choice": "needs_manual", + "precision_resolution_source": "precision_manual_contour", + "manual_boundary_choice": "accept_manual_contour", + "manual_boundary_notes": notes, + "mask_path": str(reviewed_path.relative_to(destination)), + "mask_source_dir": str(destination), + "source_mask_path": str(current_mask_path), + "source_mask_sha256": current_mask_sha256, + "reviewed_mask_sha256": reviewed_sha256, + "precision_manual_boundary_review_json_sha256": review_sha256, + "precision_manual_boundary_candidate_table_sha256": ( + candidate_table_sha256 + ), + "duplicate_suppressed": False, + } + ) + return row + + +def finalize_precision_manual_boundary_review( + sample_dir: Path, + base_resolved_dir: Path, + manual_review_json: Path, + out_dir: Path, + *, + tile_shape_yx: Tuple[int, int] = (512, 512), +) -> PrecisionManualBoundaryFinalizeResult: + """Rasterize reviewed polygons and write an immutable Precision v2.""" + + sample = _load_sample(sample_dir) + base_dir = Path(base_resolved_dir).resolve() + base_manifest = _verify_precision_resolved_delivery( + base_dir, + sample_identity=sample.review_identity, + ) + base_manifest_path = base_dir / "precision_resolved_manifest.json" + base_manifest_sha256 = _file_sha256(base_manifest_path) + review_path = Path(manual_review_json).resolve() + payload = _read_json(review_path) + identity = _validate_review_identity(sample.review_identity, payload) + if Path(str(identity["base_precision_resolved_dir"])).resolve() != base_dir: + raise ValueError("manual boundary base directory does not match review identity") + if Path(str(identity["base_precision_resolved_manifest"])).resolve() != ( + base_manifest_path + ): + raise ValueError("manual boundary base manifest path does not match") + if str(identity["base_precision_resolved_manifest_sha256"]) != ( + base_manifest_sha256 + ): + raise ValueError("manual boundary base manifest SHA-256 mismatch") + + base_candidates_path = base_dir / "precision_resolved_candidates.csv" + base_queue_path = base_dir / "manual_boundary_queue.csv" + if _file_sha256(base_candidates_path) != str( + identity["base_resolved_candidates_sha256"] + ): + raise ValueError("manual boundary base candidate SHA-256 mismatch") + if _file_sha256(base_queue_path) != str(identity["manual_boundary_queue_sha256"]): + raise ValueError("manual boundary queue SHA-256 mismatch") + candidate_table_path = Path( + str(identity["manual_boundary_candidate_table"]) + ).resolve() + if not candidate_table_path.is_file(): + raise FileNotFoundError( + f"manual boundary candidate table is missing: {candidate_table_path}" + ) + candidate_table_sha256 = _file_sha256(candidate_table_path) + if candidate_table_sha256 != str( + identity["manual_boundary_candidate_table_sha256"] + ): + raise ValueError("manual boundary candidate table SHA-256 mismatch") + manual_candidates = pd.read_csv(candidate_table_path) + review_by_key = _validate_review_rows( + payload, + manual_candidates, + image_shape_yx=sample.image_shape_yx, + ) + review_sha256 = _file_sha256(review_path) + + sample_records = sample.candidates.to_dict("records") + sample_by_key = {_review_key(row): row for row in sample_records} + base_candidates = pd.read_csv(base_candidates_path) + base_decisions = pd.read_csv(base_dir / "precision_review_decisions.csv") + if base_candidates["review_key"].astype(str).duplicated().any(): + raise ValueError("base Precision candidates contain duplicate review keys") + if base_decisions["review_key"].astype(str).duplicated().any(): + raise ValueError("base Precision decisions contain duplicate review keys") + + base_masks = [] + for row in base_candidates.to_dict("records"): + path = Path(str(row["mask_path"])) + if not path.is_absolute(): + path = base_dir / path + path = path.resolve() + if not path.is_relative_to(base_dir) or not path.is_file(): + raise ValueError(f"base Precision mask is outside its delivery: {path}") + mask = load_candidate_mask(path) + if mask.image_shape_yx != sample.image_shape_yx: + raise ValueError(f"base Precision mask image shape mismatch: {path}") + base_masks.append((str(row["review_key"]), mask, path, row)) + + manual_selections = [] + manual_decisions = [] + choice_counts: Dict[str, int] = {} + for key, review in review_by_key.items(): + choice = str(review["choice"]) + choice_counts[choice] = choice_counts.get(choice, 0) + 1 + candidate = review["candidate"] + source = sample_by_key.get(key) + if source is None: + raise ValueError(f"manual boundary source candidate is missing: {key}") + current_path = _mask_path(sample.sample_dir, source).resolve() + if current_path != Path(str(candidate["current_mask_path"])).resolve(): + raise ValueError(f"manual boundary current mask changed for {key}") + if not current_path.is_file(): + raise FileNotFoundError(f"manual boundary current mask is missing: {current_path}") + current_sha256 = _file_sha256(current_path) + if current_sha256 != str(candidate["current_mask_sha256"]): + raise ValueError(f"manual boundary current mask SHA-256 mismatch for {key}") + accepted = choice == "accept_manual_contour" + mask = None + metrics = None + if accepted: + mask, metrics = _rasterize_polygon( + review["vertices"], + center_xy=(float(candidate["center_x"]), float(candidate["center_y"])), + image_shape_yx=sample.image_shape_yx, + pixel_size_um=sample.pixel_size_um, + key=key, + ) + manual_selections.append( + { + "key": key, + "choice": choice, + "notes": str(review["notes"]), + "vertices": review["vertices"], + "candidate": candidate, + "source": source, + "current_path": current_path, + "current_sha256": current_sha256, + "mask": _tight_mask(mask), + "metrics": metrics, + } + ) + manual_decisions.append( + { + "manual_index": int(candidate["manual_index"]), + "review_index": int(candidate["review_index"]), + "boundary_index": int(candidate["boundary_index"]), + "review_key": key, + "display_id": str(candidate["display_id"]), + "detector_component_id": str(candidate["detector_component_id"]), + "detection_pass": str(candidate["detection_pass"]), + "center_x": float(candidate["center_x"]), + "center_y": float(candidate["center_y"]), + "manual_boundary_choice": choice, + "manual_boundary_notes": str(review["notes"]), + "vertex_count": len(review["vertices"]), + "vertices_xy_json": json.dumps(review["vertices"]), + "final_accepted": accepted, + "final_source": ( + "precision_manual_contour" + if accepted + else "precision_manual_boundary_excluded" + ), + "current_mask_path": str(current_path), + "current_mask_sha256": current_sha256, + "reviewed_mask_path": "", + "reviewed_mask_sha256": "", + } + ) + + all_selected_masks = [ + (key, mask) for key, mask, _, _ in base_masks + ] + [ + (str(item["key"]), item["mask"]) for item in manual_selections + ] + audit = _resolved_overlap_audit(all_selected_masks) + if not audit.empty: + pairs = ", ".join( + f"{row.left_review_key}/{row.right_review_key}=" + f"{int(row.overlap_pixel_count)}px" + for row in audit.itertuples() + ) + raise ValueError(f"Precision v2 masks overlap: {pairs}") + + destination = Path(out_dir).resolve() + manifest_path = destination / "precision_resolved_manifest.json" + if manifest_path.exists(): + raise FileExistsError(f"immutable Precision v2 already exists: {manifest_path}") + destination.mkdir(parents=True, exist_ok=True) + masks_dir = destination / "reviewed_masks" + masks_dir.mkdir(parents=True, exist_ok=True) + expected_names = {path.name for _, _, path, _ in base_masks} | { + f"{str(item['key']).replace(':', '__')}.npz" for item in manual_selections + } + for existing in masks_dir.glob("*.npz"): + if existing.name not in expected_names: + existing.unlink() + + final_candidate_rows = [] + for key, _, source_path, raw_row in base_masks: + copied_path = masks_dir / source_path.name + _atomic_copy(source_path, copied_path) + if _file_sha256(source_path) != _file_sha256(copied_path): + raise ValueError(f"base mask copy SHA-256 mismatch for {key}") + row = dict(raw_row) + row["mask_path"] = str(copied_path.relative_to(destination)) + row["mask_source_dir"] = str(destination) + row["precision_v2_source"] = "precision_resolved_v1_carry_forward" + final_candidate_rows.append(row) + + manual_decision_by_key = { + str(row["review_key"]): row for row in manual_decisions + } + for item in manual_selections: + key = str(item["key"]) + reviewed_path = masks_dir / f"{key.replace(':', '__')}.npz" + metadata = { + **dict(item["mask"].metadata), + "schema_version": 1, + "sample_id": sample.sample_id, + "candidate_id": str(item["source"]["detector_component_id"]), + "review_key": key, + "resolved_oocyte_id": key, + "profile_name": PRECISION_RESOLVED_V2_PROFILE_NAME, + "base_profile_name": sample.profile_name, + "base_profile_fingerprint": sample.profile_fingerprint, + "implementation_version": sample.implementation_version, + "precision_resolved": True, + "reviewed_manual_polygon": True, + "provisional_only": False, + "manual_boundary_choice": item["choice"], + "manual_boundary_notes": item["notes"], + "manual_polygon_vertices_xy": item["vertices"], + "precision_manual_boundary_review_json": str(review_path), + "precision_manual_boundary_review_json_sha256": review_sha256, + "precision_manual_boundary_candidate_table_sha256": ( + candidate_table_sha256 + ), + "base_precision_resolved_manifest_sha256": base_manifest_sha256, + "source_current_mask_path": str(item["current_path"]), + "source_current_mask_sha256": item["current_sha256"], + "metrics": item["metrics"], + } + _write_reviewed_mask(reviewed_path, mask=item["mask"], metadata=metadata) + reviewed_sha256 = _file_sha256(reviewed_path) + decision = manual_decision_by_key[key] + decision["reviewed_mask_path"] = str(reviewed_path) + decision["reviewed_mask_sha256"] = reviewed_sha256 + final_candidate_rows.append( + _manual_candidate_row( + item["source"], + key=key, + mask=item["mask"], + metrics=item["metrics"], + reviewed_path=reviewed_path, + reviewed_sha256=reviewed_sha256, + destination=destination, + notes=str(item["notes"]), + review_sha256=review_sha256, + candidate_table_sha256=candidate_table_sha256, + current_mask_path=item["current_path"], + current_mask_sha256=str(item["current_sha256"]), + ) + ) + + final_candidates = pd.DataFrame(final_candidate_rows) + if final_candidates["review_key"].astype(str).duplicated().any(): + raise ValueError("Precision v2 candidates contain duplicate review keys") + updated_decisions = base_decisions.copy() + for column in ("selected_variant", "reviewed_mask_path", "reviewed_mask_sha256"): + updated_decisions[column] = updated_decisions[column].fillna("").astype(str) + updated_decisions["manual_boundary_choice"] = "" + updated_decisions["manual_boundary_notes"] = "" + updated_decisions["manual_boundary_review_json_sha256"] = "" + for manual in manual_decisions: + key = str(manual["review_key"]) + matches = updated_decisions["review_key"].astype(str) == key + if int(matches.sum()) != 1: + raise ValueError(f"base Precision decision is missing for {key}") + updated_decisions.loc[matches, "manual_boundary_choice"] = manual[ + "manual_boundary_choice" + ] + updated_decisions.loc[matches, "manual_boundary_notes"] = manual[ + "manual_boundary_notes" + ] + updated_decisions.loc[ + matches, "manual_boundary_review_json_sha256" + ] = review_sha256 + updated_decisions.loc[matches, "final_accepted"] = manual["final_accepted"] + updated_decisions.loc[matches, "resolution_state"] = ( + "resolved" if manual["final_accepted"] else "excluded" + ) + updated_decisions.loc[matches, "final_source"] = manual["final_source"] + updated_decisions.loc[matches, "selected_variant"] = ( + "manual_polygon" if manual["final_accepted"] else "" + ) + updated_decisions.loc[matches, "reviewed_mask_path"] = manual[ + "reviewed_mask_path" + ] + updated_decisions.loc[matches, "reviewed_mask_sha256"] = manual[ + "reviewed_mask_sha256" + ] + + decisions_path = destination / "precision_review_decisions_v2.csv" + candidates_path = destination / "precision_resolved_candidates_v2.csv" + manual_decisions_path = destination / "manual_boundary_decisions.csv" + remaining_queue_path = destination / "manual_boundary_queue.csv" + overlap_audit_path = destination / "mask_overlap_audit.csv" + _atomic_write_csv(updated_decisions, decisions_path) + _atomic_write_csv(final_candidates, candidates_path) + _atomic_write_csv(pd.DataFrame(manual_decisions), manual_decisions_path) + queue_columns = list(pd.read_csv(base_queue_path).columns) + _atomic_write_csv(pd.DataFrame(columns=queue_columns), remaining_queue_path) + _atomic_write_csv(audit, overlap_audit_path) + + labels = export_whole_slide_labels( + final_candidates, + sample_dir=destination, + image_shape_yx=sample.image_shape_yx, + image_path=destination / "oocyte_labels_precision_resolved_v2.ome.tiff", + mapping_path=destination / "oocyte_labels_precision_resolved_v2_mapping.csv", + tile_shape_yx=tile_shape_yx, + ) + if labels.overlap_pixel_count: + raise AssertionError("manual-boundary preflight and label export disagree") + expected_pixels = sum(int(mask.mask.sum()) for _, mask in all_selected_masks) + if labels.assigned_pixel_count != expected_pixels: + raise AssertionError("Precision v2 label pixel count does not match exact masks") + + input_dir = destination / "review_inputs" + input_copies = { + "base_precision_resolved_manifest": ( + base_manifest_path, + input_dir / "base_precision_resolved_manifest.json", + ), + "manual_boundary_candidates": ( + candidate_table_path, + input_dir / "precision_manual_boundary_candidates.csv", + ), + "manual_boundary_review": ( + review_path, + input_dir / "precision_manual_boundary_review.json", + ), + } + for source_path, copy_path in input_copies.values(): + _atomic_copy(source_path, copy_path) + if _file_sha256(source_path) != _file_sha256(copy_path): + raise ValueError(f"review-input copy SHA-256 mismatch: {copy_path}") + + artifact_paths = [ + decisions_path, + candidates_path, + manual_decisions_path, + remaining_queue_path, + overlap_audit_path, + labels.image_path, + labels.mapping_path, + *(copy_path for _, copy_path in input_copies.values()), + *(masks_dir / name for name in sorted(expected_names)), + ] + manual_excluded_count = choice_counts.get("exclude", 0) + manifest = { + "schema_version": 1, + "delivery_name": PRECISION_RESOLVED_V2_PROFILE_NAME, + "delivery_status": "intermediate_precision_only", + "release_ready": False, + "precision_complete": True, + "manual_boundary_complete": True, + "recall_complete": False, + "sample": sample.review_identity, + "base_precision_resolved_dir": str(base_dir), + "base_precision_resolved_manifest_sha256": base_manifest_sha256, + "manual_boundary_review_json": str(review_path), + "manual_boundary_review_json_sha256": review_sha256, + "manual_boundary_review_exported_at": payload.get("exported_at"), + "manual_boundary_candidate_table": str(candidate_table_path), + "manual_boundary_candidate_table_sha256": candidate_table_sha256, + "manual_boundary_choice_counts": choice_counts, + "base_resolved_label_count": int(base_manifest["resolved_label_count"]), + "manual_added_count": len(manual_selections), + "manual_excluded_count": manual_excluded_count, + "resolved_label_count": labels.label_count, + "unresolved_manual_count": 0, + "overlap_audit_row_count": len(audit), + "label_export": { + "label_count": labels.label_count, + "assigned_pixel_count": labels.assigned_pixel_count, + "overlap_pixel_count": labels.overlap_pixel_count, + }, + "production_outputs_modified": False, + "base_precision_outputs_modified": False, + "input_copies": { + name: { + "source_path": str(source_path), + "copied_path": str(copy_path), + "sha256": _file_sha256(copy_path), + } + for name, (source_path, copy_path) in input_copies.items() + }, + "artifacts": { + str(path.relative_to(destination)): { + "path": str(path), + "sha256": _file_sha256(path), + "size_bytes": path.stat().st_size, + } + for path in artifact_paths + }, + } + _atomic_write_text( + manifest_path, + json.dumps(_json_safe(manifest), indent=2, sort_keys=True, allow_nan=False), + ) + return PrecisionManualBoundaryFinalizeResult( + out_dir=destination, + decisions_path=decisions_path, + candidates_path=candidates_path, + manual_decisions_path=manual_decisions_path, + remaining_queue_path=remaining_queue_path, + overlap_audit_path=overlap_audit_path, + labels=labels, + manifest_path=manifest_path, + resolved_count=labels.label_count, + manual_added_count=len(manual_selections), + manual_excluded_count=manual_excluded_count, + ) + + +__all__ = [ + "MANUAL_CONTOUR_MAX_DIAMETER_UM", + "MANUAL_CONTOUR_MIN_DIAMETER_UM", + "PRECISION_MANUAL_BOUNDARY_CHOICES", + "PRECISION_RESOLVED_V2_PROFILE_NAME", + "PrecisionManualBoundaryFinalizeResult", + "finalize_precision_manual_boundary_review", +] diff --git a/aegle/oocyte/precision_manual_boundary_review.py b/aegle/oocyte/precision_manual_boundary_review.py new file mode 100644 index 0000000..1f99d78 --- /dev/null +++ b/aegle/oocyte/precision_manual_boundary_review.py @@ -0,0 +1,365 @@ +"""Identity-bound polygon review for unresolved Precision boundaries.""" + +from __future__ import annotations + +import html +import io +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping, Sequence + +import numpy as np +import pandas as pd +from PIL import Image + +from .io import load_candidate_mask +from .manual_seed_finalize import _atomic_write_csv +from .manual_seed_review import _draw_mask +from .precision_boundary_review import _review_key +from .recall_review import ( + RecallReviewRuntime, + _atomic_save_image, + _atomic_write_text, + _file_sha256, + _json_safe, + _load_sample, + _mask_path, + _read_json, +) + + +PRECISION_MANUAL_BOUNDARY_REVIEW_SCHEMA_VERSION = 1 +PRECISION_MANUAL_BOUNDARY_RENDERER_VERSION = "precision_manual_polygon_v1" + + +@dataclass(frozen=True) +class PrecisionManualBoundaryReviewResult: + page_path: Path + candidates_path: Path + assets_dir: Path + card_count: int + + +def _verify_precision_resolved_delivery( + base_dir: Path, + *, + sample_identity: Mapping[str, Any], +) -> Dict[str, Any]: + root = Path(base_dir).resolve() + manifest_path = root / "precision_resolved_manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError( + f"Precision-resolved manifest does not exist: {manifest_path}" + ) + manifest = _read_json(manifest_path) + if manifest.get("delivery_name") != "precision_resolved_v1": + raise ValueError("base directory is not a precision_resolved_v1 delivery") + if manifest.get("sample") != sample_identity: + raise ValueError("base Precision-resolved sample identity does not match") + if manifest.get("release_ready") is not False: + raise ValueError("base Precision intermediate has an invalid release state") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping): + raise ValueError("base Precision manifest is missing artifacts") + for relative_path, record in artifacts.items(): + if not isinstance(record, Mapping): + raise ValueError(f"invalid base artifact record: {relative_path}") + path = Path(str(record.get("path", ""))).resolve() + if not path.is_file(): + raise FileNotFoundError(f"base Precision artifact is missing: {path}") + if not path.is_relative_to(root): + raise ValueError(f"base Precision artifact is outside its delivery: {path}") + if path.stat().st_size != int(record.get("size_bytes", -1)): + raise ValueError(f"base Precision artifact size mismatch: {relative_path}") + if _file_sha256(path) != str(record.get("sha256", "")): + raise ValueError(f"base Precision artifact SHA-256 mismatch: {relative_path}") + required = ( + "precision_resolved_candidates.csv", + "precision_review_decisions.csv", + "manual_boundary_queue.csv", + "oocyte_labels_precision_resolved_v1.ome.tiff", + "oocyte_labels_precision_resolved_v1_mapping.csv", + ) + for name in required: + if not (root / name).is_file(): + raise FileNotFoundError(f"base Precision delivery is missing {name}") + return manifest + + +def _resolved_mask_path(base_dir: Path, row: Mapping[str, Any]) -> Path: + path = Path(str(row["mask_path"])) + if path.is_absolute(): + return path.resolve() + source_dir = row.get("mask_source_dir") + if source_dir is None or pd.isna(source_dir) or not str(source_dir).strip(): + root = Path(base_dir) + else: + root = Path(str(source_dir)) + return (root / path).resolve() + + +def _intersects_patch(row: Mapping[str, Any], patch: Any) -> bool: + return not ( + float(row["bbox_x0"]) >= patch.bbox.x1 + or float(row["bbox_y0"]) >= patch.bbox.y1 + or float(row["bbox_x1"]) <= patch.bbox.x0 + or float(row["bbox_y1"]) <= patch.bbox.y0 + ) + + +def _render_assets( + runtime: RecallReviewRuntime, + *, + center_xy: tuple[int, int], + radius: int, + current_mask_path: Path, + resolved_candidates: pd.DataFrame, + base_dir: Path, + raw_path: Path, + context_path: Path, +) -> tuple[Any, int]: + patch = runtime.source.read_patch(center_xy, radius) + raw = Image.open( + io.BytesIO(runtime.render_patch(center_xy, radius, "local")) + ).convert("RGBA") + context = raw.copy() + neighbor_count = 0 + for row in resolved_candidates.to_dict("records"): + if not _intersects_patch(row, patch): + continue + mask_path = _resolved_mask_path(base_dir, row) + if not mask_path.is_file() or not mask_path.is_relative_to(base_dir): + raise ValueError(f"resolved mask is outside the base delivery: {mask_path}") + mask = load_candidate_mask(mask_path) + if mask.image_shape_yx != runtime.sample.image_shape_yx: + raise ValueError(f"resolved mask image shape mismatch: {mask_path}") + context = _draw_mask( + context, + runtime._place_mask(mask, patch), + color=(0, 235, 220), + ) + neighbor_count += 1 + current = load_candidate_mask(current_mask_path) + if current.image_shape_yx != runtime.sample.image_shape_yx: + raise ValueError(f"current target mask image shape mismatch: {current_mask_path}") + context = _draw_mask( + context, + runtime._place_mask(current, patch), + color=(255, 205, 55), + ) + _atomic_save_image(raw_path, raw.convert("RGB"), format_name="WEBP") + _atomic_save_image(context_path, context.convert("RGB"), format_name="WEBP") + return patch, neighbor_count + + +def _card_html(row: Mapping[str, Any]) -> str: + key = html.escape(str(row["review_key"]), quote=True) + note = html.escape(str(row["boundary_review_notes"]), quote=True) + return f'''
+
+

#{int(row['manual_index']):03d} / {html.escape(str(row['display_id']))}

manual polygon
+
{key} · x {float(row['center_x']):.1f} · y {float(row['center_y']):.1f}
+

Prior instruction: {html.escape(str(row['boundary_review_notes']))}

+
0 verticesUnreviewed
+
+
+ +
''' + + +_CSS = r''' +:root{--ink:#172522;--paper:#f4ecdd;--panel:#fffaf0;--line:#c8bca7;--teal:#047f78;--cyan:#00ebdc;--orange:#f2662e;--yellow:#ffcd37;--red:#a94336;--shadow:rgba(30,45,38,.14)}*{box-sizing:border-box}body{margin:0;color:var(--ink);background:radial-gradient(circle at 9% 0,#fff9e9 0,transparent 31%),linear-gradient(135deg,#e5dac7,#f6f0e4 62%,#dbe9e1);font-family:"Iowan Old Style","Palatino Linotype",Palatino,serif}.shell{width:min(1500px,calc(100% - 24px));margin:auto}.hero{margin:16px 0;padding:25px 29px;border:1px solid var(--line);border-radius:24px;background:linear-gradient(112deg,#fffaf0,#dff1e9);box-shadow:0 16px 38px var(--shadow)}.eyebrow,.mono{font-family:"IBM Plex Mono","Aptos Mono","Courier New",monospace}.eyebrow{text-transform:uppercase;letter-spacing:.15em;color:var(--teal);font-size:.74rem;font-weight:700}.hero h1{font-size:clamp(2.4rem,5vw,5rem);line-height:.92;margin:.18em 0}.hero p{max-width:1080px;line-height:1.5}.steps{display:grid;grid-template-columns:repeat(3,1fr);gap:8px;margin-top:15px}.steps span{padding:10px 12px;border-radius:10px;background:rgba(255,250,240,.78);border:1px solid #d7cbb8}.toolbar{position:sticky;top:6px;z-index:5;display:flex;gap:8px;align-items:center;flex-wrap:wrap;padding:10px 12px;margin:14px 0;border:1px solid var(--line);border-radius:14px;background:rgba(255,250,240,.95);backdrop-filter:blur(9px)}button,.button,input{font:inherit;border:1px solid #b9ae9d;border-radius:10px;padding:8px 11px;background:#fffaf0;color:var(--ink)}button{cursor:pointer}.button{text-decoration:none}.grow{flex:1}.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(520px,1fr));gap:18px;margin-bottom:50px}.card{overflow:hidden;border:1px solid var(--line);border-radius:20px;background:var(--panel);box-shadow:0 11px 27px rgba(32,45,39,.1)}.card.hidden{display:none}.canvas-shell{padding:12px;background:#17201d}.card canvas{display:block;width:100%;height:auto;border-radius:10px;touch-action:none;cursor:crosshair;background:#080b0a}.body{padding:15px}.title{display:flex;align-items:center;justify-content:space-between}.title h2{margin:0;font-size:1.8rem}.title span{font:12px monospace;color:var(--orange);text-transform:uppercase}.instruction{min-height:3em;padding:9px 11px;border-left:4px solid var(--yellow);background:#f7ead0;line-height:1.35}.status{display:flex;justify-content:space-between;margin:8px 0;font:12px monospace}.edit-actions,.review-actions{display:grid;gap:7px;margin-top:7px}.edit-actions{grid-template-columns:repeat(3,1fr)}.review-actions{grid-template-columns:2fr 1fr 1fr}.review-actions button.selected[data-choice=accept_manual_contour]{background:#cfece0;border-color:var(--teal)}.review-actions button.selected[data-choice=exclude]{background:#f0d4cd;border-color:var(--red)}.review-actions button.selected[data-choice=unsure]{background:#ebe1c8;border-color:#9d8243}.notes{width:100%;margin-top:8px}.footer{padding:8px 0 48px;color:#5f6a64}@media(max-width:760px){.shell{width:min(100% - 10px,1500px)}.hero{padding:18px}.steps{grid-template-columns:1fr}.toolbar{position:static}.cards{grid-template-columns:1fr}.review-actions{grid-template-columns:1fr}} +''' + + +_JS = r''' +const DATA=JSON.parse(document.getElementById('manual-boundary-data').textContent),KEY='aegle-oocyte-manual-boundary:'+DATA.identity.sample_id+':'+DATA.identity.manual_boundary_candidate_table_sha256;let state=JSON.parse(localStorage.getItem(KEY)||'{}'),filter='all';const cards=[...document.querySelectorAll('.card')],images=new Map(),dragging=new Map();function rowFor(id){return DATA.rows.find(r=>r.review_key===id)}function cardState(id){return state[id]||(state[id]={points:[],choice:'',notes:'',showMasks:true})}function save(){localStorage.setItem(KEY,JSON.stringify(state));progress()}function canvasPoint(canvas,event){const rect=canvas.getBoundingClientRect();return[(event.clientX-rect.left)*canvas.width/rect.width,(event.clientY-rect.top)*canvas.height/rect.height]}function nearest(points,p,canvas){const scale=canvas.width/canvas.getBoundingClientRect().width,limit=12*scale;let best=-1,d=Infinity;points.forEach((q,i)=>{const v=Math.hypot(q[0]-p[0],q[1]-p[1]);if(vctx.lineTo(p[0],p[1]));if(pts.length>=3){ctx.closePath();ctx.fillStyle='rgba(242,102,46,.18)';ctx.fill()}ctx.strokeStyle='#f2662e';ctx.lineWidth=2;ctx.stroke();pts.forEach((p,i)=>{ctx.beginPath();ctx.arc(p[0],p[1],i===0?5:4,0,Math.PI*2);ctx.fillStyle=i===0?'#ffcd37':'#fffaf0';ctx.fill();ctx.strokeStyle='#172522';ctx.lineWidth=1;ctx.stroke()})}ctx.strokeStyle='#ff5739';ctx.lineWidth=2;const x=row.center_local_x,y=row.center_local_y;ctx.beginPath();ctx.arc(x,y,9,0,Math.PI*2);ctx.moveTo(x-13,y);ctx.lineTo(x+13,y);ctx.moveTo(x,y-13);ctx.lineTo(x,y+13);ctx.stroke();card.dataset.review=s.choice||'unreviewed';card.querySelector('.point-count').textContent=pts.length+' vertices';card.querySelector('.choice-label').textContent=({accept_manual_contour:'Contour accepted',exclude:'Not oocyte',unsure:'Unsure'})[s.choice]||'Unreviewed';card.querySelectorAll('[data-choice]').forEach(b=>b.classList.toggle('selected',b.dataset.choice===s.choice));card.querySelector('[data-action=toggle-masks]').textContent=s.showMasks?'Hide masks':'Show masks';if(!card.querySelector('.notes').dataset.loaded){card.querySelector('.notes').value=s.notes||row.boundary_review_notes||'';card.querySelector('.notes').dataset.loaded='1'}}function loadImages(card){const id=card.dataset.id,canvas=card.querySelector('canvas'),raw=new Image(),context=new Image();let count=0;function ready(){if(++count===2){images.set(id,{raw,context});render(card)}}raw.onload=ready;context.onload=ready;raw.src=canvas.dataset.raw;context.src=canvas.dataset.context}cards.forEach(card=>{const id=card.dataset.id,canvas=card.querySelector('canvas');cardState(id);loadImages(card);canvas.onpointerdown=e=>{e.preventDefault();canvas.setPointerCapture(e.pointerId);const s=cardState(id),p=canvasPoint(canvas,e),hit=nearest(s.points,p,canvas);if(hit>=0){dragging.set(id,hit)}else{s.points.push(p);dragging.set(id,s.points.length-1);invalidate(s);save();render(card)}};canvas.onpointermove=e=>{if(!dragging.has(id))return;const s=cardState(id),p=canvasPoint(canvas,e),i=dragging.get(id);s.points[i]=[Math.max(0,Math.min(canvas.width-1,p[0])),Math.max(0,Math.min(canvas.height-1,p[1]))];invalidate(s);render(card)};canvas.onpointerup=()=>{if(dragging.delete(id)){save();render(card)}};canvas.onpointercancel=canvas.onpointerup;card.querySelector('[data-action=undo]').onclick=()=>{const s=cardState(id);s.points.pop();invalidate(s);save();render(card)};card.querySelector('[data-action=clear]').onclick=()=>{const s=cardState(id);s.points=[];invalidate(s);save();render(card)};card.querySelector('[data-action=toggle-masks]').onclick=()=>{const s=cardState(id);s.showMasks=!s.showMasks;save();render(card)};card.querySelectorAll('[data-choice]').forEach(b=>b.onclick=()=>{const s=cardState(id);if(b.dataset.choice==='accept_manual_contour'&&s.points.length<3){alert('Add at least three contour vertices first.');return}s.choice=b.dataset.choice;save();render(card);apply()});card.querySelector('.notes').onchange=e=>{const s=cardState(id);s.notes=e.target.value;save()};render(card)});function progress(){const n=DATA.rows.filter(r=>cardState(r.review_key).choice).length;document.getElementById('progress').textContent=n+' / '+DATA.rows.length+' reviewed'}function apply(){cards.forEach(c=>c.classList.toggle('hidden',filter==='unreviewed'&&c.dataset.review!=='unreviewed'))}document.querySelectorAll('[data-filter]').forEach(b=>b.onclick=()=>{filter=b.dataset.filter;apply()});function identityMatches(a,b){const keys=['sample_id','candidate_table_sha256','base_precision_resolved_manifest_sha256','manual_boundary_candidate_table_sha256'];return keys.every(k=>a&&b&&a[k]===b[k])}function exportReview(){const rows=DATA.rows.map(r=>{const s=cardState(r.review_key),vertices=(s.points||[]).map(p=>[Number((r.patch_origin_x+p[0]).toFixed(3)),Number((r.patch_origin_y+p[1]).toFixed(3))]);return{...r,manual_boundary_choice:s.choice||'',manual_boundary_notes:s.notes||r.boundary_review_notes||'',vertices_xy:vertices,vertex_count:vertices.length}}),payload={schema_version:1,review_type:'oocyte_precision_manual_boundary_review',identity:DATA.identity,exported_at:new Date().toISOString(),rows},blob=new Blob([JSON.stringify(payload,null,2)],{type:'application/json'}),a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download=DATA.identity.sample_id+'_precision_manual_boundary_review.json';a.click();setTimeout(()=>URL.revokeObjectURL(a.href),0)}document.getElementById('export-json').onclick=exportReview;const importer=document.getElementById('import-json');document.getElementById('import-button').onclick=()=>importer.click();importer.onchange=async()=>{const file=importer.files[0];if(!file)return;try{const payload=JSON.parse(await file.text());if(payload.schema_version!==1||payload.review_type!=='oocyte_precision_manual_boundary_review'||!identityMatches(payload.identity,DATA.identity))throw new Error('Review identity does not match this page.');const incoming=new Map((payload.rows||[]).map(r=>[r.review_key,r]));if(incoming.size!==DATA.rows.length||DATA.rows.some(r=>!incoming.has(r.review_key)))throw new Error('Review rows do not match this queue.');DATA.rows.forEach(r=>{const v=incoming.get(r.review_key),points=(v.vertices_xy||[]).map(p=>[Number(p[0])-r.patch_origin_x,Number(p[1])-r.patch_origin_y]);state[r.review_key]={points,choice:v.manual_boundary_choice||'',notes:v.manual_boundary_notes||'',showMasks:true}});save();cards.forEach(render);apply()}catch(error){alert(error.message)}finally{importer.value=''}};progress();apply(); +''' + + +def generate_precision_manual_boundary_review( + sample_dir: Path, + base_resolved_dir: Path, + out_dir: Path, + *, + patch_radius_px: int = 220, +) -> PrecisionManualBoundaryReviewResult: + """Generate a polygon editor for a Precision intermediate's manual queue.""" + + if patch_radius_px < 64: + raise ValueError("manual-boundary patch radius must be at least 64 pixels") + sample = _load_sample(sample_dir) + base_dir = Path(base_resolved_dir).resolve() + base_manifest = _verify_precision_resolved_delivery( + base_dir, + sample_identity=sample.review_identity, + ) + base_manifest_path = base_dir / "precision_resolved_manifest.json" + base_manifest_sha256 = _file_sha256(base_manifest_path) + queue_path = base_dir / "manual_boundary_queue.csv" + queue = pd.read_csv(queue_path) + required_queue = { + "review_index", + "boundary_index", + "review_key", + "display_id", + "detector_component_id", + "detection_pass", + "center_x", + "center_y", + "boundary_review_notes", + "current_mask_path", + } + missing = required_queue.difference(queue.columns) + if missing: + raise ValueError(f"manual boundary queue is missing columns: {sorted(missing)}") + if queue.empty: + raise ValueError("Precision intermediate has no unresolved manual boundaries") + if queue["review_key"].astype(str).duplicated().any(): + raise ValueError("manual boundary queue contains duplicate review keys") + expected_count = int(base_manifest.get("unresolved_manual_count", -1)) + if len(queue) != expected_count: + raise ValueError("manual boundary queue count does not match base manifest") + resolved_candidates_path = base_dir / "precision_resolved_candidates.csv" + resolved_candidates = pd.read_csv(resolved_candidates_path) + decisions = pd.read_csv(base_dir / "precision_review_decisions.csv") + decision_by_key = decisions.set_index("review_key", drop=False) + sample_by_key = { + _review_key(row): row for row in sample.candidates.to_dict("records") + } + + root = Path(out_dir).resolve() + assets_dir = root / "review_assets" + assets_dir.mkdir(parents=True, exist_ok=True) + output_rows = [] + with RecallReviewRuntime(sample.sample_dir) as runtime: + for manual_index, queue_row in enumerate(queue.to_dict("records"), start=1): + key = str(queue_row["review_key"]) + if key not in sample_by_key or key not in decision_by_key.index: + raise ValueError(f"manual boundary key is absent from source decisions: {key}") + decision = decision_by_key.loc[key] + if str(decision["resolution_state"]) != "manual_boundary_required": + raise ValueError(f"manual boundary decision state changed for {key}") + source = sample_by_key[key] + if str(source["detector_component_id"]) != str( + queue_row["detector_component_id"] + ): + raise ValueError(f"manual boundary candidate ID changed for {key}") + current_path = _mask_path(sample.sample_dir, source).resolve() + if current_path != Path(str(queue_row["current_mask_path"])).resolve(): + raise ValueError(f"manual boundary current mask path changed for {key}") + if not current_path.is_file(): + raise FileNotFoundError(f"manual boundary current mask is missing: {current_path}") + center = ( + int(round(float(queue_row["center_x"]))), + int(round(float(queue_row["center_y"]))), + ) + raw_name = f"manual-{manual_index:03d}-raw.webp" + context_name = f"manual-{manual_index:03d}-context.webp" + patch, neighbor_count = _render_assets( + runtime, + center_xy=center, + radius=patch_radius_px, + current_mask_path=current_path, + resolved_candidates=resolved_candidates, + base_dir=base_dir, + raw_path=assets_dir / raw_name, + context_path=assets_dir / context_name, + ) + patch_origin_x = center[0] - patch_radius_px + patch_origin_y = center[1] - patch_radius_px + output_rows.append( + { + "manual_index": manual_index, + "review_index": int(queue_row["review_index"]), + "boundary_index": int(queue_row["boundary_index"]), + "review_key": key, + "display_id": str(queue_row["display_id"]), + "detector_component_id": str(queue_row["detector_component_id"]), + "detection_pass": str(queue_row["detection_pass"]), + "center_x": float(queue_row["center_x"]), + "center_y": float(queue_row["center_y"]), + "center_local_x": float(queue_row["center_x"]) - patch_origin_x, + "center_local_y": float(queue_row["center_y"]) - patch_origin_y, + "precision_notes": str(queue_row.get("precision_notes", "")), + "boundary_review_notes": str(queue_row["boundary_review_notes"]), + "current_mask_path": str(current_path), + "current_mask_sha256": _file_sha256(current_path), + "patch_origin_x": patch_origin_x, + "patch_origin_y": patch_origin_y, + "patch_bbox_x0": patch.bbox.x0, + "patch_bbox_y0": patch.bbox.y0, + "patch_bbox_x1": patch.bbox.x1, + "patch_bbox_y1": patch.bbox.y1, + "asset_width_px": int(patch.image.shape[1]), + "asset_height_px": int(patch.image.shape[0]), + "resolved_neighbor_count": neighbor_count, + "raw_asset_name": raw_name, + "raw_asset_sha256": _file_sha256(assets_dir / raw_name), + "context_asset_name": context_name, + "context_asset_sha256": _file_sha256(assets_dir / context_name), + } + ) + + expected_assets = { + str(row[name]) + for row in output_rows + for name in ("raw_asset_name", "context_asset_name") + } + for existing in assets_dir.glob("*.webp"): + if existing.name not in expected_assets: + existing.unlink() + candidates_path = root / "precision_manual_boundary_candidates.csv" + _atomic_write_csv(pd.DataFrame(output_rows), candidates_path) + identity = { + **sample.review_identity, + "base_precision_resolved_dir": str(base_dir), + "base_precision_resolved_manifest": str(base_manifest_path), + "base_precision_resolved_manifest_sha256": base_manifest_sha256, + "base_resolved_candidates_sha256": _file_sha256(resolved_candidates_path), + "manual_boundary_queue_sha256": _file_sha256(queue_path), + "manual_boundary_candidate_table": str(candidates_path), + "manual_boundary_candidate_table_sha256": _file_sha256(candidates_path), + "renderer_version": PRECISION_MANUAL_BOUNDARY_RENDERER_VERSION, + "patch_radius_px": patch_radius_px, + } + payload = { + "identity": identity, + "rows": [_json_safe(row) for row in output_rows], + } + cards = "".join(_card_html(row) for row in output_rows) + script_payload = json.dumps(payload, allow_nan=False).replace("{html.escape(sample.sample_id)} Manual boundary review
Aegle / exact manual oocyte boundary

{html.escape(sample.sample_id)} contour desk

Trace only the intended outer oocyte boundary on native UCHL1. Cyan marks already resolved oocytes; yellow is the rejected current target mask; orange is your polygon. Browser points are evidence only until the Python finalizer validates and rasterizes the exported JSON.

1. Click around the intended boundary.2. Drag handles to refine; hide masks when needed.3. Accept each contour and export JSON.
{cards}
The first vertex is yellow. Avoid cyan neighbors and follicular halo. Any edited contour returns to Unreviewed until explicitly accepted again.
''' + page_path = root / "precision_manual_boundary_review.html" + _atomic_write_text(page_path, page) + summary = { + "schema_version": 1, + "review_type": "oocyte_precision_manual_boundary_review_pack", + "sample": sample.review_identity, + "base_precision_resolved_manifest": str(base_manifest_path), + "base_precision_resolved_manifest_sha256": base_manifest_sha256, + "manual_boundary_card_count": len(output_rows), + "page": str(page_path), + "candidates": str(candidates_path), + "production_outputs_modified": False, + "base_precision_outputs_modified": False, + } + _atomic_write_text( + root / "summary.json", + json.dumps(_json_safe(summary), indent=2, sort_keys=True, allow_nan=False), + ) + return PrecisionManualBoundaryReviewResult( + page_path=page_path, + candidates_path=candidates_path, + assets_dir=assets_dir, + card_count=len(output_rows), + ) + + +__all__ = [ + "PRECISION_MANUAL_BOUNDARY_RENDERER_VERSION", + "PRECISION_MANUAL_BOUNDARY_REVIEW_SCHEMA_VERSION", + "PrecisionManualBoundaryReviewResult", + "generate_precision_manual_boundary_review", +] diff --git a/aegle/oocyte/profiling.py b/aegle/oocyte/profiling.py new file mode 100644 index 0000000..513ebc2 --- /dev/null +++ b/aegle/oocyte/profiling.py @@ -0,0 +1,968 @@ +"""Profile final oocyte labels directly against registered raw channels.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import logging +import math +import os +import re +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, Mapping, Sequence, Tuple + +import numpy as np +import pandas as pd +import tifffile +import zarr +from skimage.measure import regionprops + +from .models import BoundingBox + + +logger = logging.getLogger(__name__) + +OOCYTE_PROFILING_VERSION = "oocyte_profiling_v1" +MAX_LABEL_VALUE = int(np.iinfo(np.uint16).max) + +MARKER_ID_COLUMNS = ["sample_id", "oocyte_id", "label_id"] +METADATA_COLUMNS = [ + "sample_id", + "oocyte_id", + "label_id", + "detector_component_id", + "detector_score", + "acceptance_mode", + "center_x", + "center_y", + "centroid_x", + "centroid_y", + "area_px", + "area_um2", + "equivalent_diameter_um", + "perimeter_px", + "perimeter_um", + "major_axis_um", + "minor_axis_um", + "eccentricity", + "solidity", + "circularity", + "bbox_x0", + "bbox_y0", + "bbox_x1", + "bbox_y1", + "assigned_pixel_count", + "overlap_pixel_count", + "mask_path", + "detection_pass", + "segmentation_pass", + "seed_source", + "source_annotation_id", + "failure_class", + "manual_mask_choice", + "shape_review_choice", + "boundary_warning", +] +OVERVIEW_COLUMNS = [ + "sample_id", + "oocyte_id", + "label_id", + "centroid_y", + "centroid_x", + "area_px", + "area_um2", +] +PROVENANCE_COLUMNS = [ + "detection_pass", + "segmentation_pass", + "seed_source", + "source_annotation_id", + "failure_class", + "manual_mask_choice", + "shape_review_choice", + "boundary_warning", +] + + +@dataclass(frozen=True) +class OocyteProfilingResult: + sample_id: str + oocyte_count: int + channel_count: int + output_dir: Path + artifact_paths: Dict[str, Path] + runtime_seconds: float + + +@dataclass(frozen=True) +class _ChannelRecord: + source_row_index: int + channel_index: int + channel_id: str + antibody_name: str + exported_column_name: str + measurement_class: str + + +@dataclass(frozen=True) +class _ObjectGeometry: + label_id: int + oocyte_id: str + bbox: BoundingBox + metadata: Dict[str, Any] + + +@dataclass(frozen=True) +class _ProfileRegion: + bbox: BoundingBox + + +class _OmeZarrReader: + """Context-managed bounded reader for one OME-TIFF series.""" + + def __init__(self, path: Path, *, require_channels: bool) -> None: + self.path = Path(path).resolve() + self.require_channels = require_channels + self._tif: tifffile.TiffFile | None = None + self._store: Any = None + self._array: Any = None + self.axes = "" + self.shape: Tuple[int, ...] = () + self.dtype = np.dtype(np.uint8) + self.image_shape_yx: Tuple[int, int] = (0, 0) + self.channel_count = 0 + self.storage: Dict[str, Any] = {} + + def __enter__(self) -> "_OmeZarrReader": + if not self.path.is_file(): + raise FileNotFoundError(self.path) + self._tif = tifffile.TiffFile(self.path) + series = self._tif.series[0] + self.axes = str(series.axes) + self.shape = tuple(int(value) for value in series.shape) + self.dtype = np.dtype(series.dtype) + if self.axes.count("Y") != 1 or self.axes.count("X") != 1: + self.close() + raise ValueError( + f"OME-TIFF must contain exactly one Y and X axis: {self.path} " + f"has axes {self.axes!r}" + ) + if self.axes.count("C") > 1: + self.close() + raise ValueError(f"OME-TIFF has multiple channel axes: {self.axes!r}") + if self.require_channels and "C" not in self.axes: + self.close() + raise ValueError(f"raw OME-TIFF has no channel axis: {self.axes!r}") + if not self.require_channels and "C" in self.axes: + channel_size = self.shape[self.axes.index("C")] + if channel_size != 1: + self.close() + raise ValueError( + f"label OME-TIFF must be two-dimensional, got {self.axes!r} " + f"with {channel_size} channels" + ) + self.image_shape_yx = ( + self.shape[self.axes.index("Y")], + self.shape[self.axes.index("X")], + ) + self.channel_count = ( + self.shape[self.axes.index("C")] if "C" in self.axes else 0 + ) + page = series.pages[0] + self.storage = { + "is_tiled": bool(page.is_tiled), + "tile_width": int(page.tilewidth or 0), + "tile_height": int(page.tilelength or 0), + "rows_per_strip": int(page.rowsperstrip or 0), + "compression": str(page.compression.name), + } + self._store = series.aszarr() + self._array = zarr.open(self._store, mode="r") + return self + + def close(self) -> None: + if self._store is not None: + close = getattr(self._store, "close", None) + if close is not None: + close() + self._store = None + if self._tif is not None: + self._tif.close() + self._tif = None + self._array = None + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None: + self.close() + + def read_region( + self, + bbox: BoundingBox, + *, + channel_index: int | None = None, + ) -> np.ndarray: + if self._array is None: + raise RuntimeError("OME reader is not open") + image_h, image_w = self.image_shape_yx + if bbox.x1 > image_w or bbox.y1 > image_h: + raise ValueError("requested OME region exceeds image dimensions") + resolved_channel_index = channel_index + if "C" in self.axes: + if not self.require_channels and resolved_channel_index is None: + resolved_channel_index = 0 + if resolved_channel_index is None: + raise ValueError("channel index is required for a channel image") + if not 0 <= resolved_channel_index < self.channel_count: + raise IndexError( + f"channel index {resolved_channel_index} outside " + f"[0, {self.channel_count})" + ) + elif channel_index is not None: + raise ValueError("channel index is invalid for a label image") + + indexer: list[Any] = [] + remaining_axes: list[str] = [] + for axis, axis_size in zip(self.axes, self.shape): + if axis == "C": + indexer.append(resolved_channel_index) + elif axis == "Y": + indexer.append(slice(bbox.y0, bbox.y1)) + remaining_axes.append(axis) + elif axis == "X": + indexer.append(slice(bbox.x0, bbox.x1)) + remaining_axes.append(axis) + elif axis_size == 1: + indexer.append(0) + else: + raise ValueError( + f"unsupported non-singleton OME axis {axis!r} with size " + f"{axis_size} in {self.path}" + ) + + region = np.asarray(self._array[tuple(indexer)]) + if set(remaining_axes) != {"Y", "X"} or region.ndim != 2: + raise ValueError( + f"OME region did not resolve to Y/X: axes={remaining_axes}, " + f"shape={region.shape}" + ) + if remaining_axes != ["Y", "X"]: + region = np.transpose( + region, + (remaining_axes.index("Y"), remaining_axes.index("X")), + ) + return region + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _atomic_write_csv(table: pd.DataFrame, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".csv", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + table.to_csv(handle, index=False) + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _atomic_write_json(payload: Mapping[str, Any], destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".json", + prefix=f".{destination.name}.", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _unique_channel_names(names: Sequence[str]) -> list[str]: + counts: Dict[str, int] = {} + used: set[str] = set() + unique: list[str] = [] + for name in names: + occurrence = counts.get(name, 0) + candidate = name if occurrence == 0 else f"{name}_{occurrence}" + while candidate in used: + occurrence += 1 + candidate = f"{name}_{occurrence}" + counts[name] = occurrence + 1 + used.add(candidate) + unique.append(candidate) + return unique + + +def _read_channels(antibodies_path: Path, channel_count: int) -> list[_ChannelRecord]: + path = Path(antibodies_path).resolve() + if not path.is_file(): + raise FileNotFoundError(path) + with path.open(newline="") as handle: + reader = csv.DictReader(handle, delimiter="\t") + if not reader.fieldnames: + raise ValueError(f"antibody table has no header: {path}") + name_column = next( + ( + name + for name in ("antibody_name", "marker_name", "name") + if name in reader.fieldnames + ), + None, + ) + if name_column is None: + raise ValueError(f"antibody table has no marker-name column: {path}") + raw_rows = list(reader) + + if len(raw_rows) != channel_count: + raise ValueError( + f"antibody row count {len(raw_rows)} does not match raw image channel " + f"count {channel_count}" + ) + names = [str(row.get(name_column, "")).strip() for row in raw_rows] + if any(not name for name in names): + raise ValueError("antibody names must not be empty") + exported_names = _unique_channel_names(names) + if set(exported_names).intersection(MARKER_ID_COLUMNS): + raise ValueError("antibody names conflict with profiling identifier columns") + + records: list[_ChannelRecord] = [] + for row_index, (row, name, exported_name) in enumerate( + zip(raw_rows, names, exported_names) + ): + channel_id = str(row.get("channel_id", "")).strip() + match = re.search(r":(\d+)$", channel_id) + channel_index = int(match.group(1)) if match else row_index + records.append( + _ChannelRecord( + source_row_index=row_index, + channel_index=channel_index, + channel_id=channel_id, + antibody_name=name, + exported_column_name=exported_name, + measurement_class=( + "nuclear_stain" + if name.casefold() == "dapi" + else "protein_marker" + ), + ) + ) + indices = [record.channel_index for record in records] + if sorted(indices) != list(range(channel_count)): + raise ValueError( + "antibody channel IDs must map one-to-one onto raw image channels " + f"0..{channel_count - 1}" + ) + return records + + +def _coerce_int_series(table: pd.DataFrame, column: str) -> pd.Series: + values = pd.to_numeric(table[column], errors="coerce") + if values.isna().any() or not np.all(np.equal(values, np.floor(values))): + raise ValueError(f"mapping column {column!r} must contain integers") + return values.astype(np.int64) + + +def _read_mapping(mapping_path: Path) -> pd.DataFrame: + path = Path(mapping_path).resolve() + if not path.is_file(): + raise FileNotFoundError(path) + mapping = pd.read_csv(path) + required = { + "label", + "bbox_x0", + "bbox_y0", + "bbox_x1", + "bbox_y1", + } + missing = required.difference(mapping.columns) + if missing: + raise ValueError(f"label mapping missing columns: {sorted(missing)}") + if "oocyte_id" not in mapping.columns and "detector_component_id" not in mapping.columns: + raise ValueError( + "label mapping requires oocyte_id or detector_component_id" + ) + for column in required: + mapping[column] = _coerce_int_series(mapping, column) + if (mapping["label"] <= 0).any(): + raise ValueError("mapping labels must be positive integers") + if mapping["label"].duplicated().any(): + raise ValueError("mapping labels must be unique") + return mapping.sort_values("label", kind="stable").reset_index(drop=True) + + +def _read_provenance(candidates_path: Path | None) -> Dict[str, Dict[str, Any]]: + if candidates_path is None: + return {} + path = Path(candidates_path).resolve() + if not path.is_file(): + raise FileNotFoundError(path) + table = pd.read_csv(path) + if "detector_component_id" not in table.columns: + raise ValueError("candidate table has no detector_component_id column") + ids = table["detector_component_id"].astype(str) + if ids.duplicated().any(): + raise ValueError("candidate table detector_component_id values must be unique") + rows: Dict[str, Dict[str, Any]] = {} + for row in table.to_dict("records"): + rows[str(row["detector_component_id"])] = row + return rows + + +def _scan_label_counts( + reader: _OmeZarrReader, + *, + strip_height_px: int, +) -> np.ndarray: + if strip_height_px < 1: + raise ValueError("label scan strip height must be positive") + image_h, image_w = reader.image_shape_yx + counts = np.zeros(1, dtype=np.int64) + for y0 in range(0, image_h, strip_height_px): + y1 = min(image_h, y0 + strip_height_px) + region = reader.read_region(BoundingBox(0, y0, image_w, y1)) + if not np.issubdtype(region.dtype, np.integer): + raise ValueError(f"label image must use an integer dtype, got {region.dtype}") + if np.issubdtype(region.dtype, np.signedinteger) and int(region.min()) < 0: + raise ValueError("label image contains negative values") + maximum = int(region.max()) + if maximum > MAX_LABEL_VALUE: + raise ValueError( + f"label value {maximum} exceeds uint16 release limit {MAX_LABEL_VALUE}" + ) + chunk_counts = np.bincount(region.ravel(), minlength=maximum + 1) + if len(chunk_counts) > len(counts): + counts = np.pad(counts, (0, len(chunk_counts) - len(counts))) + counts[: len(chunk_counts)] += chunk_counts.astype(np.int64, copy=False) + return counts + + +def _optional_value(row: Mapping[str, Any], key: str, default: Any = "") -> Any: + value = row.get(key, default) + if value is None or (not isinstance(value, (list, dict)) and pd.isna(value)): + return default + return value + + +def _coerce_bool(value: Any, *, field_name: str) -> bool: + if isinstance(value, (bool, np.bool_)): + return bool(value) + if isinstance(value, (int, np.integer)) and value in (0, 1): + return bool(value) + normalized = str(value).strip().casefold() + if normalized in {"true", "yes", "1"}: + return True + if normalized in {"false", "no", "0", ""}: + return False + raise ValueError(f"invalid boolean value for {field_name}: {value!r}") + + +def _make_oocyte_id(sample_id: str, row: Mapping[str, Any]) -> str: + supplied = str(_optional_value(row, "oocyte_id", "")).strip() + if supplied: + return supplied + component = str(_optional_value(row, "detector_component_id", "")).strip() + if not component: + raise ValueError("mapping row has no stable component identifier") + return f"{sample_id}__{component}" + + +def _extract_object_geometries( + reader: _OmeZarrReader, + mapping: pd.DataFrame, + label_counts: np.ndarray, + *, + sample_id: str, + pixel_size_um: float, + provenance: Mapping[str, Mapping[str, Any]], +) -> list[_ObjectGeometry]: + image_h, image_w = reader.image_shape_yx + geometries: list[_ObjectGeometry] = [] + seen_oocyte_ids: set[str] = set() + for row in mapping.to_dict("records"): + label_id = int(row["label"]) + mapping_bbox = BoundingBox( + int(row["bbox_x0"]), + int(row["bbox_y0"]), + int(row["bbox_x1"]), + int(row["bbox_y1"]), + ) + if mapping_bbox.x1 > image_w or mapping_bbox.y1 > image_h: + raise ValueError(f"mapping bbox exceeds image dimensions for label {label_id}") + patch = reader.read_region(mapping_bbox) + mask = patch == label_id + expected_count = int(label_counts[label_id]) + if expected_count <= 0: + raise ValueError(f"mapping label {label_id} is absent from label image") + if int(mask.sum()) != expected_count: + raise ValueError( + f"label {label_id} has pixels outside its mapping bbox: " + f"bbox count {int(mask.sum())}, whole-image count {expected_count}" + ) + local_y, local_x = np.nonzero(mask) + local_y0 = int(local_y.min()) + local_y1 = int(local_y.max()) + 1 + local_x0 = int(local_x.min()) + local_x1 = int(local_x.max()) + 1 + tight_mask = mask[local_y0:local_y1, local_x0:local_x1] + bbox = BoundingBox( + mapping_bbox.x0 + local_x0, + mapping_bbox.y0 + local_y0, + mapping_bbox.x0 + local_x1, + mapping_bbox.y0 + local_y1, + ) + props = regionprops(tight_mask.astype(np.uint8, copy=False)) + if len(props) != 1: + raise ValueError(f"label {label_id} did not produce one morphology region") + prop = props[0] + area_px = int(prop.area) + perimeter_px = float(prop.perimeter) + circularity = ( + float(4.0 * math.pi * area_px / (perimeter_px * perimeter_px)) + if perimeter_px > 0 + else 0.0 + ) + oocyte_id = _make_oocyte_id(sample_id, row) + if oocyte_id in seen_oocyte_ids: + raise ValueError(f"duplicate stable oocyte_id: {oocyte_id}") + seen_oocyte_ids.add(oocyte_id) + + component_id = str( + _optional_value(row, "detector_component_id", oocyte_id) + ) + candidate = provenance.get(component_id, {}) + metadata: Dict[str, Any] = { + "sample_id": sample_id, + "oocyte_id": oocyte_id, + "label_id": label_id, + "detector_component_id": component_id, + "detector_score": float(_optional_value(row, "detector_score", np.nan)), + "acceptance_mode": str(_optional_value(row, "acceptance_mode", "")), + "center_x": _optional_value(row, "center_x", np.nan), + "center_y": _optional_value(row, "center_y", np.nan), + "centroid_x": float(bbox.x0 + prop.centroid[1]), + "centroid_y": float(bbox.y0 + prop.centroid[0]), + "area_px": area_px, + "area_um2": float(area_px * pixel_size_um * pixel_size_um), + "equivalent_diameter_um": float( + prop.equivalent_diameter_area * pixel_size_um + ), + "perimeter_px": perimeter_px, + "perimeter_um": float(perimeter_px * pixel_size_um), + "major_axis_um": float(prop.axis_major_length * pixel_size_um), + "minor_axis_um": float(prop.axis_minor_length * pixel_size_um), + "eccentricity": float(prop.eccentricity), + "solidity": float(prop.solidity), + "circularity": circularity, + "bbox_x0": bbox.x0, + "bbox_y0": bbox.y0, + "bbox_x1": bbox.x1, + "bbox_y1": bbox.y1, + "assigned_pixel_count": int( + _optional_value(row, "assigned_pixel_count", expected_count) + ), + "overlap_pixel_count": int( + _optional_value(row, "overlap_pixel_count", 0) + ), + "mask_path": str(_optional_value(row, "mask_path", "")), + } + if metadata["assigned_pixel_count"] != expected_count: + raise ValueError( + f"mapping assigned_pixel_count mismatch for label {label_id}: " + f"{metadata['assigned_pixel_count']} != {expected_count}" + ) + for column in PROVENANCE_COLUMNS: + source = row if column in row and not pd.isna(row[column]) else candidate + default = False if column == "boundary_warning" else "" + value = _optional_value(source, column, default) + if column == "boundary_warning": + value = _coerce_bool(value, field_name=column) + metadata[column] = value + geometries.append( + _ObjectGeometry( + label_id=label_id, + oocyte_id=oocyte_id, + bbox=bbox, + metadata=metadata, + ) + ) + return geometries + + +def _build_profile_regions( + geometries: Sequence[_ObjectGeometry], + *, + max_height_px: int, + merge_gap_px: int, +) -> list[_ProfileRegion]: + if max_height_px < 1: + raise ValueError("profile region height must be positive") + if merge_gap_px < 0: + raise ValueError("profile region merge gap must be non-negative") + if not geometries: + return [] + + intervals = sorted((item.bbox.y0, item.bbox.y1) for item in geometries) + merged: list[Tuple[int, int]] = [] + start, end = intervals[0] + for next_start, next_end in intervals[1:]: + if next_start <= end + merge_gap_px: + end = max(end, next_end) + else: + merged.append((start, end)) + start, end = next_start, next_end + merged.append((start, end)) + + regions: list[_ProfileRegion] = [] + for merged_y0, merged_y1 in merged: + for y0 in range(merged_y0, merged_y1, max_height_px): + y1 = min(merged_y1, y0 + max_height_px) + intersecting = [ + item.bbox + for item in geometries + if item.bbox.y0 < y1 and item.bbox.y1 > y0 + ] + if not intersecting: + continue + regions.append( + _ProfileRegion( + bbox=BoundingBox( + min(bbox.x0 for bbox in intersecting), + y0, + max(bbox.x1 for bbox in intersecting), + y1, + ) + ) + ) + return regions + + +def _profile_marker_means( + source: _OmeZarrReader, + labels: _OmeZarrReader, + channels: Sequence[_ChannelRecord], + geometries: Sequence[_ObjectGeometry], + label_counts: np.ndarray, + *, + max_region_height_px: int, + merge_gap_px: int, +) -> tuple[np.ndarray, list[_ProfileRegion]]: + regions = _build_profile_regions( + geometries, + max_height_px=max_region_height_px, + merge_gap_px=merge_gap_px, + ) + if not geometries: + return np.empty((0, len(channels)), dtype=np.float64), regions + + maximum_label = max(item.label_id for item in geometries) + sums = np.zeros((maximum_label + 1, len(channels)), dtype=np.float64) + covered_counts = np.zeros(maximum_label + 1, dtype=np.int64) + for region_index, region in enumerate(regions, start=1): + label_patch = labels.read_region(region.bbox) + patch_counts = np.bincount(label_patch.ravel(), minlength=maximum_label + 1) + covered_counts += patch_counts[: maximum_label + 1].astype( + np.int64, copy=False + ) + logger.info( + "Profiling region %d/%d: bbox=%s, shape=%s", + region_index, + len(regions), + region.bbox.as_tuple(), + region.bbox.shape_yx, + ) + for output_index, channel in enumerate(channels): + raw_patch = source.read_region( + region.bbox, + channel_index=channel.channel_index, + ) + channel_sums = np.bincount( + label_patch.ravel(), + weights=raw_patch.ravel(), + minlength=maximum_label + 1, + ) + sums[:, output_index] += channel_sums[: maximum_label + 1] + + ordered_labels = np.asarray([item.label_id for item in geometries], dtype=np.int64) + expected_counts = label_counts[ordered_labels] + if not np.array_equal(covered_counts[ordered_labels], expected_counts): + raise RuntimeError("profile regions did not cover every final label pixel") + means = sums[ordered_labels] / expected_counts[:, None] + return means, regions + + +def _path_identity(path: Path, *, include_sha256: bool) -> Dict[str, Any]: + resolved = Path(path).resolve() + stat = resolved.stat() + identity: Dict[str, Any] = { + "path": str(resolved), + "size_bytes": int(stat.st_size), + "mtime_ns": str(stat.st_mtime_ns), + } + if include_sha256: + identity["sha256"] = _file_sha256(resolved) + return identity + + +def profile_oocyte_labels( + *, + sample_id: str, + image_path: Path, + antibodies_path: Path, + label_path: Path, + mapping_path: Path, + out_dir: Path, + pixel_size_um: float, + candidates_path: Path | None = None, + max_region_height_px: int = 512, + merge_gap_px: int = 16, + label_scan_height_px: int = 1024, +) -> OocyteProfilingResult: + """Write one raw marker-intensity row per final oocyte label. + + The final label image, not a nucleus mask or detector candidate table, defines + which pixels belong to each profiled object. The optional candidate table only + enriches provenance fields and cannot add or remove labels. + """ + + started = time.perf_counter() + sample = str(sample_id).strip() + if not sample: + raise ValueError("sample_id must not be empty") + if not np.isfinite(pixel_size_um) or pixel_size_um <= 0: + raise ValueError("pixel_size_um must be positive and finite") + + raw_path = Path(image_path).resolve() + antibodies = Path(antibodies_path).resolve() + labels_path = Path(label_path).resolve() + mapping_csv = Path(mapping_path).resolve() + candidate_csv = Path(candidates_path).resolve() if candidates_path else None + mapping = _read_mapping(mapping_csv) + provenance = _read_provenance(candidate_csv) + + with _OmeZarrReader(raw_path, require_channels=True) as source, _OmeZarrReader( + labels_path, require_channels=False + ) as labels: + if source.image_shape_yx != labels.image_shape_yx: + raise ValueError( + "raw image and final label dimensions differ: " + f"{source.image_shape_yx} != {labels.image_shape_yx}" + ) + channels = _read_channels(antibodies, source.channel_count) + label_counts = _scan_label_counts( + labels, + strip_height_px=label_scan_height_px, + ) + observed_labels = set(np.flatnonzero(label_counts[1:]) + 1) + mapped_labels = set(int(value) for value in mapping["label"].tolist()) + if observed_labels != mapped_labels: + missing_from_mapping = sorted(observed_labels - mapped_labels) + missing_from_image = sorted(mapped_labels - observed_labels) + raise ValueError( + "label image and mapping label sets differ; " + f"unmapped image labels={missing_from_mapping}, " + f"mapping labels absent from image={missing_from_image}" + ) + geometries = _extract_object_geometries( + labels, + mapping, + label_counts, + sample_id=sample, + pixel_size_um=float(pixel_size_um), + provenance=provenance, + ) + means, profile_regions = _profile_marker_means( + source, + labels, + channels, + geometries, + label_counts, + max_region_height_px=max_region_height_px, + merge_gap_px=merge_gap_px, + ) + source_series = { + "axes": source.axes, + "shape": list(source.shape), + "dtype": str(source.dtype), + "storage": source.storage, + } + label_series = { + "axes": labels.axes, + "shape": list(labels.shape), + "dtype": str(labels.dtype), + "storage": labels.storage, + } + + channel_names = [record.exported_column_name for record in channels] + marker_rows = [] + for object_index, geometry in enumerate(geometries): + row: Dict[str, Any] = { + "sample_id": sample, + "oocyte_id": geometry.oocyte_id, + "label_id": geometry.label_id, + } + row.update( + { + channel_name: float(means[object_index, channel_index]) + for channel_index, channel_name in enumerate(channel_names) + } + ) + marker_rows.append(row) + marker_table = pd.DataFrame( + marker_rows, + columns=MARKER_ID_COLUMNS + channel_names, + ) + metadata_table = pd.DataFrame( + [geometry.metadata for geometry in geometries], + columns=METADATA_COLUMNS, + ) + overview_table = metadata_table[OVERVIEW_COLUMNS].merge( + marker_table, + on=MARKER_ID_COLUMNS, + how="left", + validate="one_to_one", + ) + overview_table = overview_table[ + OVERVIEW_COLUMNS + channel_names + ] + channel_table = pd.DataFrame( + [ + { + "source_row_index": record.source_row_index, + "channel_index": record.channel_index, + "channel_id": record.channel_id, + "antibody_name": record.antibody_name, + "exported_column_name": record.exported_column_name, + "measurement_class": record.measurement_class, + "included": True, + } + for record in channels + ] + ) + + destination = Path(out_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + paths = { + "markers": destination / "oocyte_by_marker.csv", + "metadata": destination / "oocyte_metadata.csv", + "overview": destination / "oocyte_overview.csv", + "channels": destination / "channel_manifest.csv", + "manifest": destination / "profiling_manifest.json", + } + paths["manifest"].unlink(missing_ok=True) + _atomic_write_csv(marker_table, paths["markers"]) + _atomic_write_csv(metadata_table, paths["metadata"]) + _atomic_write_csv(overview_table, paths["overview"]) + _atomic_write_csv(channel_table, paths["channels"]) + + runtime_seconds = time.perf_counter() - started + profile_pixel_count = int(sum(region.bbox.width * region.bbox.height for region in profile_regions)) + manifest = { + "schema_version": 1, + "implementation_version": OOCYTE_PROFILING_VERSION, + "sample_id": sample, + "pixel_size_um": float(pixel_size_um), + "oocyte_count": len(geometries), + "channel_count": len(channels), + "runtime_seconds": runtime_seconds, + "source_image": { + **_path_identity(raw_path, include_sha256=False), + **source_series, + }, + "label_image": { + **_path_identity(labels_path, include_sha256=True), + **label_series, + }, + "mapping": _path_identity(mapping_csv, include_sha256=True), + "antibodies": _path_identity(antibodies, include_sha256=True), + "candidates": ( + _path_identity(candidate_csv, include_sha256=True) + if candidate_csv is not None + else None + ), + "measurement": { + "statistic": "raw_within_mask_mean", + "background_subtracted": False, + "transformed": False, + "normalized": False, + "object_source": "final_label_image", + }, + "bounded_read_plan": { + "region_count": len(profile_regions), + "max_region_height_px": int(max_region_height_px), + "merge_gap_px": int(merge_gap_px), + "label_scan_height_px": int(label_scan_height_px), + "profile_region_pixel_count_per_channel": profile_pixel_count, + "profile_region_channel_pixel_count": profile_pixel_count * len(channels), + "largest_region_shape_yx": ( + list( + max( + (region.bbox.shape_yx for region in profile_regions), + key=lambda shape: shape[0] * shape[1], + ) + ) + if profile_regions + else [0, 0] + ), + }, + "artifacts": { + path.name: { + "size_bytes": int(path.stat().st_size), + "sha256": _file_sha256(path), + } + for key, path in paths.items() + if key != "manifest" + }, + } + _atomic_write_json(manifest, paths["manifest"]) + logger.info( + "Oocyte profiling complete: sample=%s, oocytes=%d, channels=%d, runtime=%.1fs", + sample, + len(geometries), + len(channels), + runtime_seconds, + ) + return OocyteProfilingResult( + sample_id=sample, + oocyte_count=len(geometries), + channel_count=len(channels), + output_dir=destination, + artifact_paths=paths, + runtime_seconds=runtime_seconds, + ) + + +__all__ = [ + "OOCYTE_PROFILING_VERSION", + "OocyteProfilingResult", + "profile_oocyte_labels", +] diff --git a/aegle/oocyte/qc.py b/aegle/oocyte/qc.py new file mode 100644 index 0000000..deebb66 --- /dev/null +++ b/aegle/oocyte/qc.py @@ -0,0 +1,297 @@ +"""Spatial quality-control plots and duplicate diagnostics.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from itertools import combinations +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import Circle +import numpy as np +import pandas as pd +from PIL import Image + + +@dataclass(frozen=True) +class SpatialQcResult: + overview_path: Path + duplicate_suspects: pd.DataFrame + + +@dataclass(frozen=True) +class BatchSpatialQcResult: + qc_dir: Path + overview_index_path: Path + duplicate_suspects_path: Path + overview_atlas_path: Path | None + + +def accepted_duplicate_suspects( + candidates: pd.DataFrame, + *, + pixel_size_um: float, + overlap_fraction_threshold: float = 0.25, +) -> pd.DataFrame: + """Flag accepted pairs whose diameter-derived circles substantially overlap.""" + + if pixel_size_um <= 0: + raise ValueError("pixel_size_um must be positive") + accepted = candidates[candidates["accepted"].astype(bool)] + rows = [] + for left, right in combinations(accepted.to_dict("records"), 2): + left_radius_px = ( + 0.5 * float(left["local_equivalent_diameter_um"]) / pixel_size_um + ) + right_radius_px = ( + 0.5 * float(right["local_equivalent_diameter_um"]) / pixel_size_um + ) + distance_px = float( + np.hypot( + float(left["center_x"]) - float(right["center_x"]), + float(left["center_y"]) - float(right["center_y"]), + ) + ) + overlap_px = max(0.0, left_radius_px + right_radius_px - distance_px) + overlap_fraction = overlap_px / max( + min(left_radius_px, right_radius_px), + 1e-6, + ) + if overlap_fraction < overlap_fraction_threshold: + continue + rows.append( + { + "detector_component_id_a": str(left["detector_component_id"]), + "detector_component_id_b": str(right["detector_component_id"]), + "center_x_a": float(left["center_x"]), + "center_y_a": float(left["center_y"]), + "center_x_b": float(right["center_x"]), + "center_y_b": float(right["center_y"]), + "diameter_um_a": float(left["local_equivalent_diameter_um"]), + "diameter_um_b": float(right["local_equivalent_diameter_um"]), + "pair_distance_px": distance_px, + "radius_px_a": left_radius_px, + "radius_px_b": right_radius_px, + "overlap_fraction_smaller": overlap_fraction, + } + ) + columns = [ + "detector_component_id_a", + "detector_component_id_b", + "center_x_a", + "center_y_a", + "center_x_b", + "center_y_b", + "diameter_um_a", + "diameter_um_b", + "pair_distance_px", + "radius_px_a", + "radius_px_b", + "overlap_fraction_smaller", + ] + output = pd.DataFrame(rows, columns=columns) + if not output.empty: + output = output.sort_values( + "overlap_fraction_smaller", + ascending=False, + ).reset_index(drop=True) + return output + + +def render_spatial_overview( + downsampled_uchl1: np.ndarray, + candidates: pd.DataFrame, + *, + downsample_factor: int, + pixel_size_um: float, + sample_id: str, + out_path: Path, +) -> SpatialQcResult: + """Render whole-slide candidate locations over the raw UCHL1 mean map.""" + + if downsample_factor <= 0: + raise ValueError("downsample_factor must be positive") + background = np.log1p(np.asarray(downsampled_uchl1, dtype=np.float32)) + accepted = candidates[candidates["accepted"].astype(bool)].copy() + rejected = candidates[~candidates["accepted"].astype(bool)].copy() + duplicates = accepted_duplicate_suspects( + candidates, + pixel_size_um=pixel_size_um, + ) + figure, axes = plt.subplots(1, 2, figsize=(16, 7), constrained_layout=True) + for axis in axes: + axis.imshow(background, cmap="gray", interpolation="nearest") + axis.set_axis_off() + + axes[0].set_title( + f"{sample_id} raw UCHL1 proposals\n" + f"accepted={len(accepted)} rejected={len(rejected)}" + ) + if not rejected.empty: + axes[0].scatter( + rejected["center_x"] / downsample_factor, + rejected["center_y"] / downsample_factor, + s=7, + c="#7f8c8d", + alpha=0.45, + linewidths=0, + label="rejected", + ) + if not accepted.empty: + axes[0].scatter( + accepted["center_x"] / downsample_factor, + accepted["center_y"] / downsample_factor, + s=16, + c="#00d4c8", + edgecolors="#102a2a", + linewidths=0.3, + label="accepted", + ) + axes[0].legend(loc="lower right", framealpha=0.8) + + axes[1].set_title( + f"Accepted mask-scale circles\n" + f"duplicate suspects={len(duplicates)}" + ) + score_min = float(accepted["detector_score"].min()) if not accepted.empty else 0.0 + score_max = float(accepted["detector_score"].max()) if not accepted.empty else 1.0 + score_span = max(score_max - score_min, 1e-6) + for record in accepted.to_dict("records"): + score = float(record["detector_score"]) + normalized = (score - score_min) / score_span + color = plt.cm.turbo(0.15 + 0.75 * normalized) + radius_ds = ( + 0.5 + * float(record["local_equivalent_diameter_um"]) + / pixel_size_um + / downsample_factor + ) + axes[1].add_patch( + Circle( + ( + float(record["center_x"]) / downsample_factor, + float(record["center_y"]) / downsample_factor, + ), + radius=max(radius_ds, 1.0), + fill=False, + edgecolor=color, + linewidth=0.8, + alpha=0.9, + ) + ) + for record in duplicates.to_dict("records"): + axes[1].plot( + [ + float(record["center_x_a"]) / downsample_factor, + float(record["center_x_b"]) / downsample_factor, + ], + [ + float(record["center_y_a"]) / downsample_factor, + float(record["center_y_b"]) / downsample_factor, + ], + color="#ff2e63", + linewidth=0.8, + alpha=0.8, + ) + destination = Path(out_path) + destination.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(destination, dpi=180, bbox_inches="tight") + plt.close(figure) + return SpatialQcResult( + overview_path=destination, + duplicate_suspects=duplicates, + ) + + +def compile_batch_spatial_qc( + batch_dir: Path, + summary: pd.DataFrame, + *, + atlas_columns: int = 2, + thumbnail_size: tuple[int, int] = (900, 420), +) -> BatchSpatialQcResult: + """Build a batch index, duplicate table, and overview contact sheet.""" + + if atlas_columns <= 0: + raise ValueError("atlas_columns must be positive") + batch_root = Path(batch_dir) + qc_dir = batch_root / "spatial_qc" + qc_dir.mkdir(parents=True, exist_ok=True) + index_rows = [] + duplicate_tables = [] + overview_paths = [] + complete = summary[summary["status"] == "complete"] + for record in complete.to_dict("records"): + sample_id = str(record["sample_id"]) + sample_dir = batch_root / sample_id + overview_path = sample_dir / "overview.png" + duplicates_path = sample_dir / "accepted_duplicate_suspects.csv" + duplicates = pd.read_csv(duplicates_path) + if not duplicates.empty: + duplicates.insert(0, "sample_id", sample_id) + duplicate_tables.append(duplicates) + index_rows.append( + { + "sample_id": sample_id, + "status": "complete", + "accepted_candidate_count": int(record["accepted_candidate_count"]), + "duplicate_suspect_count": int(len(duplicates)), + "overview_path": str(overview_path), + "duplicate_suspects_path": str(duplicates_path), + } + ) + overview_paths.append((sample_id, overview_path)) + + index_path = qc_dir / "overview_index.csv" + duplicate_path = qc_dir / "accepted_duplicate_suspects.csv" + pd.DataFrame(index_rows).to_csv(index_path, index=False) + duplicate_columns = [ + "sample_id", + "detector_component_id_a", + "detector_component_id_b", + "center_x_a", + "center_y_a", + "center_x_b", + "center_y_b", + "diameter_um_a", + "diameter_um_b", + "pair_distance_px", + "radius_px_a", + "radius_px_b", + "overlap_fraction_smaller", + ] + combined_duplicates = ( + pd.concat(duplicate_tables, ignore_index=True) + if duplicate_tables + else pd.DataFrame(columns=duplicate_columns) + ) + combined_duplicates.to_csv(duplicate_path, index=False) + + atlas_path = None + if overview_paths: + cell_w, cell_h = thumbnail_size + atlas_rows = math.ceil(len(overview_paths) / atlas_columns) + atlas = Image.new( + "RGB", + (cell_w * atlas_columns, cell_h * atlas_rows), + color="white", + ) + for index, (_, overview_path) in enumerate(overview_paths): + with Image.open(overview_path) as image: + tile = image.convert("RGB") + tile.thumbnail(thumbnail_size, Image.Resampling.LANCZOS) + x0 = (index % atlas_columns) * cell_w + (cell_w - tile.width) // 2 + y0 = (index // atlas_columns) * cell_h + (cell_h - tile.height) // 2 + atlas.paste(tile, (x0, y0)) + atlas_path = qc_dir / "all_samples_overview.png" + atlas.save(atlas_path, format="PNG", optimize=True) + return BatchSpatialQcResult( + qc_dir=qc_dir, + overview_index_path=index_path, + duplicate_suspects_path=duplicate_path, + overview_atlas_path=atlas_path, + ) diff --git a/aegle/oocyte/recall_manual_boundary_finalize.py b/aegle/oocyte/recall_manual_boundary_finalize.py new file mode 100644 index 0000000..028a382 --- /dev/null +++ b/aegle/oocyte/recall_manual_boundary_finalize.py @@ -0,0 +1,536 @@ +"""Finalize reviewed Recall polygons into a new manual-seed v2 delivery.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping, Tuple + +import numpy as np +import pandas as pd + +from .export import LabelExportResult, export_whole_slide_labels +from .io import load_candidate_mask +from .manual_seed_finalize import ( + _atomic_write_csv, + _overlap_metrics, + _write_reviewed_mask, +) +from .precision_boundary_finalize import _atomic_copy +from .precision_manual_boundary_finalize import ( + _clean_text, + _normalize_vertices, + _rasterize_polygon, +) +from .recall_manual_boundary_review import ( + RECALL_MANUAL_BOUNDARY_REVIEW_TYPE, + _resolved_mask_path, + _verify_manual_seed_delivery, +) +from .recall_overlay import overlay_dir_from_identity +from .recall_review import ( + _atomic_write_text, + _file_sha256, + _json_safe, + _load_sample, + _read_json, +) + + +RECALL_MANUAL_CONTOUR_PROFILE_NAME = "manual_seed_manual_contour_v1" +RECALL_MANUAL_BOUNDARY_CHOICES = { + "accept_manual_contour", + "exclude", + "unsure", +} + + +@dataclass(frozen=True) +class RecallManualBoundaryFinalizeResult: + out_dir: Path + decisions_path: Path + candidates_path: Path + combined_candidates_path: Path + overlap_audit_path: Path + labels: LabelExportResult + manifest_path: Path + manual_added_count: int + manual_excluded_count: int + combined_label_count: int + + +def _validate_review_identity( + sample_identity: Mapping[str, Any], + payload: Mapping[str, Any], +) -> Mapping[str, Any]: + if payload.get("schema_version") != 1: + raise ValueError("Recall manual-boundary review schema_version must be 1") + if payload.get("review_type") != RECALL_MANUAL_BOUNDARY_REVIEW_TYPE: + raise ValueError(f"review_type must be {RECALL_MANUAL_BOUNDARY_REVIEW_TYPE!r}") + identity = payload.get("identity") + if not isinstance(identity, Mapping): + raise ValueError("Recall manual-boundary review identity is missing") + for field, expected in sample_identity.items(): + if identity.get(field) != expected: + raise ValueError(f"Recall manual-boundary identity mismatch for {field}") + required = ( + "base_manual_seed_dir", + "base_manual_seed_manifest", + "base_manual_seed_manifest_sha256", + "base_combined_candidates_sha256", + "manual_seed_review", + "manual_seed_review_sha256", + "recall_manual_boundary_candidate_table", + "recall_manual_boundary_candidate_table_sha256", + ) + for field in required: + if not identity.get(field): + raise ValueError(f"Recall manual-boundary identity is missing {field}") + return identity + + +def _validate_review_rows( + payload: Mapping[str, Any], + candidate_table: pd.DataFrame, + *, + image_shape_yx: Tuple[int, int], +) -> Dict[str, Dict[str, Any]]: + if candidate_table["review_key"].astype(str).duplicated().any(): + raise ValueError("Recall manual-boundary candidate keys are not unique") + candidates = { + str(row["review_key"]): row for row in candidate_table.to_dict("records") + } + rows = payload.get("rows") + if not isinstance(rows, list): + raise ValueError("Recall manual-boundary review rows must be a list") + review_by_key: Dict[str, Dict[str, Any]] = {} + for row_number, raw_row in enumerate(rows, start=1): + if not isinstance(raw_row, Mapping): + raise ValueError(f"Recall manual-boundary row {row_number} must be an object") + key = str(raw_row.get("review_key", "")) + if key in review_by_key: + raise ValueError(f"Recall manual-boundary review contains duplicate key {key}") + candidate = candidates.get(key) + if candidate is None: + raise ValueError(f"Recall manual-boundary review contains unknown key {key}") + for field in ( + "manual_index", + "review_index", + "boundary_index", + "annotation_id", + "detector_component_id", + "detection_pass", + ): + if str(raw_row.get(field, "")) != str(candidate[field]): + raise ValueError(f"Recall manual-boundary metadata changed for {key}: {field}") + for field in ("center_x", "center_y"): + if not np.isclose( + float(raw_row.get(field, np.nan)), + float(candidate[field]), + atol=0.01, + ): + raise ValueError( + f"Recall manual-boundary coordinates changed for {key}: {field}" + ) + choice = str(raw_row.get("manual_boundary_choice", "")).strip() + if choice not in RECALL_MANUAL_BOUNDARY_CHOICES: + raise ValueError( + f"invalid or missing Recall manual-boundary choice for {key}" + ) + if choice == "unsure": + raise ValueError(f"Recall manual-boundary review remains unsure for {key}") + vertices: list[tuple[float, float]] = [] + if choice == "accept_manual_contour": + vertices = _normalize_vertices( + raw_row.get("vertices_xy"), + candidate=candidate, + image_shape_yx=image_shape_yx, + key=key, + ) + review_by_key[key] = { + "choice": choice, + "notes": _clean_text(raw_row.get("manual_boundary_notes")), + "vertices": vertices, + "candidate": candidate, + } + if set(review_by_key) != set(candidates): + raise ValueError("Recall manual-boundary rows do not match the candidate table") + return review_by_key + + +def _contour_candidate_row( + *, + source: Mapping[str, Any], + mask_path: Path, + mask_sha256: str, + destination: Path, + metrics: Mapping[str, Any], + notes: str, + review_sha256: str, + candidate_table_sha256: str, +) -> Dict[str, Any]: + review_index = int(source["review_index"]) + candidate_id = f"manual_contour_{review_index:03d}" + mask = load_candidate_mask(mask_path) + ys, xs = np.nonzero(mask.mask) + centroid_x = mask.bbox.x0 + float(xs.mean()) + centroid_y = mask.bbox.y0 + float(ys.mean()) + return { + "detector_component_id": candidate_id, + "source_annotation_id": str(source["annotation_id"]), + "display_id": f"#R{review_index:03d}", + "html_id": f"manual-contour-{review_index:03d}", + "accepted": True, + "accepted_strict": False, + "accepted_rescue": False, + "detector_score": 1.0, + "acceptance_mode": "manual_seed_reviewed_contour", + "detection_pass": RECALL_MANUAL_CONTOUR_PROFILE_NAME, + "segmentation_pass": "manual_polygon", + "center_x": int(round(centroid_x)), + "center_y": int(round(centroid_y)), + "component_centroid_x": centroid_x, + "component_centroid_y": centroid_y, + "bbox_x0": mask.bbox.x0, + "bbox_y0": mask.bbox.y0, + "bbox_x1": mask.bbox.x1, + "bbox_y1": mask.bbox.y1, + "local_area_px": int(mask.mask.sum()), + "local_equivalent_diameter_um": metrics.get("equivalent_diameter_um"), + "local_major_axis_um": metrics.get("major_axis_um"), + "local_minor_axis_um": metrics.get("minor_axis_um"), + "local_eccentricity": metrics.get("eccentricity"), + "local_solidity": metrics.get("solidity"), + "local_circularity": metrics.get("circularity"), + "local_centroid_offset_px": metrics.get("centroid_offset_px"), + "local_mean_intensity": metrics.get("mean_intensity"), + "local_max_intensity": metrics.get("max_intensity"), + "threshold_method": "manual_polygon", + "threshold": None, + "selection_mode": "reviewed_manual_polygon", + "failure_class": "manual_boundary_required", + "manual_review_index": review_index, + "manual_mask_choice": "accept_manual_contour", + "manual_notes": notes, + "boundary_warning": False, + "quality_class": "reviewed_manual_contour", + "mask_path": str(mask_path.relative_to(destination)), + "mask_source_dir": str(destination), + "reviewed_mask_sha256": mask_sha256, + "manual_boundary_review_sha256": review_sha256, + "manual_boundary_candidate_table_sha256": candidate_table_sha256, + "duplicate_suppressed": False, + } + + +def finalize_recall_manual_boundary_review( + sample_dir: Path, + base_finalize_dir: Path, + review_json: Path, + out_dir: Path, + *, + tile_shape_yx: Tuple[int, int] = (512, 512), +) -> RecallManualBoundaryFinalizeResult: + """Validate reviewed Recall contours and compose an immutable v2 delivery.""" + + review_path = Path(review_json).resolve() + payload = _read_json(review_path) + raw_identity = payload.get("identity") + if not isinstance(raw_identity, Mapping): + raise ValueError("Recall manual-boundary review identity is missing") + sample = _load_sample( + sample_dir, + overlay_dir=overlay_dir_from_identity(raw_identity), + ) + identity = _validate_review_identity(sample.review_identity, payload) + base_dir = Path(base_finalize_dir).resolve() + if base_dir != Path(str(identity["base_manual_seed_dir"])).resolve(): + raise ValueError("requested base manual-seed directory does not match review") + base_manifest_path = base_dir / "manual_seed_finalize_manifest.json" + if base_manifest_path != Path(str(identity["base_manual_seed_manifest"])).resolve(): + raise ValueError("base manual-seed manifest path does not match review") + if _file_sha256(base_manifest_path) != str( + identity["base_manual_seed_manifest_sha256"] + ): + raise ValueError("base manual-seed manifest SHA-256 mismatch") + manual_seed_review_path = Path(str(identity["manual_seed_review"])).resolve() + if _file_sha256(manual_seed_review_path) != str( + identity["manual_seed_review_sha256"] + ): + raise ValueError("manual-seed review SHA-256 mismatch") + _verify_manual_seed_delivery( + base_dir, + sample_identity=sample.review_identity, + review_sha256=str(identity["manual_seed_review_sha256"]), + ) + combined_path = base_dir / "oocyte_candidates_rescue_v1_plus_manual_seed_v1.csv" + if _file_sha256(combined_path) != str(identity["base_combined_candidates_sha256"]): + raise ValueError("base combined candidate table SHA-256 mismatch") + candidate_table_path = Path( + str(identity["recall_manual_boundary_candidate_table"]) + ).resolve() + if not candidate_table_path.is_file(): + raise FileNotFoundError( + f"Recall manual-boundary candidate table is missing: {candidate_table_path}" + ) + if _file_sha256(candidate_table_path) != str( + identity["recall_manual_boundary_candidate_table_sha256"] + ): + raise ValueError("Recall manual-boundary candidate table SHA-256 mismatch") + candidates = pd.read_csv(candidate_table_path) + review_by_key = _validate_review_rows( + payload, + candidates, + image_shape_yx=sample.image_shape_yx, + ) + base_candidates = pd.read_csv(combined_path) + base_masks = [] + normalized_base_rows = [] + for row in base_candidates.to_dict("records"): + mask_path = _resolved_mask_path(base_dir, row) + if not mask_path.is_file(): + raise FileNotFoundError(f"base combined mask is missing: {mask_path}") + persisted = load_candidate_mask(mask_path) + if persisted.image_shape_yx != sample.image_shape_yx: + raise ValueError(f"base combined mask shape mismatch: {mask_path}") + base_masks.append((str(row["detector_component_id"]), persisted)) + normalized = dict(row) + normalized["mask_path"] = str(mask_path) + normalized["mask_source_dir"] = "" + normalized_base_rows.append(normalized) + + destination = Path(out_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + manifest_path = destination / "recall_manual_boundary_finalize_manifest.json" + if manifest_path.exists(): + manifest_path.unlink() + masks_dir = destination / "reviewed_masks" + masks_dir.mkdir(parents=True, exist_ok=True) + review_sha256 = _file_sha256(review_path) + candidate_table_sha256 = _file_sha256(candidate_table_path) + decisions = [] + contour_rows = [] + overlap_rows = [] + accepted_contours = [] + expected_masks = set() + for key, item in review_by_key.items(): + source = item["candidate"] + choice = str(item["choice"]) + decision: Dict[str, Any] = { + "review_key": key, + "annotation_id": str(source["annotation_id"]), + "review_index": int(source["review_index"]), + "center_x": float(source["center_x"]), + "center_y": float(source["center_y"]), + "manual_boundary_choice": choice, + "manual_boundary_notes": item["notes"], + "accepted": choice == "accept_manual_contour", + "reviewed_mask_path": "", + "reviewed_mask_sha256": "", + "vertex_count": len(item["vertices"]), + } + if choice == "accept_manual_contour": + mask, metrics = _rasterize_polygon( + item["vertices"], + center_xy=(float(source["center_x"]), float(source["center_y"])), + image_shape_yx=sample.image_shape_yx, + pixel_size_um=sample.pixel_size_um, + key=key, + ) + for base_id, base_mask in base_masks: + pixels, contour_fraction, base_fraction = _overlap_metrics( + mask, + base_mask, + ) + if pixels: + overlap_rows.append( + { + "manual_id": key, + "base_id": base_id, + "overlap_pixel_count": pixels, + "manual_overlap_fraction": contour_fraction, + "base_overlap_fraction": base_fraction, + } + ) + for prior_id, prior_mask in accepted_contours: + pixels, contour_fraction, prior_fraction = _overlap_metrics( + mask, + prior_mask, + ) + if pixels: + overlap_rows.append( + { + "manual_id": key, + "base_id": prior_id, + "overlap_pixel_count": pixels, + "manual_overlap_fraction": contour_fraction, + "base_overlap_fraction": prior_fraction, + } + ) + if overlap_rows: + pairs = ", ".join( + f"{row['manual_id']}/{row['base_id']}={row['overlap_pixel_count']} px" + for row in overlap_rows + ) + raise ValueError(f"manual contour overlaps resolved masks: {pairs}") + review_index = int(source["review_index"]) + candidate_id = f"manual_contour_{review_index:03d}" + reviewed_path = masks_dir / f"{candidate_id}.npz" + expected_masks.add(reviewed_path.name) + metadata = { + **dict(mask.metadata), + "schema_version": 1, + "sample_id": sample.sample_id, + "candidate_id": candidate_id, + "profile_name": RECALL_MANUAL_CONTOUR_PROFILE_NAME, + "base_profile_name": sample.profile_name, + "base_profile_fingerprint": sample.profile_fingerprint, + "implementation_version": sample.implementation_version, + "reviewed_manual_seed": True, + "reviewed_manual_contour": True, + "provisional_only": False, + "source_annotation_id": str(source["annotation_id"]), + "manual_notes": item["notes"], + "manual_boundary_review_sha256": review_sha256, + "manual_boundary_candidate_table_sha256": candidate_table_sha256, + "metrics": metrics, + } + _write_reviewed_mask(reviewed_path, mask=mask, metadata=metadata) + mask_sha256 = _file_sha256(reviewed_path) + decision["reviewed_mask_path"] = str(reviewed_path) + decision["reviewed_mask_sha256"] = mask_sha256 + contour_rows.append( + _contour_candidate_row( + source=source, + mask_path=reviewed_path, + mask_sha256=mask_sha256, + destination=destination, + metrics=metrics, + notes=item["notes"], + review_sha256=review_sha256, + candidate_table_sha256=candidate_table_sha256, + ) + ) + accepted_contours.append((key, mask)) + decisions.append(decision) + for existing in masks_dir.glob("*.npz"): + if existing.name not in expected_masks: + existing.unlink() + + decisions_path = destination / "recall_manual_boundary_decisions.csv" + candidates_path = destination / "recall_manual_contour_candidates.csv" + combined_candidates_path = destination / "oocyte_candidates_reviewed_manual_seed_v2.csv" + overlap_audit_path = destination / "mask_overlap_audit.csv" + overlap_columns = ( + "manual_id", + "base_id", + "overlap_pixel_count", + "manual_overlap_fraction", + "base_overlap_fraction", + ) + _atomic_write_csv(pd.DataFrame(decisions), decisions_path) + contour_table = pd.DataFrame(contour_rows) + _atomic_write_csv(contour_table, candidates_path) + combined = pd.DataFrame([*normalized_base_rows, *contour_rows]) + if combined["detector_component_id"].astype(str).duplicated().any(): + raise ValueError("v2 combined candidates contain duplicate IDs") + _atomic_write_csv(combined, combined_candidates_path) + _atomic_write_csv( + pd.DataFrame(overlap_rows, columns=overlap_columns), + overlap_audit_path, + ) + labels = export_whole_slide_labels( + combined, + sample_dir=destination, + image_shape_yx=sample.image_shape_yx, + image_path=destination / "oocyte_labels_reviewed_manual_seed_v2.ome.tiff", + mapping_path=destination / "oocyte_labels_reviewed_manual_seed_v2_mapping.csv", + tile_shape_yx=tile_shape_yx, + ) + review_inputs = destination / "review_inputs" + copied_inputs = [ + _atomic_copy( + base_manifest_path, + review_inputs / "base_manual_seed_finalize_manifest.json", + ), + _atomic_copy( + manual_seed_review_path, + review_inputs / "manual_seed_mask_review.json", + ), + _atomic_copy( + candidate_table_path, + review_inputs / "recall_manual_boundary_candidates.csv", + ), + _atomic_copy( + review_path, + review_inputs / "recall_manual_boundary_review.json", + ), + ] + artifact_paths = [ + decisions_path, + candidates_path, + combined_candidates_path, + overlap_audit_path, + labels.image_path, + labels.mapping_path, + *masks_dir.glob("*.npz"), + *copied_inputs, + ] + manifest = { + "schema_version": 1, + "delivery_name": "reviewed_manual_seed_delta_v2", + "sample": sample.review_identity, + "review_identity": dict(identity), + "base_manual_seed_dir": str(base_dir), + "base_manual_seed_manifest": str(base_manifest_path), + "base_manual_seed_manifest_sha256": _file_sha256(base_manifest_path), + "manual_boundary_review": str(review_path), + "manual_boundary_review_sha256": review_sha256, + "manual_boundary_candidate_table": str(candidate_table_path), + "manual_boundary_candidate_table_sha256": candidate_table_sha256, + "base_label_count": len(base_candidates), + "manual_boundary_card_count": len(candidates), + "manual_added_count": len(contour_rows), + "manual_excluded_count": sum( + row["manual_boundary_choice"] == "exclude" for row in decisions + ), + "combined_label_count": labels.label_count, + "overlap_pixel_count": labels.overlap_pixel_count, + "assigned_pixel_count": labels.assigned_pixel_count, + "remaining_manual_boundary_count": 0, + "production_outputs_modified": False, + "base_manual_seed_outputs_modified": False, + "artifacts": { + str(path.relative_to(destination)): { + "path": str(path), + "sha256": _file_sha256(path), + "size_bytes": path.stat().st_size, + } + for path in artifact_paths + }, + } + _atomic_write_text( + manifest_path, + json.dumps(_json_safe(manifest), indent=2, sort_keys=True, allow_nan=False), + ) + return RecallManualBoundaryFinalizeResult( + out_dir=destination, + decisions_path=decisions_path, + candidates_path=candidates_path, + combined_candidates_path=combined_candidates_path, + overlap_audit_path=overlap_audit_path, + labels=labels, + manifest_path=manifest_path, + manual_added_count=len(contour_rows), + manual_excluded_count=int(manifest["manual_excluded_count"]), + combined_label_count=labels.label_count, + ) + + +__all__ = [ + "RECALL_MANUAL_CONTOUR_PROFILE_NAME", + "RECALL_MANUAL_BOUNDARY_CHOICES", + "RecallManualBoundaryFinalizeResult", + "finalize_recall_manual_boundary_review", +] diff --git a/aegle/oocyte/recall_manual_boundary_review.py b/aegle/oocyte/recall_manual_boundary_review.py new file mode 100644 index 0000000..b0a3f3f --- /dev/null +++ b/aegle/oocyte/recall_manual_boundary_review.py @@ -0,0 +1,374 @@ +"""Identity-bound polygon review for unresolved Recall manual-seed boundaries.""" + +from __future__ import annotations + +import html +import io +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping, Sequence + +import pandas as pd +from PIL import Image + +from .io import load_candidate_mask +from .manual_seed_finalize import _atomic_write_csv, _validate_review_identity +from .manual_seed_review import _draw_mask +from .precision_manual_boundary_review import ( + _CSS as _MANUAL_BOUNDARY_CSS, + _JS as _PRECISION_MANUAL_BOUNDARY_JS, + _card_html, + _intersects_patch, +) +from .recall_overlay import overlay_dir_from_identity +from .recall_review import ( + RecallReviewRuntime, + _atomic_save_image, + _atomic_write_text, + _file_sha256, + _json_safe, + _load_sample, + _read_json, +) + + +RECALL_MANUAL_BOUNDARY_REVIEW_SCHEMA_VERSION = 1 +RECALL_MANUAL_BOUNDARY_RENDERER_VERSION = "recall_manual_polygon_v1" +RECALL_MANUAL_BOUNDARY_REVIEW_TYPE = "oocyte_recall_manual_boundary_review" + + +@dataclass(frozen=True) +class RecallManualBoundaryReviewResult: + page_path: Path + candidates_path: Path + assets_dir: Path + card_count: int + + +def _verify_manual_seed_delivery( + base_dir: Path, + *, + sample_identity: Mapping[str, Any], + review_sha256: str, +) -> Dict[str, Any]: + root = Path(base_dir).resolve() + manifest_path = root / "manual_seed_finalize_manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError( + f"manual-seed finalize manifest does not exist: {manifest_path}" + ) + manifest = _read_json(manifest_path) + if manifest.get("delivery_name") != "reviewed_manual_seed_delta_v1": + raise ValueError("base directory is not a reviewed_manual_seed_delta_v1 delivery") + if manifest.get("sample") != sample_identity: + raise ValueError("base manual-seed sample identity does not match") + if str(manifest.get("review_json_sha256", "")) != review_sha256: + raise ValueError("base manual-seed review SHA-256 does not match") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping): + raise ValueError("base manual-seed manifest is missing artifacts") + for relative_path, record in artifacts.items(): + if not isinstance(record, Mapping): + raise ValueError(f"invalid base artifact record: {relative_path}") + path = Path(str(record.get("path", ""))).resolve() + if not path.is_file() or not path.is_relative_to(root): + raise FileNotFoundError(f"base manual-seed artifact is missing: {path}") + if path.stat().st_size != int(record.get("size_bytes", -1)): + raise ValueError(f"base artifact size mismatch: {relative_path}") + if _file_sha256(path) != str(record.get("sha256", "")): + raise ValueError(f"base artifact SHA-256 mismatch: {relative_path}") + required = ( + "manual_seed_review_decisions.csv", + "manual_seed_accepted_candidates.csv", + "oocyte_candidates_rescue_v1_plus_manual_seed_v1.csv", + "oocyte_labels_rescue_v1_plus_manual_seed_v1.ome.tiff", + "oocyte_labels_rescue_v1_plus_manual_seed_v1_mapping.csv", + ) + for name in required: + if not (root / name).is_file(): + raise FileNotFoundError(f"base manual-seed delivery is missing {name}") + return manifest + + +def _resolved_mask_path(base_dir: Path, row: Mapping[str, Any]) -> Path: + path = Path(str(row["mask_path"])) + if path.is_absolute(): + return path.resolve() + source_dir = row.get("mask_source_dir") + root = ( + Path(base_dir) + if source_dir is None or pd.isna(source_dir) or not str(source_dir).strip() + else Path(str(source_dir)) + ) + return (root / path).resolve() + + +def _render_assets( + runtime: RecallReviewRuntime, + *, + center_xy: tuple[int, int], + radius: int, + current_mask_path: Path, + resolved_candidates: pd.DataFrame, + base_dir: Path, + allowed_mask_roots: Sequence[Path], + raw_path: Path, + context_path: Path, +) -> tuple[Any, int]: + patch = runtime.source.read_patch(center_xy, radius) + raw = Image.open( + io.BytesIO(runtime.render_patch(center_xy, radius, "local")) + ).convert("RGBA") + context = raw.copy() + neighbor_count = 0 + roots = tuple(Path(root).resolve() for root in allowed_mask_roots) + for row in resolved_candidates.to_dict("records"): + if not _intersects_patch(row, patch): + continue + mask_path = _resolved_mask_path(base_dir, row) + if not mask_path.is_file() or not any( + mask_path.is_relative_to(root) for root in roots + ): + raise ValueError(f"resolved mask is outside an identity-bound delivery: {mask_path}") + mask = load_candidate_mask(mask_path) + if mask.image_shape_yx != runtime.sample.image_shape_yx: + raise ValueError(f"resolved mask image shape mismatch: {mask_path}") + context = _draw_mask( + context, + runtime._place_mask(mask, patch), + color=(0, 235, 220), + ) + neighbor_count += 1 + current = load_candidate_mask(current_mask_path) + if current.image_shape_yx != runtime.sample.image_shape_yx: + raise ValueError(f"current target mask image shape mismatch: {current_mask_path}") + context = _draw_mask( + context, + runtime._place_mask(current, patch), + color=(255, 205, 55), + ) + _atomic_save_image(raw_path, raw.convert("RGB"), format_name="WEBP") + _atomic_save_image(context_path, context.convert("RGB"), format_name="WEBP") + return patch, neighbor_count + + +def _recall_manual_boundary_javascript() -> str: + script = _PRECISION_MANUAL_BOUNDARY_JS + replacements = { + "aegle-oocyte-manual-boundary:": "aegle-oocyte-recall-manual-boundary:", + "DATA.identity.manual_boundary_candidate_table_sha256": ( + "DATA.identity.recall_manual_boundary_candidate_table_sha256" + ), + "['sample_id','candidate_table_sha256','base_precision_resolved_manifest_sha256','manual_boundary_candidate_table_sha256']": ( + "['sample_id','recall_window_geometry_sha256'," + "'base_manual_seed_manifest_sha256'," + "'recall_manual_boundary_candidate_table_sha256']" + ), + "oocyte_precision_manual_boundary_review": RECALL_MANUAL_BOUNDARY_REVIEW_TYPE, + "_precision_manual_boundary_review.json": "_recall_manual_boundary_review.json", + } + for old, new in replacements.items(): + if old not in script: + raise RuntimeError(f"manual-boundary JavaScript template changed: {old}") + script = script.replace(old, new) + return script + + +def generate_recall_manual_boundary_review( + sample_dir: Path, + manual_review_json: Path, + base_finalize_dir: Path, + out_dir: Path, + *, + patch_radius_px: int = 220, +) -> RecallManualBoundaryReviewResult: + """Generate a polygon editor for confirmed oocytes with two bad masks.""" + + if patch_radius_px < 64: + raise ValueError("manual-boundary patch radius must be at least 64 pixels") + review_path = Path(manual_review_json).resolve() + payload = _read_json(review_path) + identity = payload.get("identity") + if not isinstance(identity, Mapping): + raise ValueError("manual-seed review is missing its identity object") + overlay_dir = overlay_dir_from_identity(identity) + sample = _load_sample(sample_dir, overlay_dir=overlay_dir) + _validate_review_identity(sample.review_identity, payload) + review_sha256 = _file_sha256(review_path) + base_dir = Path(base_finalize_dir).resolve() + _verify_manual_seed_delivery( + base_dir, + sample_identity=sample.review_identity, + review_sha256=review_sha256, + ) + base_manifest_path = base_dir / "manual_seed_finalize_manifest.json" + base_manifest_sha256 = _file_sha256(base_manifest_path) + analysis_path = Path(str(identity.get("analysis_table", ""))).resolve() + if not analysis_path.is_file(): + raise FileNotFoundError(f"recall analysis table does not exist: {analysis_path}") + if _file_sha256(analysis_path) != str(identity.get("analysis_sha256", "")): + raise ValueError("recall analysis SHA-256 does not match manual review identity") + analysis = pd.read_csv(analysis_path) + analysis_by_id = analysis.set_index("annotation_id", drop=False) + review_rows = payload.get("rows") + if not isinstance(review_rows, list): + raise ValueError("manual-seed review rows must be a list") + unresolved = [] + for review_index, row in enumerate(review_rows, start=1): + if not isinstance(row, Mapping): + raise ValueError("manual-seed review row must be an object") + choice = str(row.get("manual_mask_choice", "")).strip() + notes = str(row.get("manual_notes", "")).strip() + if choice == "neither" and "needs_manual_boundary" in notes.casefold(): + unresolved.append((review_index, row)) + if not unresolved: + raise ValueError("manual-seed review has no confirmed manual-boundary queue") + + combined_path = base_dir / "oocyte_candidates_rescue_v1_plus_manual_seed_v1.csv" + resolved_candidates = pd.read_csv(combined_path) + allowed_roots = [base_dir, sample.sample_dir] + if overlay_dir is not None: + allowed_roots.append(overlay_dir) + root = Path(out_dir).resolve() + assets_dir = root / "review_assets" + assets_dir.mkdir(parents=True, exist_ok=True) + output_rows = [] + with RecallReviewRuntime(sample.sample_dir, overlay_dir=overlay_dir) as runtime: + for manual_index, (review_index, review_row) in enumerate( + unresolved, + start=1, + ): + annotation_id = str(review_row.get("annotation_id", "")) + if annotation_id not in analysis_by_id.index: + raise ValueError( + f"manual boundary annotation is absent from analysis: {annotation_id}" + ) + source = analysis_by_id.loc[annotation_id] + current_path = Path(str(source["manual_conservative_mask_path"])).resolve() + if not current_path.is_file() or not current_path.is_relative_to( + analysis_path.parent + ): + raise ValueError( + f"manual boundary current mask is outside analysis: {current_path}" + ) + center = ( + int(round(float(source["x"]))), + int(round(float(source["y"]))), + ) + raw_name = f"manual-{manual_index:03d}-raw.webp" + context_name = f"manual-{manual_index:03d}-context.webp" + patch, neighbor_count = _render_assets( + runtime, + center_xy=center, + radius=patch_radius_px, + current_mask_path=current_path, + resolved_candidates=resolved_candidates, + base_dir=base_dir, + allowed_mask_roots=allowed_roots, + raw_path=assets_dir / raw_name, + context_path=assets_dir / context_name, + ) + patch_origin_x = center[0] - patch_radius_px + patch_origin_y = center[1] - patch_radius_px + output_rows.append( + { + "manual_index": manual_index, + "review_index": review_index, + "boundary_index": manual_index, + "review_key": annotation_id, + "annotation_id": annotation_id, + "display_id": f"#R{review_index:03d}", + "detector_component_id": annotation_id, + "detection_pass": "manual_seed_review_v1", + "center_x": float(source["x"]), + "center_y": float(source["y"]), + "center_local_x": float(source["x"]) - patch_origin_x, + "center_local_y": float(source["y"]) - patch_origin_y, + "precision_notes": "", + "boundary_review_notes": str(review_row.get("manual_notes", "")), + "current_mask_path": str(current_path), + "current_mask_sha256": _file_sha256(current_path), + "expanded_mask_path": str( + Path(str(source["manual_expanded_mask_path"])).resolve() + ), + "patch_origin_x": patch_origin_x, + "patch_origin_y": patch_origin_y, + "patch_bbox_x0": patch.bbox.x0, + "patch_bbox_y0": patch.bbox.y0, + "patch_bbox_x1": patch.bbox.x1, + "patch_bbox_y1": patch.bbox.y1, + "asset_width_px": int(patch.image.shape[1]), + "asset_height_px": int(patch.image.shape[0]), + "resolved_neighbor_count": neighbor_count, + "raw_asset_name": raw_name, + "raw_asset_sha256": _file_sha256(assets_dir / raw_name), + "context_asset_name": context_name, + "context_asset_sha256": _file_sha256(assets_dir / context_name), + } + ) + expected_assets = { + str(row[field]) + for row in output_rows + for field in ("raw_asset_name", "context_asset_name") + } + for existing in assets_dir.glob("*.webp"): + if existing.name not in expected_assets: + existing.unlink() + candidates_path = root / "recall_manual_boundary_candidates.csv" + _atomic_write_csv(pd.DataFrame(output_rows), candidates_path) + page_identity: Dict[str, Any] = { + **dict(identity), + "base_manual_seed_dir": str(base_dir), + "base_manual_seed_manifest": str(base_manifest_path), + "base_manual_seed_manifest_sha256": base_manifest_sha256, + "base_combined_candidates_sha256": _file_sha256(combined_path), + "manual_seed_review": str(review_path), + "manual_seed_review_sha256": review_sha256, + "recall_manual_boundary_candidate_table": str(candidates_path), + "recall_manual_boundary_candidate_table_sha256": _file_sha256( + candidates_path + ), + "renderer_version": RECALL_MANUAL_BOUNDARY_RENDERER_VERSION, + "patch_radius_px": patch_radius_px, + } + page_payload = { + "identity": page_identity, + "rows": [_json_safe(row) for row in output_rows], + } + cards = "".join(_card_html(row) for row in output_rows) + script_payload = json.dumps(page_payload, allow_nan=False).replace("{html.escape(sample.sample_id)} Recall manual boundary
Aegle / Recall delta / exact manual boundary

{html.escape(sample.sample_id)} recall contour desk

Trace the intended outer oocyte boundary on native UCHL1. Cyan marks all 128 currently resolved masks; yellow is the rejected conservative fragment; orange is your polygon. The expanded proposal was rejected because it merged a neighbor. The exported contour remains provisional until Python validates and rasterizes it.

1. Click around the intended boundary.2. Drag handles to refine; hide masks when needed.3. Accept the contour and export JSON.
{cards}
The first vertex is yellow. Avoid cyan neighbors and follicular halo. Any edited contour returns to Unreviewed until explicitly accepted again.
''' + page_path = root / "recall_manual_boundary_review.html" + _atomic_write_text(page_path, page) + summary = { + "schema_version": RECALL_MANUAL_BOUNDARY_REVIEW_SCHEMA_VERSION, + "review_type": "oocyte_recall_manual_boundary_review_pack", + "sample": sample.review_identity, + "base_manual_seed_manifest": str(base_manifest_path), + "base_manual_seed_manifest_sha256": base_manifest_sha256, + "manual_boundary_card_count": len(output_rows), + "page": str(page_path), + "candidates": str(candidates_path), + "production_outputs_modified": False, + "base_manual_seed_outputs_modified": False, + } + _atomic_write_text( + root / "summary.json", + json.dumps(_json_safe(summary), indent=2, sort_keys=True, allow_nan=False), + ) + return RecallManualBoundaryReviewResult( + page_path=page_path, + candidates_path=candidates_path, + assets_dir=assets_dir, + card_count=len(output_rows), + ) + + +__all__ = [ + "RECALL_MANUAL_BOUNDARY_RENDERER_VERSION", + "RECALL_MANUAL_BOUNDARY_REVIEW_SCHEMA_VERSION", + "RECALL_MANUAL_BOUNDARY_REVIEW_TYPE", + "RecallManualBoundaryReviewResult", + "generate_recall_manual_boundary_review", +] diff --git a/aegle/oocyte/recall_overlay.py b/aegle/oocyte/recall_overlay.py new file mode 100644 index 0000000..d4e10f0 --- /dev/null +++ b/aegle/oocyte/recall_overlay.py @@ -0,0 +1,224 @@ +"""Validated exact-mask overlays for coverage-based Recall review.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping, Tuple + +import numpy as np +import pandas as pd + +from .io import load_candidate_mask + + +REVIEWED_OVERLAY_CANDIDATE_FILES = { + "precision_resolved_v1": "precision_resolved_candidates.csv", + "precision_resolved_v2": "precision_resolved_candidates_v2.csv", +} + + +@dataclass(frozen=True) +class RecallMaskOverlay: + candidates: pd.DataFrame + identity_fields: Dict[str, Any] + delivery_dir: Path | None + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _read_manifest(path: Path) -> Dict[str, Any]: + with Path(path).open() as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise ValueError(f"reviewed overlay manifest must be an object: {path}") + return payload + + +def _as_bool(value: Any) -> bool: + if isinstance(value, (bool, np.bool_)): + return bool(value) + return str(value).strip().casefold() in {"true", "1", "yes"} + + +def _validate_manifest_artifacts(root: Path, manifest: Mapping[str, Any]) -> None: + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping): + raise ValueError("reviewed overlay manifest is missing artifacts") + for relative_path, raw_record in artifacts.items(): + if not isinstance(raw_record, Mapping): + raise ValueError(f"invalid reviewed overlay artifact: {relative_path}") + path = Path(str(raw_record.get("path", ""))).resolve() + if not path.is_relative_to(root): + raise ValueError(f"reviewed overlay artifact is outside delivery: {path}") + if not path.is_file(): + raise FileNotFoundError(f"reviewed overlay artifact is missing: {path}") + if path.stat().st_size != int(raw_record.get("size_bytes", -1)): + raise ValueError(f"reviewed overlay artifact size mismatch: {relative_path}") + if _file_sha256(path) != str(raw_record.get("sha256", "")): + raise ValueError( + f"reviewed overlay artifact SHA-256 mismatch: {relative_path}" + ) + + +def _validate_candidates( + candidates: pd.DataFrame, + *, + root: Path, + sample_id: str, + image_shape_yx: Tuple[int, int], +) -> None: + required = { + "detector_component_id", + "review_key", + "accepted", + "detector_score", + "acceptance_mode", + "center_x", + "center_y", + "bbox_x0", + "bbox_y0", + "bbox_x1", + "bbox_y1", + "mask_path", + "reviewed_mask_sha256", + } + missing = required.difference(candidates.columns) + if missing: + raise ValueError( + f"reviewed overlay candidate table is missing columns: {sorted(missing)}" + ) + if candidates["review_key"].astype(str).duplicated().any(): + raise ValueError("reviewed overlay review_key values are not unique") + if candidates["detector_component_id"].astype(str).duplicated().any(): + raise ValueError("reviewed overlay detector_component_id values are not unique") + if not all(_as_bool(value) for value in candidates["accepted"]): + raise ValueError("reviewed overlay candidate table contains unaccepted rows") + for row in candidates.to_dict("records"): + path = Path(str(row["mask_path"])) + if not path.is_absolute(): + path = root / path + path = path.resolve() + if not path.is_relative_to(root): + raise ValueError(f"reviewed overlay mask is outside delivery: {path}") + if not path.is_file(): + raise FileNotFoundError(f"reviewed overlay mask is missing: {path}") + if _file_sha256(path) != str(row["reviewed_mask_sha256"]): + raise ValueError(f"reviewed overlay mask SHA-256 mismatch: {path}") + persisted = load_candidate_mask(path) + if persisted.image_shape_yx != image_shape_yx: + raise ValueError(f"reviewed overlay mask image shape mismatch: {path}") + if str(persisted.metadata.get("sample_id", "")) != sample_id: + raise ValueError(f"reviewed overlay mask sample mismatch: {path}") + if persisted.bbox.as_tuple() != tuple( + int(round(float(row[field]))) + for field in ("bbox_x0", "bbox_y0", "bbox_x1", "bbox_y1") + ): + raise ValueError(f"reviewed overlay mask bounding box mismatch: {path}") + + +def load_recall_mask_overlay( + detector_candidates: pd.DataFrame, + *, + sample_identity: Mapping[str, Any], + image_shape_yx: Tuple[int, int], + overlay_dir: Path | None = None, +) -> RecallMaskOverlay: + """Return automatic masks or a fully validated reviewed delivery.""" + + if overlay_dir is None: + return RecallMaskOverlay( + candidates=detector_candidates.copy(), + identity_fields={}, + delivery_dir=None, + ) + root = Path(overlay_dir).resolve() + manifest_path = root / "precision_resolved_manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError( + f"reviewed overlay manifest does not exist: {manifest_path}" + ) + manifest = _read_manifest(manifest_path) + delivery_name = str(manifest.get("delivery_name", "")) + candidate_name = REVIEWED_OVERLAY_CANDIDATE_FILES.get(delivery_name) + if candidate_name is None: + raise ValueError( + f"unsupported reviewed Recall overlay delivery: {delivery_name!r}" + ) + if manifest.get("sample") != dict(sample_identity): + raise ValueError("reviewed overlay sample identity does not match detector output") + if manifest.get("precision_complete") is not True: + raise ValueError("reviewed overlay Precision review is not complete") + if int(manifest.get("unresolved_manual_count", -1)) != 0: + raise ValueError("reviewed overlay still contains unresolved manual boundaries") + _validate_manifest_artifacts(root, manifest) + candidates_path = root / candidate_name + if not candidates_path.is_file(): + raise FileNotFoundError( + f"reviewed overlay candidate table is missing: {candidates_path}" + ) + candidates = pd.read_csv(candidates_path) + expected_count = int(manifest.get("resolved_label_count", -1)) + if len(candidates) != expected_count: + raise ValueError("reviewed overlay candidate count does not match manifest") + if "mask_source_dir" not in candidates: + candidates["mask_source_dir"] = str(root) + else: + candidates["mask_source_dir"] = str(root) + if "display_id" not in candidates: + candidates["display_id"] = [ + f"#R{index:03d}" for index in range(1, len(candidates) + 1) + ] + if "detection_pass" not in candidates: + candidates["detection_pass"] = "reviewed_precision" + _validate_candidates( + candidates, + root=root, + sample_id=str(sample_identity["sample_id"]), + image_shape_yx=image_shape_yx, + ) + identity_fields = { + "overlay_mode": "reviewed_delivery", + "overlay_delivery_name": delivery_name, + "overlay_delivery_dir": str(root), + "overlay_manifest": str(manifest_path), + "overlay_manifest_sha256": _file_sha256(manifest_path), + "overlay_candidate_table": str(candidates_path), + "overlay_candidate_table_sha256": _file_sha256(candidates_path), + "overlay_candidate_count": len(candidates), + } + return RecallMaskOverlay( + candidates=candidates, + identity_fields=identity_fields, + delivery_dir=root, + ) + + +def overlay_dir_from_identity(identity: Mapping[str, Any]) -> Path | None: + """Recover a reviewed overlay path from a bound Recall identity.""" + + mode = str(identity.get("overlay_mode", "")) + if not mode: + return None + if mode != "reviewed_delivery": + raise ValueError(f"unsupported Recall overlay mode: {mode!r}") + value = str(identity.get("overlay_delivery_dir", "")).strip() + if not value: + raise ValueError("Recall overlay identity is missing overlay_delivery_dir") + return Path(value).resolve() + + +__all__ = [ + "REVIEWED_OVERLAY_CANDIDATE_FILES", + "RecallMaskOverlay", + "load_recall_mask_overlay", + "overlay_dir_from_identity", +] diff --git a/aegle/oocyte/recall_review.py b/aegle/oocyte/recall_review.py new file mode 100644 index 0000000..18e96d5 --- /dev/null +++ b/aegle/oocyte/recall_review.py @@ -0,0 +1,1871 @@ +"""Coverage-based recall review for standalone raw-UCHL1 oocyte detection.""" + +from __future__ import annotations + +import hashlib +import io +import json +import logging +import math +import tempfile +import threading +from dataclasses import dataclass, replace as dataclass_replace +from datetime import datetime, timezone +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Dict, Iterable, Mapping, Sequence, Tuple +from urllib.parse import parse_qs, urlparse + +import matplotlib + +matplotlib.use("Agg") +import numpy as np +import pandas as pd +import tifffile +import zarr +from matplotlib import colormaps +from PIL import Image, ImageDraw +from scipy import ndimage as ndi +from skimage import feature, measure, segmentation + +from .config import DONOR13_V6 +from .io import load_candidate_mask +from .models import BoundingBox, ExtractedPatch, LocalSegmentationResult, PersistedMask +from .recall_overlay import load_recall_mask_overlay, overlay_dir_from_identity +from .recall_review_page import recall_review_page_html, review_console_page_html +from .segmentation import _segment_oocyte_patch_components + + +LOGGER = logging.getLogger(__name__) +RECALL_REVIEW_SCHEMA_VERSION = 1 +RECALL_COVERAGE_IDENTITY_VERSION = 1 +SURVEY_COVERAGE_PROFILE = "survey_v1" +DETAIL_COVERAGE_PROFILE = "detail_v1" +SURVEY_WINDOW_RADIUS_PX = 1280 +SURVEY_WINDOW_STRIDE_PX = 2304 +DETAIL_WINDOW_RADIUS_PX = 512 +DETAIL_WINDOW_STRIDE_PX = 768 +DEFAULT_WINDOW_RADIUS_PX = SURVEY_WINDOW_RADIUS_PX +DEFAULT_WINDOW_STRIDE_PX = SURVEY_WINDOW_STRIDE_PX +DEFAULT_OVERVIEW_DOWNSAMPLE = 16 +MIN_RECALL_WINDOW_RADIUS_PX = 128 +MAX_RECALL_WINDOW_RADIUS_PX = SURVEY_WINDOW_RADIUS_PX + + +@dataclass(frozen=True) +class RecallReviewBundle: + sample_id: str + sample_dir: Path + page_path: Path + overview_path: Path + metadata_path: Path + console_path: Path + window_count: int + + +@dataclass(frozen=True) +class RecallReviewSample: + sample_id: str + sample_dir: Path + source_image: Path + source_image_size_bytes: int + channel_index: int + image_shape_yx: Tuple[int, int] + pixel_size_um: float + profile_name: str + profile_fingerprint: str + implementation_version: str + candidates: pd.DataFrame + detector_candidates: pd.DataFrame + refined_candidates: pd.DataFrame + coarse_candidates: pd.DataFrame + suppressed_candidates: pd.DataFrame + overlay_delivery_dir: Path | None + review_identity: Dict[str, Any] + + +@dataclass(frozen=True) +class ProbeSegmentation: + patch: ExtractedPatch + p99: LocalSegmentationResult | None + p95: LocalSegmentationResult | None + p99_error: str | None + p95_error: str | None + + +@dataclass(frozen=True) +class ManualSeedSegmentation: + patch: ExtractedPatch + conservative: LocalSegmentationResult | None + expanded: LocalSegmentationResult | None + conservative_percentile: float | None + expanded_percentile: float | None + error: str | None + + +def _json_safe(value: Any) -> Any: + if value is None: + return None + if isinstance(value, Mapping): + return {str(key): _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + if isinstance(value, (np.bool_, bool)): + return bool(value) + if isinstance(value, (np.integer, int)): + return int(value) + if isinstance(value, (np.floating, float)): + number = float(value) + return number if np.isfinite(number) else None + if isinstance(value, Path): + return str(value) + return value if isinstance(value, str) else str(value) + + +def _read_json(path: Path) -> Dict[str, Any]: + with Path(path).open() as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise ValueError(f"expected a JSON object: {path}") + return payload + + +def _atomic_write_text(path: Path, text: str) -> None: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{destination.name}.", + suffix=".tmp", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + handle.write(text) + handle.flush() + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _atomic_save_image(path: Path, image: Image.Image, *, format_name: str) -> None: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + prefix=f".{destination.name}.", + suffix=f".{format_name.lower()}", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + image.save(temporary_path, format=format_name, quality=90, method=6) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _mask_path(sample_dir: Path, row: Mapping[str, Any]) -> Path: + raw_mask_path = Path(str(row["mask_path"])) + if raw_mask_path.is_absolute(): + return raw_mask_path + source_value = row.get("mask_source_dir") + if source_value is None or pd.isna(source_value) or not str(source_value).strip(): + source_dir = sample_dir + else: + source_dir = Path(str(source_value)) + return source_dir / raw_mask_path + + +def _load_sample( + sample_dir: Path, + *, + overlay_dir: Path | None = None, +) -> RecallReviewSample: + root = Path(sample_dir).resolve() + manifest_path = root / "run_manifest.json" + summary_path = root / "summary.json" + candidates_path = root / "html_candidates.csv" + if not manifest_path.is_file() or not summary_path.is_file(): + raise FileNotFoundError("sample directory must contain run_manifest.json and summary.json") + if not candidates_path.is_file(): + raise FileNotFoundError( + f"missing {candidates_path}; generate the combined HTML report before recall review" + ) + manifest = _read_json(manifest_path) + summary = _read_json(summary_path) + sample_id = str(manifest["sample_id"]) + source_image = Path(str(manifest["source_image"])).resolve() + if not source_image.is_file(): + raise FileNotFoundError(f"raw source image does not exist: {source_image}") + image_shape = tuple(int(value) for value in summary["image_shape_yx"]) + if len(image_shape) != 2: + raise ValueError("summary image_shape_yx must contain Y and X") + detector_candidates = pd.read_csv(candidates_path) + required = { + "detector_component_id", + "center_x", + "center_y", + "bbox_x0", + "bbox_y0", + "bbox_x1", + "bbox_y1", + "mask_path", + } + missing = required.difference(detector_candidates.columns) + if missing: + raise ValueError(f"html_candidates.csv missing columns: {sorted(missing)}") + if "mask_source_dir" not in detector_candidates: + detector_candidates["mask_source_dir"] = str(root) + if "display_id" not in detector_candidates: + detector_candidates["display_id"] = [ + f"#{index:03d}" for index in range(1, len(detector_candidates) + 1) + ] + if "detection_pass" not in detector_candidates: + detector_candidates["detection_pass"] = "baseline_v6" + if "detector_score" not in detector_candidates: + detector_candidates["detector_score"] = 0.0 + for row in detector_candidates.to_dict("records"): + path = _mask_path(root, row) + if not path.is_file(): + raise FileNotFoundError(f"candidate mask does not exist: {path}") + + refined_path = root / "candidates.csv" + coarse_path = root / "coarse_candidates.csv" + refined = pd.read_csv(refined_path) if refined_path.is_file() else pd.DataFrame() + coarse = pd.read_csv(coarse_path) if coarse_path.is_file() else pd.DataFrame() + suppressed_path = root / "combined_duplicate_suppressed.csv" + suppressed = pd.read_csv(suppressed_path) if suppressed_path.is_file() else pd.DataFrame() + if not suppressed.empty and "detector_component_id" in suppressed and not refined.empty: + suppressed = suppressed.merge( + refined, + on="detector_component_id", + how="left", + suffixes=("", "_refined"), + ) + + source_stat = source_image.stat() + candidate_sha = _file_sha256(candidates_path) + detector_identity = { + "sample_id": sample_id, + "source_image": str(source_image), + "source_image_size_bytes": int(source_stat.st_size), + # Nanosecond timestamps exceed JavaScript's exact integer range. + "source_image_mtime_ns": str(source_stat.st_mtime_ns), + "profile_name": str(manifest["profile_name"] if "profile_name" in manifest else summary["profile_name"]), + "profile_fingerprint": str(manifest["profile_fingerprint"]), + "implementation_version": str(manifest["implementation_version"]), + "candidate_table_sha256": candidate_sha, + "combined_candidate_count": int(len(detector_candidates)), + } + overlay = load_recall_mask_overlay( + detector_candidates, + sample_identity=detector_identity, + image_shape_yx=(image_shape[0], image_shape[1]), + overlay_dir=overlay_dir, + ) + review_identity = {**detector_identity, **overlay.identity_fields} + resolved_config = manifest.get("resolved_config", {}) + pixel_size_um = float(resolved_config.get("pixel_size_um", 0.5)) + return RecallReviewSample( + sample_id=sample_id, + sample_dir=root, + source_image=source_image, + source_image_size_bytes=int(source_stat.st_size), + channel_index=int(manifest["resolved_channel_index"]), + image_shape_yx=(image_shape[0], image_shape[1]), + pixel_size_um=pixel_size_um, + profile_name=str(review_identity["profile_name"]), + profile_fingerprint=str(review_identity["profile_fingerprint"]), + implementation_version=str(review_identity["implementation_version"]), + candidates=overlay.candidates, + detector_candidates=detector_candidates, + refined_candidates=refined, + coarse_candidates=coarse, + suppressed_candidates=suppressed, + overlay_delivery_dir=overlay.delivery_dir, + review_identity=review_identity, + ) + + +class _OmeChannelSource: + """Keep one tifffile-backed zarr source open for bounded channel reads.""" + + def __init__(self, path: Path, channel_index: int): + self.path = Path(path) + self.channel_index = int(channel_index) + self._tif = tifffile.TiffFile(self.path) + self._series = self._tif.series[0] + self.axes = self._series.axes + self.shape = tuple(int(value) for value in self._series.shape) + if "C" not in self.axes or "Y" not in self.axes or "X" not in self.axes: + self.close() + raise ValueError(f"expected C/Y/X axes in OME-TIFF, got {self.axes!r}") + channel_count = self.shape[self.axes.index("C")] + if not 0 <= self.channel_index < channel_count: + self.close() + raise IndexError(f"channel {self.channel_index} outside [0, {channel_count})") + for axis, size in zip(self.axes, self.shape): + if axis not in {"C", "Y", "X"} and size != 1: + self.close() + raise ValueError(f"unsupported non-singleton OME axis {axis!r}: {size}") + self.image_shape_yx = ( + self.shape[self.axes.index("Y")], + self.shape[self.axes.index("X")], + ) + self._store = self._series.aszarr() + self._array = zarr.open(self._store, mode="r") + self._lock = threading.Lock() + + def close(self) -> None: + tif = getattr(self, "_tif", None) + if tif is not None: + tif.close() + self._tif = None + + def __enter__(self) -> "_OmeChannelSource": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + def _indexer(self, y_slice: slice, x_slice: slice) -> Tuple[Any, ...]: + output = [] + for axis, size in zip(self.axes, self.shape): + if axis == "C": + output.append(self.channel_index) + elif axis == "Y": + output.append(y_slice) + elif axis == "X": + output.append(x_slice) + elif size == 1: + output.append(0) + else: + raise ValueError(f"unsupported OME axis {axis!r}") + return tuple(output) + + def read_region(self, y0: int, y1: int, x0: int, x1: int) -> np.ndarray: + image_h, image_w = self.image_shape_yx + if not (0 <= y0 < y1 <= image_h and 0 <= x0 < x1 <= image_w): + raise ValueError("region must be a positive image-bounded rectangle") + with self._lock: + region = np.asarray(self._array[self._indexer(slice(y0, y1), slice(x0, x1))]) + if region.ndim != 2: + raise ValueError(f"channel region did not resolve to Y/X: {region.shape}") + return region + + def read_patch(self, center_xy: Tuple[int, int], radius: int) -> ExtractedPatch: + if radius < 1: + raise ValueError("patch radius must be positive") + image_h, image_w = self.image_shape_yx + center_x, center_y = (int(center_xy[0]), int(center_xy[1])) + if not 0 <= center_x < image_w or not 0 <= center_y < image_h: + raise ValueError("patch center must be inside the source image") + requested_x0 = center_x - radius + requested_y0 = center_y - radius + requested_x1 = center_x + radius + 1 + requested_y1 = center_y + radius + 1 + x0, y0 = max(0, requested_x0), max(0, requested_y0) + x1, y1 = min(image_w, requested_x1), min(image_h, requested_y1) + bbox = BoundingBox(x0=x0, y0=y0, x1=x1, y1=y1) + padding = ( + y0 - requested_y0, + requested_y1 - y1, + x0 - requested_x0, + requested_x1 - x1, + ) + patch = self.read_region(y0, y1, x0, x1) + if any(padding): + top, bottom, left, right = padding + patch = np.pad(patch, ((top, bottom), (left, right)), mode="edge") + return ExtractedPatch( + image=np.asarray(patch), + bbox=bbox, + image_shape_yx=self.image_shape_yx, + padding_tblr=padding, + ) + + +def _reduce_strip(strip: np.ndarray, factor: int) -> np.ndarray: + pad_h = (-strip.shape[0]) % factor + pad_w = (-strip.shape[1]) % factor + if pad_h or pad_w: + strip = np.pad(strip, ((0, pad_h), (0, pad_w)), mode="edge") + return strip.reshape( + strip.shape[0] // factor, + factor, + strip.shape[1] // factor, + factor, + ).mean(axis=(1, 3)) + + +def _build_overview( + source: _OmeChannelSource, + *, + downsample: int, + strip_height: int = 1024, +) -> Tuple[np.ndarray, float, float]: + if downsample < 1: + raise ValueError("overview downsample must be positive") + image_h, image_w = source.image_shape_yx + output = np.zeros( + ( + math.ceil(image_h / downsample), + math.ceil(image_w / downsample), + ), + dtype=np.float32, + ) + out_y = 0 + for y0 in range(0, image_h, strip_height): + y1 = min(image_h, y0 + strip_height) + strip = source.read_region(y0, y1, 0, image_w).astype(np.float32, copy=False) + reduced = _reduce_strip(strip, downsample) + output[out_y : out_y + reduced.shape[0], : reduced.shape[1]] = reduced + out_y += reduced.shape[0] + transformed = np.log1p(np.maximum(output, 0.0)) + finite = transformed[np.isfinite(transformed)] + if finite.size: + low, high = (float(value) for value in np.percentile(finite, [1.0, 99.9])) + else: + low, high = 0.0, 1.0 + if high <= low: + high = low + 1.0 + return output, low, high + + +def _magma_image(array: np.ndarray, low: float, high: float) -> Image.Image: + transformed = np.log1p(np.maximum(np.asarray(array, dtype=np.float32), 0.0)) + normalized = np.clip((transformed - low) / max(high - low, 1e-6), 0.0, 1.0) + rgb = np.asarray(colormaps["magma"](normalized)[..., :3] * 255.0, dtype=np.uint8) + return Image.fromarray(rgb) + + +def _axis_centers(length: int, radius: int, stride: int) -> Sequence[int]: + if length < 1 or radius < 1 or stride < 1: + raise ValueError("axis length, radius, and stride must be positive") + if length <= 2 * radius + 1: + return [length // 2] + first = radius + last = length - radius - 1 + centers = list(range(first, last + 1, stride)) + if centers[-1] != last: + centers.append(last) + return centers + + +def _coverage_profile_name( + window_radius_px: int, + window_stride_px: int, + overview_downsample: int, +) -> str: + geometry = (window_radius_px, window_stride_px, overview_downsample) + if geometry == ( + SURVEY_WINDOW_RADIUS_PX, + SURVEY_WINDOW_STRIDE_PX, + DEFAULT_OVERVIEW_DOWNSAMPLE, + ): + return SURVEY_COVERAGE_PROFILE + if geometry == ( + DETAIL_WINDOW_RADIUS_PX, + DETAIL_WINDOW_STRIDE_PX, + DEFAULT_OVERVIEW_DOWNSAMPLE, + ): + return DETAIL_COVERAGE_PROFILE + return "custom" + + +def _coverage_geometry_sha256(windows: Sequence[Mapping[str, Any]]) -> str: + """Hash spatial coverage only, independent of risk-based review ordering.""" + + geometry = [] + for row in sorted( + windows, + key=lambda item: (int(item["spatial_row"]), int(item["spatial_column"])), + ): + bbox = row["bbox"] + geometry.append( + { + "bbox": { + "x0": int(bbox["x0"]), + "x1": int(bbox["x1"]), + "y0": int(bbox["y0"]), + "y1": int(bbox["y1"]), + }, + "center_x": int(row["center_x"]), + "center_y": int(row["center_y"]), + "spatial_column": int(row["spatial_column"]), + "spatial_row": int(row["spatial_row"]), + "window_id": str(row["window_id"]), + } + ) + canonical = json.dumps( + geometry, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + return hashlib.sha256(canonical).hexdigest() + + +def _coverage_identity( + windows: Sequence[Mapping[str, Any]], + *, + window_radius_px: int, + window_stride_px: int, + overview_downsample: int, +) -> Dict[str, Any]: + return { + "recall_coverage_identity_version": RECALL_COVERAGE_IDENTITY_VERSION, + "recall_coverage_profile": _coverage_profile_name( + window_radius_px, + window_stride_px, + overview_downsample, + ), + "recall_window_radius_px": int(window_radius_px), + "recall_window_stride_px": int(window_stride_px), + "recall_overview_downsample": int(overview_downsample), + "recall_window_count": len(windows), + "recall_window_geometry_sha256": _coverage_geometry_sha256(windows), + } + + +def _identity_contains_required( + actual: Mapping[str, Any], + required: Mapping[str, Any], +) -> bool: + return all(actual.get(field) == expected for field, expected in required.items()) + + +def _validate_bundle_coverage_identity(metadata: Mapping[str, Any]) -> None: + identity = metadata.get("review_identity") + windows = metadata.get("windows") + if not isinstance(identity, Mapping) or not isinstance(windows, list): + return + geometry_fields = { + "recall_coverage_identity_version", + "recall_coverage_profile", + "recall_window_radius_px", + "recall_window_stride_px", + "recall_overview_downsample", + "recall_window_count", + "recall_window_geometry_sha256", + } + present_fields = geometry_fields.intersection(identity) + if not present_fields: + return + if present_fields != geometry_fields: + missing = sorted(geometry_fields.difference(identity)) + raise ValueError(f"recall review bundle coverage identity is incomplete: {missing}") + expected = _coverage_identity( + windows, + window_radius_px=int(metadata["window_radius_px"]), + window_stride_px=int(metadata["window_stride_px"]), + overview_downsample=int(metadata["overview_downsample"]), + ) + if not _identity_contains_required(identity, expected): + raise ValueError("recall review bundle coverage identity does not match its grid") + + +def _points_in_box(table: pd.DataFrame, x_column: str, y_column: str, bbox: BoundingBox) -> int: + if table.empty or x_column not in table or y_column not in table: + return 0 + x = pd.to_numeric(table[x_column], errors="coerce").to_numpy(dtype=float) + y = pd.to_numeric(table[y_column], errors="coerce").to_numpy(dtype=float) + return int(((x >= bbox.x0) & (x < bbox.x1) & (y >= bbox.y0) & (y < bbox.y1)).sum()) + + +def _browser_candidate_rows(candidates: pd.DataFrame) -> Sequence[Dict[str, Any]]: + output = [] + for row in candidates.to_dict("records"): + output.append( + { + "detector_component_id": str(row["detector_component_id"]), + "display_id": str(row.get("display_id", row["detector_component_id"])), + "center_x": float(row["center_x"]), + "center_y": float(row["center_y"]), + "bbox": { + "x0": int(round(float(row["bbox_x0"]))), + "y0": int(round(float(row["bbox_y0"]))), + "x1": int(round(float(row["bbox_x1"]))), + "y1": int(round(float(row["bbox_y1"]))), + }, + "detector_score": float(row.get("detector_score", 0.0)), + "detection_pass": str(row.get("detection_pass", "baseline_v6")), + "resolution_source": str( + row.get( + "precision_resolution_source", + row.get("acceptance_mode", "automatic_candidate"), + ) + ), + } + ) + return output + + +def _coverage_windows( + sample: RecallReviewSample, + overview: np.ndarray, + *, + window_radius_px: int, + window_stride_px: int, + overview_downsample: int, +) -> Sequence[Dict[str, Any]]: + image_h, image_w = sample.image_shape_yx + accepted = sample.candidates + refined = sample.refined_candidates + if not refined.empty and "accepted" in refined: + accepted_flag = refined["accepted"].astype(str).str.lower().isin({"true", "1"}) + score = pd.to_numeric(refined.get("detector_score", 0.0), errors="coerce").fillna(0.0) + near_rejected = refined[(~accepted_flag) & (score >= 0.30)] + else: + near_rejected = pd.DataFrame() + spatial_rows = [] + for row_index, center_y in enumerate(_axis_centers(image_h, window_radius_px, window_stride_px)): + for column_index, center_x in enumerate(_axis_centers(image_w, window_radius_px, window_stride_px)): + bbox = BoundingBox( + x0=max(0, center_x - window_radius_px), + y0=max(0, center_y - window_radius_px), + x1=min(image_w, center_x + window_radius_px + 1), + y1=min(image_h, center_y + window_radius_px + 1), + ) + accepted_count = _points_in_box(accepted, "center_x", "center_y", bbox) + near_rejected_count = _points_in_box( + near_rejected, "center_x", "center_y", bbox + ) + coarse_count = _points_in_box( + sample.coarse_candidates, + "coarse_center_x", + "coarse_center_y", + bbox, + ) + overview_x0 = max(0, bbox.x0 // overview_downsample) + overview_y0 = max(0, bbox.y0 // overview_downsample) + overview_x1 = min(overview.shape[1], math.ceil(bbox.x1 / overview_downsample)) + overview_y1 = min(overview.shape[0], math.ceil(bbox.y1 / overview_downsample)) + overview_crop = overview[overview_y0:overview_y1, overview_x0:overview_x1] + mean_signal = float(np.mean(np.log1p(np.maximum(overview_crop, 0.0)))) + risk_score = ( + 5.0 * near_rejected_count + + 1.25 * coarse_count + + 0.5 * accepted_count + + mean_signal + ) + spatial_rows.append( + { + "window_id": f"grid-r{row_index:03d}-c{column_index:03d}", + "spatial_row": row_index, + "spatial_column": column_index, + "center_x": int(center_x), + "center_y": int(center_y), + "bbox": { + "x0": bbox.x0, + "y0": bbox.y0, + "x1": bbox.x1, + "y1": bbox.y1, + }, + "risk_score": float(risk_score), + "accepted_count": accepted_count, + "near_rejected_count": near_rejected_count, + "coarse_count": coarse_count, + "overview_log_mean": mean_signal, + } + ) + ordered = sorted( + spatial_rows, + key=lambda row: ( + -int(row["near_rejected_count"]), + -int(row["coarse_count"]), + -float(row["overview_log_mean"]), + int(row["spatial_row"]), + int(row["spatial_column"]), + ), + ) + for index, row in enumerate(ordered, start=1): + row["review_order"] = index + return ordered + + +def generate_recall_review_bundle( + sample_dir: Path, + *, + overlay_dir: Path | None = None, + window_radius_px: int = DEFAULT_WINDOW_RADIUS_PX, + window_stride_px: int = DEFAULT_WINDOW_STRIDE_PX, + overview_downsample: int = DEFAULT_OVERVIEW_DOWNSAMPLE, +) -> RecallReviewBundle: + """Generate deterministic metadata, overview, and recall-review HTML.""" + + if not MIN_RECALL_WINDOW_RADIUS_PX <= window_radius_px <= MAX_RECALL_WINDOW_RADIUS_PX: + raise ValueError( + "window radius must be between " + f"{MIN_RECALL_WINDOW_RADIUS_PX} and {MAX_RECALL_WINDOW_RADIUS_PX} px" + ) + if window_stride_px < 1 or window_stride_px > 2 * window_radius_px + 1: + raise ValueError("window stride must be positive and cannot leave coverage gaps") + if overview_downsample < 1: + raise ValueError("overview downsample must be positive") + sample = _load_sample(sample_dir, overlay_dir=overlay_dir) + assets_dir = sample.sample_dir / "recall_review" + overview_path = assets_dir / "overview.webp" + metadata_path = assets_dir / "metadata.json" + page_path = sample.sample_dir / "recall_review.html" + console_path = sample.sample_dir / "review_console.html" + LOGGER.info("Recall bundle reading raw UCHL1 overview for %s", sample.sample_id) + with _OmeChannelSource(sample.source_image, sample.channel_index) as source: + if source.image_shape_yx != sample.image_shape_yx: + raise ValueError( + f"manifest/source image shape mismatch: {sample.image_shape_yx} != {source.image_shape_yx}" + ) + overview, global_low, global_high = _build_overview( + source, + downsample=overview_downsample, + ) + overview_image = _magma_image(overview, global_low, global_high) + _atomic_save_image(overview_path, overview_image, format_name="WEBP") + windows = _coverage_windows( + sample, + overview, + window_radius_px=window_radius_px, + window_stride_px=window_stride_px, + overview_downsample=overview_downsample, + ) + coverage_identity = _coverage_identity( + windows, + window_radius_px=window_radius_px, + window_stride_px=window_stride_px, + overview_downsample=overview_downsample, + ) + review_identity = {**sample.review_identity, **coverage_identity} + metadata = { + "schema_version": RECALL_REVIEW_SCHEMA_VERSION, + "review_type": "oocyte_recall", + "generated_at": datetime.now(timezone.utc).isoformat(), + "sample_id": sample.sample_id, + "image_height": sample.image_shape_yx[0], + "image_width": sample.image_shape_yx[1], + "pixel_size_um": sample.pixel_size_um, + "channel_index": sample.channel_index, + "window_radius_px": window_radius_px, + "window_stride_px": window_stride_px, + "overview_downsample": overview_downsample, + "recall_coverage_profile": coverage_identity["recall_coverage_profile"], + "overview_width": int(overview.shape[1]), + "overview_height": int(overview.shape[0]), + "global_log_low": global_low, + "global_log_high": global_high, + "review_identity": review_identity, + "mask_overlay": { + "mode": sample.review_identity.get("overlay_mode", "automatic_candidates"), + "delivery_name": sample.review_identity.get( + "overlay_delivery_name", sample.profile_name + ), + "candidate_count": len(sample.candidates), + "candidate_table_sha256": sample.review_identity.get( + "overlay_candidate_table_sha256", + sample.review_identity["candidate_table_sha256"], + ), + "manifest_sha256": sample.review_identity.get("overlay_manifest_sha256"), + }, + "candidates": _browser_candidate_rows(sample.candidates), + "windows": windows, + } + _atomic_write_text( + metadata_path, + json.dumps(_json_safe(metadata), indent=2, sort_keys=True, allow_nan=False), + ) + _atomic_write_text(page_path, recall_review_page_html(sample.sample_id)) + _atomic_write_text( + console_path, + review_console_page_html( + sample_id=sample.sample_id, + profile_name=sample.profile_name, + candidate_count=len(sample.candidates), + window_count=len(windows), + image_shape_yx=sample.image_shape_yx, + overlay_name=str( + sample.review_identity.get("overlay_delivery_name", "automatic detector") + ), + ), + ) + LOGGER.info( + "Recall bundle completed %s: %s windows", + sample.sample_id, + len(windows), + ) + return RecallReviewBundle( + sample_id=sample.sample_id, + sample_dir=sample.sample_dir, + page_path=page_path, + overview_path=overview_path, + metadata_path=metadata_path, + console_path=console_path, + window_count=len(windows), + ) + + +def _distance_to_rows( + table: pd.DataFrame, + x: float, + y: float, + *, + x_columns: Iterable[str], + y_columns: Iterable[str], +) -> float | None: + if table.empty: + return None + x_column = next((name for name in x_columns if name in table), None) + y_column = next((name for name in y_columns if name in table), None) + if x_column is None or y_column is None: + return None + xs = pd.to_numeric(table[x_column], errors="coerce").to_numpy(dtype=float) + ys = pd.to_numeric(table[y_column], errors="coerce").to_numpy(dtype=float) + valid = np.isfinite(xs) & np.isfinite(ys) + if not valid.any(): + return None + return float(np.hypot(xs[valid] - x, ys[valid] - y).min()) + + +def classify_recall_failure( + *, + already_covered: bool, + nearest_suppressed_distance_px: float | None, + nearest_coarse_distance_px: float | None, + nearest_refined_distance_px: float | None, + suppressed_radius_px: float = 100.0, + coarse_radius_px: float = 160.0, + refined_radius_px: float = 100.0, +) -> str: + """Classify the earliest detector stage associated with a manual click.""" + + if already_covered: + return "already_covered" + if ( + nearest_suppressed_distance_px is not None + and nearest_suppressed_distance_px <= suppressed_radius_px + ): + return "dedup_error" + if nearest_refined_distance_px is not None and nearest_refined_distance_px <= refined_radius_px: + return "acceptance_miss" + if nearest_coarse_distance_px is not None and nearest_coarse_distance_px <= coarse_radius_px: + return "segmentation_miss" + return "proposal_miss" + + +def _split_touching_manual_component( + selected: LocalSegmentationResult, + smooth: np.ndarray, + center_y: int, + center_x: int, +) -> LocalSegmentationResult: + """Split a multi-lobed component and retain the basin nearest the click.""" + + distance = ndi.distance_transform_edt(selected.mask) + peaks = feature.peak_local_max( + distance, + min_distance=30, + threshold_abs=5.0, + labels=selected.mask.astype(np.uint8), + exclude_border=False, + ) + if len(peaks) < 2: + return selected + ordered = sorted( + (tuple(int(value) for value in peak) for peak in peaks), + key=lambda peak: float(np.hypot(peak[0] - center_y, peak[1] - center_x)), + ) + markers = np.zeros(selected.mask.shape, dtype=np.int32) + for label, (peak_y, peak_x) in enumerate(ordered, start=1): + markers[peak_y, peak_x] = label + basins = segmentation.watershed(-distance, markers, mask=selected.mask) + basin_areas = np.bincount(basins.ravel()) + target = np.asarray(basins == 1, dtype=np.bool_) + min_basin_area_px = 400 + other_areas = basin_areas[2:] if len(basin_areas) > 2 else np.asarray([]) + if int(target.sum()) < min_basin_area_px or not np.any( + other_areas >= min_basin_area_px + ): + return selected + + prop = measure.regionprops( + target.astype(np.uint8), + intensity_image=smooth, + )[0] + pixel_size_um = DONOR13_V6.pixel_size_um + circularity = ( + 0.0 + if prop.perimeter <= 0 + else float(4.0 * np.pi * prop.area / (prop.perimeter**2)) + ) + metrics = dataclass_replace( + selected.metrics, + selection_mode="manual_seed_watershed_component", + area_px=int(prop.area), + equivalent_diameter_um=float(prop.equivalent_diameter_area * pixel_size_um), + major_axis_um=float(prop.axis_major_length * pixel_size_um), + minor_axis_um=float(prop.axis_minor_length * pixel_size_um), + eccentricity=float(prop.eccentricity), + solidity=float(prop.solidity), + circularity=circularity, + centroid_y_px=float(prop.centroid[0]), + centroid_x_px=float(prop.centroid[1]), + centroid_offset_px=float( + np.hypot(prop.centroid[1] - center_x, prop.centroid[0] - center_y) + ), + mean_intensity=float(prop.mean_intensity), + max_intensity=float(prop.max_intensity), + ) + return LocalSegmentationResult(mask=target, metrics=metrics) + + +def _segment_manual_seed_patch( + patch: np.ndarray, + *, + annulus_floor_percentile: float, + annulus_inner_px: int | None = None, + annulus_outer_px: int | None = None, +) -> LocalSegmentationResult: + """Select the plausible thresholded component nearest the manual click.""" + + smooth, components = _segment_oocyte_patch_components( + patch, + DONOR13_V6, + annulus_inner_px=( + DONOR13_V6.local.annulus_inner_px + if annulus_inner_px is None + else annulus_inner_px + ), + annulus_outer_px=( + DONOR13_V6.local.annulus_outer_px + if annulus_outer_px is None + else annulus_outer_px + ), + annulus_floor_percentile=annulus_floor_percentile, + ) + if not components: + raise ValueError("no connected components found after thresholding") + plausible = [ + component + for component in components + if 10.0 <= component.metrics.equivalent_diameter_um <= 100.0 + and component.metrics.centroid_offset_px <= 120.0 + ] + pool = plausible or components + center_y = patch.shape[0] // 2 + center_x = patch.shape[1] // 2 + + def rank(component: LocalSegmentationResult) -> Tuple[float, float, float]: + click_distance = float( + ndi.distance_transform_edt(~component.mask)[center_y, center_x] + ) + return ( + click_distance, + float(component.metrics.centroid_offset_px), + -float(component.metrics.area_px), + ) + + selected = _split_touching_manual_component( + min(pool, key=rank), + smooth, + center_y, + center_x, + ) + if selected.metrics.selection_mode == "manual_seed_watershed_component": + return selected + metrics = dataclass_replace( + selected.metrics, + selection_mode="manual_seed_nearest_component", + ) + return LocalSegmentationResult(mask=selected.mask, metrics=metrics) + + +class RecallReviewRuntime: + """Read-only runtime used by both HTTP routes and offline analysis.""" + + def __init__( + self, + sample_dir: Path, + *, + overlay_dir: Path | None = None, + ): + sample_root = Path(sample_dir).resolve() + self.bundle_dir = sample_root / "recall_review" + self.metadata_path = self.bundle_dir / "metadata.json" + self.page_path = sample_root / "recall_review.html" + self.overview_path = self.bundle_dir / "overview.webp" + if not self.metadata_path.is_file() or not self.page_path.is_file(): + raise FileNotFoundError("recall review bundle is missing; generate it first") + self.metadata = _read_json(self.metadata_path) + _validate_bundle_coverage_identity(self.metadata) + identity = self.metadata.get("review_identity") + if not isinstance(identity, Mapping): + raise ValueError("recall review bundle is missing its review identity") + bound_overlay_dir = overlay_dir_from_identity(identity) + if overlay_dir is not None and bound_overlay_dir != Path(overlay_dir).resolve(): + raise ValueError("requested Recall overlay does not match generated bundle") + effective_overlay_dir = ( + Path(overlay_dir).resolve() if overlay_dir is not None else bound_overlay_dir + ) + self.sample = _load_sample( + sample_root, + overlay_dir=effective_overlay_dir, + ) + if not _identity_contains_required(identity, self.sample.review_identity): + raise ValueError("recall review bundle identity does not match current sample output") + self.source = _OmeChannelSource( + self.sample.source_image, + self.sample.channel_index, + ) + self._mask_cache: Dict[str, PersistedMask] = {} + + def close(self) -> None: + self.source.close() + + def __enter__(self) -> "RecallReviewRuntime": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + def _candidate_mask(self, row: Mapping[str, Any]) -> PersistedMask: + path = _mask_path(self.sample.sample_dir, row).resolve() + key = str(path) + if key not in self._mask_cache: + self._mask_cache[key] = load_candidate_mask(path) + return self._mask_cache[key] + + def _intersecting_rows(self, patch: ExtractedPatch) -> Sequence[Dict[str, Any]]: + rows = [] + for row in self.sample.candidates.to_dict("records"): + x0, y0 = float(row["bbox_x0"]), float(row["bbox_y0"]) + x1, y1 = float(row["bbox_x1"]), float(row["bbox_y1"]) + if x0 >= patch.bbox.x1 or y0 >= patch.bbox.y1 or x1 <= patch.bbox.x0 or y1 <= patch.bbox.y0: + continue + rows.append(row) + return rows + + @staticmethod + def _place_mask(mask: PersistedMask, patch: ExtractedPatch) -> np.ndarray: + placed = np.zeros(patch.image.shape, dtype=np.bool_) + x0 = max(mask.bbox.x0, patch.bbox.x0) + y0 = max(mask.bbox.y0, patch.bbox.y0) + x1 = min(mask.bbox.x1, patch.bbox.x1) + y1 = min(mask.bbox.y1, patch.bbox.y1) + if x0 >= x1 or y0 >= y1: + return placed + top, _, left, _ = patch.padding_tblr + source = mask.mask[ + y0 - mask.bbox.y0 : y1 - mask.bbox.y0, + x0 - mask.bbox.x0 : x1 - mask.bbox.x0, + ] + target_y = top + y0 - patch.bbox.y0 + target_x = left + x0 - patch.bbox.x0 + placed[target_y : target_y + source.shape[0], target_x : target_x + source.shape[1]] = source + return placed + + def render_patch(self, center_xy: Tuple[int, int], radius: int, contrast: str) -> bytes: + patch = self.source.read_patch(center_xy, radius) + transformed = np.log1p(np.maximum(patch.image.astype(np.float32), 0.0)) + finite = transformed[np.isfinite(transformed)] + if contrast == "global": + low = float(self.metadata["global_log_low"]) + high = float(self.metadata["global_log_high"]) + elif contrast == "local": + if finite.size: + low, high = (float(value) for value in np.percentile(finite, [2.0, 99.8])) + else: + low, high = 0.0, 1.0 + else: + raise ValueError("contrast must be 'local' or 'global'") + if high <= low: + high = low + 1.0 + normalized = np.clip((transformed - low) / (high - low), 0.0, 1.0) + rgb = np.asarray(colormaps["magma"](normalized)[..., :3] * 255.0, dtype=np.uint8) + output = io.BytesIO() + Image.fromarray(rgb).save(output, format="WEBP", quality=90, method=5) + return output.getvalue() + + def render_overlay(self, center_xy: Tuple[int, int], radius: int) -> bytes: + patch = self.source.read_patch(center_xy, radius) + rgba = np.zeros((*patch.image.shape, 4), dtype=np.uint8) + rows = self._intersecting_rows(patch) + label_positions = [] + for row in rows: + mask = self._candidate_mask(row) + placed = self._place_mask(mask, patch) + if not placed.any(): + continue + rgba[placed] = np.array([0, 235, 222, 42], dtype=np.uint8) + boundary = np.logical_xor(placed, ndi.binary_erosion(placed)) + boundary = ndi.binary_dilation(boundary, iterations=1) + rgba[boundary] = np.array([0, 255, 242, 245], dtype=np.uint8) + requested_x0 = center_xy[0] - radius + requested_y0 = center_xy[1] - radius + label_positions.append( + ( + int(round(float(row["center_x"]))) - requested_x0, + int(round(float(row["center_y"]))) - requested_y0, + str(row.get("display_id", row["detector_component_id"])), + ) + ) + image = Image.fromarray(rgba) + draw = ImageDraw.Draw(image) + for x, y, label in label_positions: + draw.rectangle((x + 5, y - 15, x + 49, y + 2), fill=(10, 25, 24, 210)) + draw.text((x + 8, y - 14), label, fill=(255, 255, 244, 255)) + output = io.BytesIO() + image.save(output, format="PNG", optimize=True) + return output.getvalue() + + def window_payload(self, center_xy: Tuple[int, int], radius: int) -> Dict[str, Any]: + patch = self.source.read_patch(center_xy, radius) + candidates = [] + for row in self._intersecting_rows(patch): + candidates.append( + { + "detector_component_id": str(row["detector_component_id"]), + "display_id": str(row.get("display_id", row["detector_component_id"])), + "center_x": float(row["center_x"]), + "center_y": float(row["center_y"]), + "detector_score": float(row.get("detector_score", 0.0)), + "detection_pass": str(row.get("detection_pass", "baseline_v6")), + "resolution_source": str( + row.get( + "precision_resolution_source", + row.get("acceptance_mode", "automatic_candidate"), + ) + ), + } + ) + return { + "center_x": center_xy[0], + "center_y": center_xy[1], + "radius": radius, + "bbox": { + "x0": patch.bbox.x0, + "y0": patch.bbox.y0, + "x1": patch.bbox.x1, + "y1": patch.bbox.y1, + }, + "padding_tblr": list(patch.padding_tblr), + "candidates": candidates, + } + + def _point_covered(self, x: float, y: float) -> bool: + px, py = int(round(x)), int(round(y)) + for row in self.sample.candidates.to_dict("records"): + if not ( + float(row["bbox_x0"]) <= px < float(row["bbox_x1"]) + and float(row["bbox_y0"]) <= py < float(row["bbox_y1"]) + ): + continue + mask = self._candidate_mask(row) + if bool(mask.mask[py - mask.bbox.y0, px - mask.bbox.x0]): + return True + return False + + def segment_probe(self, x: float, y: float) -> ProbeSegmentation: + radius = DONOR13_V6.local.window_radius_px + patch = self.source.read_patch((int(round(x)), int(round(y))), radius) + p99 = p95 = None + p99_error = p95_error = None + try: + p99 = _segment_manual_seed_patch( + patch.image, + annulus_floor_percentile=99.0, + ) + except (ValueError, TypeError) as exc: + p99_error = str(exc) + try: + p95 = _segment_manual_seed_patch( + patch.image, + annulus_floor_percentile=95.0, + ) + except (ValueError, TypeError) as exc: + p95_error = str(exc) + return ProbeSegmentation( + patch=patch, + p99=p99, + p95=p95, + p99_error=p99_error, + p95_error=p95_error, + ) + + def segment_manual_provisionals( + self, + x: float, + y: float, + *, + exclude_points_xy: Sequence[Tuple[float, float]] = (), + allow_shape_recovery: bool = False, + ) -> ManualSeedSegmentation: + radius = 100 + rounded_x = int(round(x)) + rounded_y = int(round(y)) + patch = self.source.read_patch((rounded_x, rounded_y), radius) + center_y = patch.image.shape[0] // 2 + center_x = patch.image.shape[1] // 2 + options = [] + errors = [] + for percentile in (95.0, 90.0, 85.0, 80.0, 75.0, 70.0, 65.0, 60.0): + try: + result = _segment_manual_seed_patch( + patch.image, + annulus_floor_percentile=percentile, + annulus_inner_px=40, + annulus_outer_px=90, + ) + except (ValueError, TypeError) as exc: + errors.append(f"P{percentile:g}: {exc}") + continue + distance_to_mask = ndi.distance_transform_edt(~result.mask) + click_distance = float(distance_to_mask[center_y, center_x]) + near_other_seed = False + for other_x, other_y in exclude_points_xy: + if float(np.hypot(other_x - x, other_y - y)) < 25.0: + continue + local_x = center_x + int(round(other_x)) - rounded_x + local_y = center_y + int(round(other_y)) - rounded_y + if not ( + 0 <= local_y < result.mask.shape[0] + and 0 <= local_x < result.mask.shape[1] + ): + continue + if float(distance_to_mask[local_y, local_x]) <= 5.0: + near_other_seed = True + break + options.append((percentile, result, click_distance, near_other_seed)) + valid = [ + option + for option in options + if option[2] <= 25.0 + and not option[3] + and option[1].metrics.centroid_offset_px <= 50.0 + and 12.0 <= option[1].metrics.equivalent_diameter_um <= 100.0 + ] + if not valid: + return ManualSeedSegmentation( + patch=patch, + conservative=None, + expanded=None, + conservative_percentile=None, + expanded_percentile=None, + error="; ".join(errors) or "no click-targeted component passed geometry gates", + ) + conservative_percentile, conservative, _, _ = valid[0] + conservative_area = max(float(conservative.metrics.area_px), 1.0) + expansion_options = [] + for percentile, result, click_distance, _ in valid: + intersection = int(np.logical_and(conservative.mask, result.mask).sum()) + conservative_overlap = intersection / conservative_area + area_ratio = float(result.metrics.area_px) / conservative_area + standard_expansion = area_ratio <= 4.0 + shape_recovery = bool( + allow_shape_recovery + and area_ratio <= 6.0 + and result.metrics.equivalent_diameter_um >= 20.0 + and result.metrics.circularity >= 0.80 + and result.metrics.solidity >= 0.90 + and result.metrics.centroid_offset_px <= 25.0 + ) + if ( + click_distance <= 25.0 + and conservative_overlap >= 0.70 + and (standard_expansion or shape_recovery) + ): + expansion_options.append((percentile, result, area_ratio)) + expanded_percentile, expanded, _ = max( + expansion_options, + key=lambda item: (float(item[1].metrics.area_px), item[0]), + ) + return ManualSeedSegmentation( + patch=patch, + conservative=conservative, + expanded=expanded, + conservative_percentile=conservative_percentile, + expanded_percentile=expanded_percentile, + error=None, + ) + + def probe(self, x: float, y: float) -> Dict[str, Any]: + image_h, image_w = self.sample.image_shape_yx + if not (0 <= x < image_w and 0 <= y < image_h): + raise ValueError("probe coordinate must be inside the source image") + nearest_accepted = _distance_to_rows( + self.sample.candidates, + x, + y, + x_columns=("component_centroid_x", "center_x"), + y_columns=("component_centroid_y", "center_y"), + ) + nearest_refined = _distance_to_rows( + self.sample.refined_candidates, + x, + y, + x_columns=("component_centroid_x", "center_x", "seed_center_x"), + y_columns=("component_centroid_y", "center_y", "seed_center_y"), + ) + nearest_coarse = _distance_to_rows( + self.sample.coarse_candidates, + x, + y, + x_columns=("coarse_center_x",), + y_columns=("coarse_center_y",), + ) + nearest_suppressed = _distance_to_rows( + self.sample.suppressed_candidates, + x, + y, + x_columns=("component_centroid_x", "center_x", "seed_center_x"), + y_columns=("component_centroid_y", "center_y", "seed_center_y"), + ) + covered = self._point_covered(x, y) + failure_class = classify_recall_failure( + already_covered=covered, + nearest_suppressed_distance_px=nearest_suppressed, + nearest_coarse_distance_px=nearest_coarse, + nearest_refined_distance_px=nearest_refined, + ) + segmentation = self.segment_probe(x, y) + provisionals = self.segment_manual_provisionals(x, y) + return { + "x": float(x), + "y": float(y), + "already_covered": covered, + "failure_class": failure_class, + "nearest_accepted_distance_px": nearest_accepted, + "nearest_refined_distance_px": nearest_refined, + "nearest_coarse_distance_px": nearest_coarse, + "nearest_suppressed_distance_px": nearest_suppressed, + "p99_metrics": None if segmentation.p99 is None else segmentation.p99.metrics.to_dict(), + "p95_metrics": None if segmentation.p95 is None else segmentation.p95.metrics.to_dict(), + "p99_error": segmentation.p99_error, + "p95_error": segmentation.p95_error, + "manual_conservative_metrics": ( + None + if provisionals.conservative is None + else provisionals.conservative.metrics.to_dict() + ), + "manual_expanded_metrics": ( + None + if provisionals.expanded is None + else provisionals.expanded.metrics.to_dict() + ), + "manual_conservative_percentile": provisionals.conservative_percentile, + "manual_expanded_percentile": provisionals.expanded_percentile, + "manual_provisional_error": provisionals.error, + } + + +def _query_number(query: Mapping[str, Sequence[str]], name: str) -> float: + values = query.get(name) + if not values or len(values) != 1: + raise ValueError(f"query parameter {name!r} is required exactly once") + try: + value = float(values[0]) + except ValueError as exc: + raise ValueError(f"query parameter {name!r} must be numeric") from exc + if not np.isfinite(value): + raise ValueError(f"query parameter {name!r} must be finite") + return value + + +def _request_geometry( + runtime: RecallReviewRuntime, + query: Mapping[str, Sequence[str]], +) -> Tuple[Tuple[int, int], int]: + x = int(round(_query_number(query, "x"))) + y = int(round(_query_number(query, "y"))) + radius = int(round(_query_number(query, "radius"))) + image_h, image_w = runtime.sample.image_shape_yx + if not 0 <= x < image_w or not 0 <= y < image_h: + raise ValueError("x and y must be inside the source image") + if not MIN_RECALL_WINDOW_RADIUS_PX <= radius <= MAX_RECALL_WINDOW_RADIUS_PX: + raise ValueError( + "radius must be between " + f"{MIN_RECALL_WINDOW_RADIUS_PX} and {MAX_RECALL_WINDOW_RADIUS_PX} px" + ) + return (x, y), radius + + +def _handler_for(runtime: RecallReviewRuntime): + class RecallReviewHandler(BaseHTTPRequestHandler): + server_version = "AegleRecallReview/1" + + def log_message(self, format_string: str, *args: Any) -> None: + LOGGER.info("Recall HTTP %s - %s", self.address_string(), format_string % args) + + def _send(self, payload: bytes, content_type: str, *, status: int = 200, head: bool = False) -> None: + try: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + self.end_headers() + if not head: + self.wfile.write(payload) + except (BrokenPipeError, ConnectionResetError): + # Browsers cancel stale patch requests while the focal window moves. + LOGGER.debug("Recall HTTP client cancelled %s", self.path) + + def _json(self, payload: Mapping[str, Any], *, status: int = 200, head: bool = False) -> None: + body = json.dumps(_json_safe(payload), allow_nan=False).encode("utf-8") + self._send(body, "application/json; charset=utf-8", status=status, head=head) + + def _handle(self, *, head: bool) -> None: + parsed = urlparse(self.path) + query = parse_qs(parsed.query, keep_blank_values=True) + try: + if parsed.path == "/health": + self._json( + { + "status": "ok", + "sample_id": runtime.sample.sample_id, + "read_only": True, + }, + head=head, + ) + return + if parsed.path in {"/", "/recall_review.html"}: + self._send( + runtime.page_path.read_bytes(), + "text/html; charset=utf-8", + head=head, + ) + return + if parsed.path == "/review_console.html": + console_page = runtime.sample.sample_dir / "review_console.html" + if not console_page.is_file(): + raise FileNotFoundError("sample review console is unavailable") + self._send( + console_page.read_bytes(), + "text/html; charset=utf-8", + head=head, + ) + return + if parsed.path == "/oocytes.html": + precision_page = runtime.sample.sample_dir / "oocytes.html" + if not precision_page.is_file(): + raise FileNotFoundError("precision review page is unavailable") + self._send( + precision_page.read_bytes(), + "text/html; charset=utf-8", + head=head, + ) + return + if parsed.path.startswith("/html_assets/"): + asset_name = Path(parsed.path).name + if ( + parsed.path != f"/html_assets/{asset_name}" + or not asset_name.endswith(".webp") + ): + raise ValueError("invalid precision-review asset path") + asset_path = runtime.sample.sample_dir / "html_assets" / asset_name + if not asset_path.is_file(): + raise FileNotFoundError("precision-review asset is unavailable") + self._send(asset_path.read_bytes(), "image/webp", head=head) + return + if parsed.path in { + "/oocyte_review_index.html", + "/oocyte_review_console.html", + "/oocyte_detection_algorithm.html", + }: + shared_path = runtime.sample.sample_dir.parent / parsed.path.lstrip("/") + if not shared_path.is_file(): + raise FileNotFoundError("shared review page is unavailable") + self._send( + shared_path.read_bytes(), + "text/html; charset=utf-8", + head=head, + ) + return + if parsed.path.startswith("/recall_analysis"): + relative_path = Path(parsed.path.lstrip("/")) + requested_path = (runtime.sample.sample_dir / relative_path).resolve() + try: + requested_path.relative_to(runtime.sample.sample_dir) + except ValueError as exc: + raise ValueError("invalid recall-analysis asset path") from exc + content_types = { + ".html": "text/html; charset=utf-8", + ".webp": "image/webp", + ".json": "application/json; charset=utf-8", + ".csv": "text/csv; charset=utf-8", + } + content_type = content_types.get(requested_path.suffix.lower()) + if content_type is None or not requested_path.is_file(): + raise FileNotFoundError("recall-analysis asset is unavailable") + self._send(requested_path.read_bytes(), content_type, head=head) + return + if parsed.path == "/recall_review/overview.webp": + self._send(runtime.overview_path.read_bytes(), "image/webp", head=head) + return + if parsed.path == "/api/metadata": + self._json(runtime.metadata, head=head) + return + if parsed.path in {"/api/patch.webp", "/api/overlay.png", "/api/window"}: + center, radius = _request_geometry(runtime, query) + if parsed.path == "/api/patch.webp": + contrast = query.get("contrast", ["local"])[0] + self._send( + runtime.render_patch(center, radius, contrast), + "image/webp", + head=head, + ) + elif parsed.path == "/api/overlay.png": + self._send(runtime.render_overlay(center, radius), "image/png", head=head) + else: + self._json(runtime.window_payload(center, radius), head=head) + return + if parsed.path == "/api/probe": + x = _query_number(query, "x") + y = _query_number(query, "y") + self._json(runtime.probe(x, y), head=head) + return + self._json({"error": "route not found"}, status=HTTPStatus.NOT_FOUND, head=head) + except (ValueError, FileNotFoundError) as exc: + self._json({"error": str(exc)}, status=HTTPStatus.BAD_REQUEST, head=head) + except Exception as exc: # pragma: no cover - defensive server boundary + LOGGER.exception("Recall review request failed") + self._json( + {"error": f"internal server error: {type(exc).__name__}"}, + status=HTTPStatus.INTERNAL_SERVER_ERROR, + head=head, + ) + + def do_GET(self) -> None: # noqa: N802 - stdlib HTTP method name + self._handle(head=False) + + def do_HEAD(self) -> None: # noqa: N802 - stdlib HTTP method name + self._handle(head=True) + + return RecallReviewHandler + + +def serve_recall_review( + sample_dir: Path, + *, + overlay_dir: Path | None = None, + host: str = "127.0.0.1", + port: int = 8767, + generate: bool = True, + window_radius_px: int = DEFAULT_WINDOW_RADIUS_PX, + window_stride_px: int = DEFAULT_WINDOW_STRIDE_PX, + overview_downsample: int = DEFAULT_OVERVIEW_DOWNSAMPLE, +) -> None: + """Generate if requested, then serve one sample until interrupted.""" + + if not 1 <= int(port) <= 65535: + raise ValueError("port must be in [1, 65535]") + if generate: + generate_recall_review_bundle( + sample_dir, + overlay_dir=overlay_dir, + window_radius_px=window_radius_px, + window_stride_px=window_stride_px, + overview_downsample=overview_downsample, + ) + with RecallReviewRuntime(sample_dir, overlay_dir=overlay_dir) as runtime: + server = ThreadingHTTPServer((host, int(port)), _handler_for(runtime)) + server.daemon_threads = True + LOGGER.info( + "Recall review serving sample=%s shape=%s channel=%s source=%s", + runtime.sample.sample_id, + runtime.sample.image_shape_yx, + runtime.sample.channel_index, + runtime.sample.source_image, + ) + LOGGER.info("Review console URL: http://%s:%s/review_console.html", host, port) + LOGGER.info("Recall review URL: http://%s:%s/recall_review.html", host, port) + LOGGER.info("Production detector outputs are read-only") + try: + server.serve_forever() + except KeyboardInterrupt: + LOGGER.info("Recall review server interrupted") + finally: + server.server_close() + + +def _validate_review_payload(sample: RecallReviewSample, payload: Mapping[str, Any]) -> None: + if int(payload.get("schema_version", -1)) != RECALL_REVIEW_SCHEMA_VERSION: + raise ValueError("unsupported recall review schema_version") + if payload.get("review_type") != "oocyte_recall": + raise ValueError("review_type must be 'oocyte_recall'") + identity = payload.get("sample") + if not isinstance(identity, Mapping): + raise ValueError("review JSON must contain a sample identity object") + for field, expected in sample.review_identity.items(): + if identity.get(field) != expected: + raise ValueError(f"review sample identity mismatch for {field}") + misses = payload.get("missing_oocytes") + if not isinstance(misses, list): + raise ValueError("review JSON missing_oocytes must be a list") + + +def _save_provisional_mask( + path: Path, + *, + result: LocalSegmentationResult, + patch: ExtractedPatch, + annotation_id: str, + percentile: float, +) -> Path: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + bounded_mask = patch.crop_to_image_bounds(result.mask).astype(np.bool_) + metadata = { + "schema_version": 1, + "annotation_id": annotation_id, + "annulus_floor_percentile": percentile, + "metrics": result.metrics.to_dict(), + "provisional_only": True, + } + np.savez_compressed( + destination, + mask=bounded_mask, + bbox_xyxy=np.asarray(patch.bbox.as_tuple(), dtype=np.int64), + image_shape_yx=np.asarray(patch.image_shape_yx, dtype=np.int64), + metadata_json=np.asarray(json.dumps(_json_safe(metadata), sort_keys=True)), + ) + return destination + + +def analyze_recall_review( + sample_dir: Path, + review_json: Path, + out_dir: Path, + *, + overlay_dir: Path | None = None, +) -> Path: + """Validate an exported review and classify every missing-oocyte click.""" + + payload = _read_json(Path(review_json)) + identity = payload.get("sample") + if not isinstance(identity, Mapping): + raise ValueError("review JSON must contain a sample identity object") + bundle_metadata_path = Path(sample_dir).resolve() / "recall_review/metadata.json" + if not bundle_metadata_path.is_file(): + raise FileNotFoundError( + f"current Recall bundle metadata is missing: {bundle_metadata_path}" + ) + bundle_metadata = _read_json(bundle_metadata_path) + bundle_identity = bundle_metadata.get("review_identity") + if not isinstance(bundle_identity, Mapping): + raise ValueError("current Recall bundle identity is missing") + for field in sorted(set(identity) | set(bundle_identity)): + if identity.get(field) != bundle_identity.get(field): + raise ValueError( + f"review sample identity mismatch for current bundle field {field}" + ) + bound_overlay_dir = overlay_dir_from_identity(bundle_identity) + if overlay_dir is not None and bound_overlay_dir != Path(overlay_dir).resolve(): + raise ValueError("requested Recall overlay does not match review JSON") + effective_overlay_dir = ( + Path(overlay_dir).resolve() if overlay_dir is not None else bound_overlay_dir + ) + sample = _load_sample(sample_dir, overlay_dir=effective_overlay_dir) + _validate_review_payload(sample, payload) + destination = Path(out_dir).resolve() + masks_dir = destination / "provisional_masks" + destination.mkdir(parents=True, exist_ok=True) + rows = [] + annotations = payload["missing_oocytes"] + centers = [ + (float(annotation["x"]), float(annotation["y"])) + for annotation in annotations + if isinstance(annotation, Mapping) + ] + with RecallReviewRuntime( + sample.sample_dir, + overlay_dir=effective_overlay_dir, + ) as runtime: + for index, annotation in enumerate(annotations, start=1): + if not isinstance(annotation, Mapping): + raise ValueError(f"missing_oocytes[{index - 1}] must be an object") + annotation_id = str(annotation.get("annotation_id", f"miss-{index:04d}")) + x = float(annotation["x"]) + y = float(annotation["y"]) + probe = runtime.probe(x, y) + provisionals = runtime.segment_manual_provisionals( + x, + y, + exclude_points_xy=tuple( + point + for point_index, point in enumerate(centers) + if point_index != index - 1 + ), + ) + conservative_path = expanded_path = "" + if provisionals.conservative is not None: + conservative_path = str( + _save_provisional_mask( + masks_dir / f"{annotation_id}_conservative.npz", + result=provisionals.conservative, + patch=provisionals.patch, + annotation_id=annotation_id, + percentile=float(provisionals.conservative_percentile), + ) + ) + if provisionals.expanded is not None: + expanded_path = str( + _save_provisional_mask( + masks_dir / f"{annotation_id}_expanded.npz", + result=provisionals.expanded, + patch=provisionals.patch, + annotation_id=annotation_id, + percentile=float(provisionals.expanded_percentile), + ) + ) + row = { + "annotation_id": annotation_id, + "window_id": str(annotation.get("window_id", "")), + "x": x, + "y": y, + "notes": str(annotation.get("notes", "")), + "failure_class": probe["failure_class"], + "already_covered": bool(probe["already_covered"]), + "nearest_accepted_distance_px": probe["nearest_accepted_distance_px"], + "nearest_refined_distance_px": probe["nearest_refined_distance_px"], + "nearest_coarse_distance_px": probe["nearest_coarse_distance_px"], + "nearest_suppressed_distance_px": probe["nearest_suppressed_distance_px"], + "manual_conservative_mask_path": conservative_path, + "manual_expanded_mask_path": expanded_path, + "p99_error": probe["p99_error"], + "p95_error": probe["p95_error"], + "manual_conservative_percentile": provisionals.conservative_percentile, + "manual_expanded_percentile": provisionals.expanded_percentile, + "manual_provisional_error": provisionals.error, + } + for prefix in ("p99", "p95"): + metrics = probe[f"{prefix}_metrics"] or {} + for field in ( + "equivalent_diameter_um", + "circularity", + "solidity", + "centroid_offset_px", + "mean_intensity", + "max_intensity", + ): + row[f"{prefix}_{field}"] = metrics.get(field) + manual_results = { + "manual_conservative": provisionals.conservative, + "manual_expanded": provisionals.expanded, + } + for prefix, result in manual_results.items(): + metrics = {} if result is None else result.metrics.to_dict() + for field in ( + "equivalent_diameter_um", + "circularity", + "solidity", + "centroid_offset_px", + "mean_intensity", + "max_intensity", + ): + row[f"{prefix}_{field}"] = metrics.get(field) + rows.append(row) + columns = [ + "annotation_id", + "window_id", + "x", + "y", + "notes", + "failure_class", + "already_covered", + "nearest_accepted_distance_px", + "nearest_refined_distance_px", + "nearest_coarse_distance_px", + "nearest_suppressed_distance_px", + "manual_conservative_mask_path", + "manual_expanded_mask_path", + "p99_error", + "p95_error", + "manual_conservative_percentile", + "manual_expanded_percentile", + "manual_provisional_error", + "p99_equivalent_diameter_um", + "p99_circularity", + "p99_solidity", + "p99_centroid_offset_px", + "p99_mean_intensity", + "p99_max_intensity", + "p95_equivalent_diameter_um", + "p95_circularity", + "p95_solidity", + "p95_centroid_offset_px", + "p95_mean_intensity", + "p95_max_intensity", + "manual_conservative_equivalent_diameter_um", + "manual_conservative_circularity", + "manual_conservative_solidity", + "manual_conservative_centroid_offset_px", + "manual_conservative_mean_intensity", + "manual_conservative_max_intensity", + "manual_expanded_equivalent_diameter_um", + "manual_expanded_circularity", + "manual_expanded_solidity", + "manual_expanded_centroid_offset_px", + "manual_expanded_mean_intensity", + "manual_expanded_max_intensity", + ] + table_path = destination / "recall_failure_analysis.csv" + pd.DataFrame(rows, columns=columns).to_csv(table_path, index=False) + summary = { + "schema_version": 1, + "sample": dict(identity), + "review_json": str(Path(review_json).resolve()), + "reviewed_window_count": len(payload.get("windows", [])), + "missing_oocyte_count": len(rows), + "failure_class_counts": ( + pd.Series([row["failure_class"] for row in rows]).value_counts().to_dict() + if rows + else {} + ), + "analysis_table": str(table_path), + "production_outputs_modified": False, + } + _atomic_write_text( + destination / "summary.json", + json.dumps(_json_safe(summary), indent=2, sort_keys=True, allow_nan=False), + ) + from .manual_seed_review import generate_manual_seed_review + + review = generate_manual_seed_review(sample.sample_dir, destination) + summary["manual_seed_review_page"] = str(review.page_path) + summary["manual_seed_review_card_count"] = review.card_count + _atomic_write_text( + destination / "summary.json", + json.dumps(_json_safe(summary), indent=2, sort_keys=True, allow_nan=False), + ) + return table_path + + +__all__ = [ + "DEFAULT_OVERVIEW_DOWNSAMPLE", + "DEFAULT_WINDOW_RADIUS_PX", + "DEFAULT_WINDOW_STRIDE_PX", + "MAX_RECALL_WINDOW_RADIUS_PX", + "MIN_RECALL_WINDOW_RADIUS_PX", + "RECALL_REVIEW_SCHEMA_VERSION", + "RecallReviewBundle", + "RecallReviewRuntime", + "analyze_recall_review", + "classify_recall_failure", + "generate_recall_review_bundle", + "serve_recall_review", +] diff --git a/aegle/oocyte/recall_review_batch.py b/aegle/oocyte/recall_review_batch.py new file mode 100644 index 0000000..a4b3973 --- /dev/null +++ b/aegle/oocyte/recall_review_batch.py @@ -0,0 +1,553 @@ +"""Batch index and one-port server for per-sample oocyte review consoles.""" + +from __future__ import annotations + +import html +import json +import logging +from contextlib import ExitStack +from dataclasses import dataclass +from datetime import datetime, timezone +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Dict, Mapping, Sequence +from urllib.parse import parse_qs, urlparse + +from .recall_review import ( + DEFAULT_OVERVIEW_DOWNSAMPLE, + DEFAULT_WINDOW_RADIUS_PX, + DEFAULT_WINDOW_STRIDE_PX, + RecallReviewRuntime, + _atomic_write_text, + _file_sha256, + _identity_contains_required, + _json_safe, + _load_sample, + _query_number, + _read_json, + _request_geometry, + generate_recall_review_bundle, +) +from .recall_review_page import recall_review_page_html, review_console_page_html +from .recall_overlay import overlay_dir_from_identity + + +LOGGER = logging.getLogger(__name__) +BATCH_REVIEW_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class BatchRecallReviewBundle: + batch_dir: Path + index_path: Path + manifest_path: Path + sample_ids: tuple[str, ...] + total_window_count: int + total_candidate_count: int + + +def _sample_ids(batch_dir: Path, requested: Sequence[str] | None) -> tuple[str, ...]: + root = Path(batch_dir).resolve() + if requested is None: + values = sorted( + path.name + for path in root.iterdir() + if path.is_dir() + and (path / "run_manifest.json").is_file() + and (path / "html_candidates.csv").is_file() + ) + else: + values = [str(value).strip() for value in requested] + if not values: + raise ValueError("batch recall review requires at least one sample") + if len(set(values)) != len(values): + raise ValueError("batch recall review sample IDs must be unique") + for sample_id in values: + if ( + not sample_id + or sample_id in {".", ".."} + or Path(sample_id).name != sample_id + or "/" in sample_id + or "\\" in sample_id + ): + raise ValueError(f"invalid batch sample ID: {sample_id!r}") + sample_dir = (root / sample_id).resolve() + try: + sample_dir.relative_to(root) + except ValueError as exc: + raise ValueError(f"sample directory is outside batch root: {sample_id}") from exc + if not sample_dir.is_dir(): + raise FileNotFoundError(f"sample directory does not exist: {sample_dir}") + return tuple(values) + + +def _batch_index_html(records: Sequence[Mapping[str, Any]]) -> str: + cards = [] + for record in records: + sample_id = html.escape(str(record["sample_id"])) + cards.append( + f""" +
+
PANEL1 / OVARY

{sample_id}

review required
+
{int(record['candidate_count'])} masks{int(record['window_count'])} windows{int(record['image_width'])} x {int(record['image_height'])} px
+

Profile {html.escape(str(record['profile_name']))}; overlay {html.escape(str(record['overlay_name']))}. Review completion requires exported decisions; recall exports are source- and overlay-identity matched.

+
Open consolePrecisionRecall
+
+ """ + ) + return f""" + + + + + Panel1 oocyte review consoles + + + +
+
Aegle / raw UCHL1 / panel1

Oocyte review consoles

Sample-specific workspaces keep precision review, whole-slide recall review, and mask finalization distinct. Start with the algorithm contract, then export both review records before any final labels or expression matrices are created.

AlgorithmPrecision batch index
+
+
Review workflow

Complete each sample in this order

Open console is navigation only. Opening a page does not create or complete any review decision.

+
+
01 / OPEN CONSOLE

Enter the sample workspace

Read the algorithm summary and choose Precision or Recall. Page visits alone are not review evidence.

+
02 / PRECISION

Inspect existing masks

Decide whether each mask is an oocyte, whether its boundary is acceptable, and whether it is duplicated. Export <sample>_oocyte_review.json when complete.

+
03 / RECALL

Find missing oocytes

Move through the whole-slide windows, mark each one Complete, Has misses, or Unsure, and click the center of every missed oocyte. Export <sample>_recall_review.json.

+
04 / RETURN

Return the review records

Place both JSON files in notes/oocytes_detection/reviews/. Recall clicks are centers, not final masks; boundary review and finalization follow.

+
+
+
{''.join(cards)}
+
Human action remains required. These consoles prepare and preserve review evidence; they do not mark candidates or windows complete automatically.
+
Serve this directory with the Aegle batch review server. Raw images and existing detector outputs remain read-only.
+
+ +""" + + +def generate_batch_recall_review_bundle( + batch_dir: Path, + *, + sample_ids: Sequence[str] | None = None, + overlay_dirs: Mapping[str, Path] | None = None, + generate_samples: bool = True, + window_radius_px: int = DEFAULT_WINDOW_RADIUS_PX, + window_stride_px: int = DEFAULT_WINDOW_STRIDE_PX, + overview_downsample: int = DEFAULT_OVERVIEW_DOWNSAMPLE, +) -> BatchRecallReviewBundle: + """Generate sample consoles plus one identity-bound batch landing page.""" + + root = Path(batch_dir).resolve() + if not root.is_dir(): + raise FileNotFoundError(root) + selected_ids = _sample_ids(root, sample_ids) + resolved_overlays = { + str(sample_id): Path(path).resolve() + for sample_id, path in (overlay_dirs or {}).items() + } + unknown_overlays = set(resolved_overlays).difference(selected_ids) + if unknown_overlays: + raise ValueError( + f"Recall overlays reference unselected samples: {sorted(unknown_overlays)}" + ) + records: list[Dict[str, Any]] = [] + for sample_id in selected_ids: + sample_dir = root / sample_id + requested_overlay = resolved_overlays.get(sample_id) + if generate_samples: + bundle = generate_recall_review_bundle( + sample_dir, + overlay_dir=requested_overlay, + window_radius_px=window_radius_px, + window_stride_px=window_stride_px, + overview_downsample=overview_downsample, + ) + metadata_path = bundle.metadata_path + console_path = bundle.console_path + else: + metadata_path = sample_dir / "recall_review/metadata.json" + console_path = sample_dir / "review_console.html" + if not metadata_path.is_file() or not (sample_dir / "recall_review.html").is_file(): + raise FileNotFoundError( + f"recall review bundle is missing for sample {sample_id}" + ) + metadata = _read_json(metadata_path) + identity = metadata.get("review_identity") + if not isinstance(identity, Mapping): + raise ValueError(f"recall metadata identity is invalid for {sample_id}") + bound_overlay = overlay_dir_from_identity(identity) + if requested_overlay is not None and bound_overlay != requested_overlay: + raise ValueError(f"Recall overlay identity mismatch for sample {sample_id}") + sample = _load_sample(sample_dir, overlay_dir=bound_overlay) + if not _identity_contains_required(identity, sample.review_identity): + raise ValueError( + f"recall review bundle identity mismatch for sample {sample_id}" + ) + windows = metadata.get("windows") + if not isinstance(windows, list): + raise ValueError(f"recall metadata windows are invalid for {sample_id}") + _atomic_write_text( + sample_dir / "recall_review.html", + recall_review_page_html(sample.sample_id), + ) + _atomic_write_text( + console_path, + review_console_page_html( + sample_id=sample.sample_id, + profile_name=sample.profile_name, + candidate_count=len(sample.candidates), + window_count=len(windows), + image_shape_yx=sample.image_shape_yx, + overlay_name=str( + sample.review_identity.get( + "overlay_delivery_name", "automatic detector" + ) + ), + ), + ) + records.append( + { + "sample_id": sample.sample_id, + "sample_dir": str(sample.sample_dir), + "profile_name": sample.profile_name, + "profile_fingerprint": sample.profile_fingerprint, + "implementation_version": sample.implementation_version, + "image_height": sample.image_shape_yx[0], + "image_width": sample.image_shape_yx[1], + "candidate_count": len(sample.candidates), + "overlay_name": str( + sample.review_identity.get( + "overlay_delivery_name", "automatic detector" + ) + ), + "overlay_manifest_sha256": sample.review_identity.get( + "overlay_manifest_sha256" + ), + "window_count": len(windows), + "metadata_path": str(metadata_path), + "metadata_sha256": _file_sha256(metadata_path), + "console_path": str(console_path), + "review_status": "requires_export", + } + ) + + index_path = root / "oocyte_review_console.html" + manifest_path = root / "oocyte_review_console_manifest.json" + _atomic_write_text(index_path, _batch_index_html(records)) + manifest = { + "schema_version": BATCH_REVIEW_SCHEMA_VERSION, + "review_type": "oocyte_batch_review_console", + "generated_at": datetime.now(timezone.utc).isoformat(), + "batch_dir": str(root), + "sample_count": len(records), + "total_candidate_count": sum(int(row["candidate_count"]) for row in records), + "total_window_count": sum(int(row["window_count"]) for row in records), + "samples": records, + } + _atomic_write_text( + manifest_path, + json.dumps(_json_safe(manifest), indent=2, sort_keys=True, allow_nan=False), + ) + return BatchRecallReviewBundle( + batch_dir=root, + index_path=index_path, + manifest_path=manifest_path, + sample_ids=selected_ids, + total_window_count=int(manifest["total_window_count"]), + total_candidate_count=int(manifest["total_candidate_count"]), + ) + + +def _batch_handler_for( + runtimes: Mapping[str, RecallReviewRuntime], + bundle: BatchRecallReviewBundle, +): + class BatchRecallReviewHandler(BaseHTTPRequestHandler): + server_version = "AegleBatchRecallReview/1" + + def log_message(self, format_string: str, *args: Any) -> None: + LOGGER.info( + "Batch recall HTTP %s - %s", + self.address_string(), + format_string % args, + ) + + def _send( + self, + payload: bytes, + content_type: str, + *, + status: int = 200, + head: bool = False, + ) -> None: + try: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + self.end_headers() + if not head: + self.wfile.write(payload) + except (BrokenPipeError, ConnectionResetError): + LOGGER.debug("Batch recall HTTP client cancelled %s", self.path) + + def _json( + self, + payload: Mapping[str, Any], + *, + status: int = 200, + head: bool = False, + ) -> None: + body = json.dumps(_json_safe(payload), allow_nan=False).encode("utf-8") + self._send( + body, + "application/json; charset=utf-8", + status=status, + head=head, + ) + + def _redirect(self, location: str) -> None: + self.send_response(HTTPStatus.PERMANENT_REDIRECT) + self.send_header("Location", location) + self.send_header("Content-Length", "0") + self.send_header("Cache-Control", "no-store") + self.end_headers() + + def _shared(self, route: str, *, head: bool) -> bool: + if route in {"/", "/oocyte_review_console.html"}: + self._send( + bundle.index_path.read_bytes(), + "text/html; charset=utf-8", + head=head, + ) + return True + if route == "/health": + self._json( + { + "status": "ok", + "read_only": True, + "sample_ids": list(bundle.sample_ids), + }, + head=head, + ) + return True + if route in { + "/oocyte_review_index.html", + "/oocyte_detection_algorithm.html", + }: + path = bundle.batch_dir / route.lstrip("/") + if not path.is_file(): + raise FileNotFoundError("shared review page is unavailable") + self._send(path.read_bytes(), "text/html; charset=utf-8", head=head) + return True + return False + + def _sample_route( + self, + runtime: RecallReviewRuntime, + route: str, + query: Mapping[str, Sequence[str]], + *, + head: bool, + ) -> bool: + if route == "/health": + self._json( + { + "status": "ok", + "sample_id": runtime.sample.sample_id, + "read_only": True, + }, + head=head, + ) + return True + if route in {"/", "/review_console.html"}: + page = runtime.sample.sample_dir / "review_console.html" + if not page.is_file(): + raise FileNotFoundError("sample review console is unavailable") + self._send(page.read_bytes(), "text/html; charset=utf-8", head=head) + return True + if route == "/recall_review.html": + self._send( + runtime.page_path.read_bytes(), + "text/html; charset=utf-8", + head=head, + ) + return True + if route == "/oocytes.html": + page = runtime.sample.sample_dir / "oocytes.html" + if not page.is_file(): + raise FileNotFoundError("precision review page is unavailable") + self._send(page.read_bytes(), "text/html; charset=utf-8", head=head) + return True + if route.startswith("/html_assets/"): + asset_name = Path(route).name + if route != f"/html_assets/{asset_name}" or not asset_name.endswith( + ".webp" + ): + raise ValueError("invalid precision-review asset path") + path = runtime.sample.sample_dir / "html_assets" / asset_name + if not path.is_file(): + raise FileNotFoundError("precision-review asset is unavailable") + self._send(path.read_bytes(), "image/webp", head=head) + return True + if route.startswith("/recall_analysis"): + relative_path = Path(route.lstrip("/")) + path = (runtime.sample.sample_dir / relative_path).resolve() + try: + path.relative_to(runtime.sample.sample_dir) + except ValueError as exc: + raise ValueError("invalid recall-analysis asset path") from exc + content_types = { + ".html": "text/html; charset=utf-8", + ".webp": "image/webp", + ".json": "application/json; charset=utf-8", + ".csv": "text/csv; charset=utf-8", + } + content_type = content_types.get(path.suffix.lower()) + if content_type is None or not path.is_file(): + raise FileNotFoundError("recall-analysis asset is unavailable") + self._send(path.read_bytes(), content_type, head=head) + return True + if route == "/recall_review/overview.webp": + self._send(runtime.overview_path.read_bytes(), "image/webp", head=head) + return True + if route == "/api/metadata": + self._json(runtime.metadata, head=head) + return True + if route in {"/api/patch.webp", "/api/overlay.png", "/api/window"}: + center, radius = _request_geometry(runtime, query) + if route == "/api/patch.webp": + contrast = query.get("contrast", ["local"])[0] + self._send( + runtime.render_patch(center, radius, contrast), + "image/webp", + head=head, + ) + elif route == "/api/overlay.png": + self._send( + runtime.render_overlay(center, radius), + "image/png", + head=head, + ) + else: + self._json(runtime.window_payload(center, radius), head=head) + return True + if route == "/api/probe": + x = _query_number(query, "x") + y = _query_number(query, "y") + self._json(runtime.probe(x, y), head=head) + return True + return False + + def _handle(self, *, head: bool) -> None: + parsed = urlparse(self.path) + query = parse_qs(parsed.query, keep_blank_values=True) + try: + if self._shared(parsed.path, head=head): + return + parts = parsed.path.lstrip("/").split("/", 1) + sample_id = parts[0] if parts else "" + runtime = runtimes.get(sample_id) + if runtime is None: + self._json( + {"error": "route not found"}, + status=HTTPStatus.NOT_FOUND, + head=head, + ) + return + if len(parts) == 1 and not parsed.path.endswith("/"): + self._redirect(f"/{sample_id}/") + return + route = "/" + parts[1] if len(parts) == 2 and parts[1] else "/" + if not self._sample_route(runtime, route, query, head=head): + self._json( + {"error": "route not found"}, + status=HTTPStatus.NOT_FOUND, + head=head, + ) + except (ValueError, FileNotFoundError) as exc: + self._json( + {"error": str(exc)}, + status=HTTPStatus.BAD_REQUEST, + head=head, + ) + except Exception as exc: # pragma: no cover - defensive server boundary + LOGGER.exception("Batch recall review request failed") + self._json( + {"error": f"internal server error: {type(exc).__name__}"}, + status=HTTPStatus.INTERNAL_SERVER_ERROR, + head=head, + ) + + def do_GET(self) -> None: # noqa: N802 - stdlib HTTP method name + self._handle(head=False) + + def do_HEAD(self) -> None: # noqa: N802 - stdlib HTTP method name + self._handle(head=True) + + return BatchRecallReviewHandler + + +def serve_batch_recall_review( + batch_dir: Path, + *, + sample_ids: Sequence[str] | None = None, + overlay_dirs: Mapping[str, Path] | None = None, + host: str = "127.0.0.1", + port: int = 8767, + generate: bool = True, + window_radius_px: int = DEFAULT_WINDOW_RADIUS_PX, + window_stride_px: int = DEFAULT_WINDOW_STRIDE_PX, + overview_downsample: int = DEFAULT_OVERVIEW_DOWNSAMPLE, +) -> None: + """Serve multiple identity-isolated sample consoles on one local port.""" + + if not 1 <= int(port) <= 65535: + raise ValueError("port must be in [1, 65535]") + bundle = generate_batch_recall_review_bundle( + batch_dir, + sample_ids=sample_ids, + overlay_dirs=overlay_dirs, + generate_samples=generate, + window_radius_px=window_radius_px, + window_stride_px=window_stride_px, + overview_downsample=overview_downsample, + ) + with ExitStack() as stack: + runtimes = { + sample_id: stack.enter_context( + RecallReviewRuntime(bundle.batch_dir / sample_id) + ) + for sample_id in bundle.sample_ids + } + server = ThreadingHTTPServer( + (host, int(port)), + _batch_handler_for(runtimes, bundle), + ) + server.daemon_threads = True + LOGGER.info( + "Batch review serving samples=%s candidates=%s windows=%s", + ",".join(bundle.sample_ids), + bundle.total_candidate_count, + bundle.total_window_count, + ) + LOGGER.info("Batch review URL: http://%s:%s/", host, port) + LOGGER.info("Source images and production detector outputs are read-only") + try: + server.serve_forever() + except KeyboardInterrupt: + LOGGER.info("Batch review server interrupted") + finally: + server.server_close() + + +__all__ = [ + "BATCH_REVIEW_SCHEMA_VERSION", + "BatchRecallReviewBundle", + "generate_batch_recall_review_bundle", + "serve_batch_recall_review", +] diff --git a/aegle/oocyte/recall_review_page.py b/aegle/oocyte/recall_review_page.py new file mode 100644 index 0000000..3758f93 --- /dev/null +++ b/aegle/oocyte/recall_review_page.py @@ -0,0 +1,165 @@ +"""Browser UI for coverage-based oocyte recall review.""" + +from __future__ import annotations + +import html + + +_PAGE = r''' + + + + + + __SAMPLE_ID__ oocyte recall review + + + +
+
+
Aegle / raw UCHL1 / false-negative search

__SAMPLE_ID__ recall review

Move the focal window across the slide, inspect every current exact mask, and click only oocytes that lack a satisfactory mask. Overlay source: loading. A window is evidence only after you explicitly mark it complete, containing misses, or unsure.

+
Sample consolePrecision reviewBatch index
+
+ + +
Review state is stored only in this browser until exported. Global recall requires an explicit disposition for every full-slide survey window.
+
+ + +''' + + +_CONSOLE_PAGE = r''' + + + + + + __SAMPLE_ID__ oocyte review console + + + +
+
+
Aegle / raw UCHL1 / review console
+

__SAMPLE_ID__

+

Use precision review to remove false positives and recall review to find oocytes with no satisfactory mask. These are separate biological questions and require separate exported records.

+
profile __PROFILE_NAME__overlay __OVERLAY_NAME____CANDIDATE_COUNT__ current masks__WINDOW_COUNT__ recall windowssource __IMAGE_WIDTH__ x __IMAGE_HEIGHT__ px
+
+
+
01 / Understand

Algorithm

Review the detector contract, raw-UCHL1 segmentation stages, exact-mask exports, rescue behavior, and known limitations.

Open algorithm
+
02 / Precision

Candidate review

Inspect every machine-accepted mask, reject non-oocytes, flag poor boundaries, and record duplicate groups.

Open precision review
+
03 / Recall

Coverage review

Move through the whole-slide queue, overlay exact masks, explicitly classify windows, and click missed oocyte centers.

Open recall review
+
04 / Finalize

Export evidence

Export JSON from both review sessions. Manual centers require a second boundary review before final labels or profiling change.

Return to batch index
+
+
Review status is not inferred from page visits. A sample remains incomplete until candidate decisions and required recall-window dispositions are exported and unresolved mask choices are finalized.
+
Source images and detector outputs are read-only. Browser local storage is a convenience, not the durable scientific record.
+
+ +''' + + +def recall_review_page_html(sample_id: str) -> str: + """Return the standalone recall-review page for one sample.""" + + return _PAGE.replace("__SAMPLE_ID__", html.escape(sample_id)) + + +def review_console_page_html( + *, + sample_id: str, + profile_name: str, + candidate_count: int, + window_count: int, + image_shape_yx: tuple[int, int], + overlay_name: str = "automatic detector", +) -> str: + """Return the static landing page for one sample's review tasks.""" + + image_height, image_width = image_shape_yx + replacements = { + "__SAMPLE_ID__": html.escape(sample_id), + "__PROFILE_NAME__": html.escape(profile_name), + "__OVERLAY_NAME__": html.escape(overlay_name), + "__CANDIDATE_COUNT__": str(int(candidate_count)), + "__WINDOW_COUNT__": str(int(window_count)), + "__IMAGE_WIDTH__": str(int(image_width)), + "__IMAGE_HEIGHT__": str(int(image_height)), + } + page = _CONSOLE_PAGE + for placeholder, value in replacements.items(): + page = page.replace(placeholder, value) + return page + + +__all__ = ["recall_review_page_html", "review_console_page_html"] diff --git a/aegle/oocyte/release.py b/aegle/oocyte/release.py new file mode 100644 index 0000000..67bc151 --- /dev/null +++ b/aegle/oocyte/release.py @@ -0,0 +1,1120 @@ +"""Build and validate immutable reviewed oocyte release packages.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import tempfile +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, Literal, Mapping, Sequence + +import numpy as np +import pandas as pd +import tifffile +import yaml +import zarr + +from .release_console import write_embedded_release_console + + +OOCYTE_RELEASE_SCHEMA_VERSION = 1 +OOCYTE_RELEASE_IMPLEMENTATION_VERSION = "oocyte_release_v5" +RELEASE_ROLES = {"positive", "negative_control"} +PROFILE_ARTIFACT_NAMES = ( + "oocyte_by_marker.csv", + "oocyte_metadata.csv", + "oocyte_overview.csv", + "channel_manifest.csv", +) + + +@dataclass(frozen=True) +class OocyteReleaseSample: + sample_id: str + role: Literal["positive", "negative_control"] + image_path: Path + antibodies_path: Path + final_labels_path: Path + final_mapping_path: Path + final_candidates_path: Path + profiling_dir: Path + review_exports: tuple[Path, ...] + provenance_files: tuple[Path, ...] + detector_candidates_path: Path | None = None + rescue_diagnostics_path: Path | None = None + + +@dataclass(frozen=True) +class OocyteReleaseSpec: + release_name: str + samples: tuple[OocyteReleaseSample, ...] + algorithm_document: Path | None = None + + +@dataclass(frozen=True) +class OocyteReleaseResult: + release_dir: Path + manifest_path: Path + sample_count: int + oocyte_count: int + validation: Dict[str, Any] + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _artifact_record(path: Path) -> Dict[str, Any]: + resolved = Path(path).resolve() + return { + "sha256": _file_sha256(resolved), + "size_bytes": int(resolved.stat().st_size), + } + + +def _atomic_write_text(path: Path, text: str) -> None: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{destination.name}.", + suffix=".tmp", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + handle.write(text) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + _atomic_write_text( + path, + json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n", + ) + + +def _write_csv(path: Path, table: pd.DataFrame) -> None: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + newline="", + prefix=f".{destination.name}.", + suffix=".tmp", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + table.to_csv(handle, index=False) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _concat_tables(tables: Sequence[pd.DataFrame]) -> pd.DataFrame: + populated = [table for table in tables if not table.empty] + if populated: + return pd.concat(populated, ignore_index=True) + return tables[0].iloc[0:0].copy() + + +def _read_object(path: Path) -> Dict[str, Any]: + source = Path(path) + with source.open() as handle: + if source.suffix.casefold() == ".json": + value = json.load(handle) + else: + value = yaml.safe_load(handle) + if not isinstance(value, dict): + raise ValueError(f"release spec must be an object: {source}") + return value + + +def _resolve_path(base: Path, value: Any, *, field: str) -> Path: + text = str(value or "").strip() + if not text: + raise ValueError(f"release sample is missing {field}") + path = Path(text).expanduser() + if not path.is_absolute(): + path = base / path + return path.resolve() + + +def _resolve_optional_path(base: Path, value: Any) -> Path | None: + text = str(value or "").strip() + if not text: + return None + path = Path(text).expanduser() + if not path.is_absolute(): + path = base / path + return path.resolve() + + +def _resolve_path_list(base: Path, value: Any, *, field: str) -> tuple[Path, ...]: + if value is None: + return () + if not isinstance(value, list): + raise ValueError(f"release sample {field} must be a list") + return tuple(_resolve_path(base, item, field=field) for item in value) + + +def load_oocyte_release_spec(path: Path) -> OocyteReleaseSpec: + """Load and resolve one YAML or JSON release specification.""" + + source = Path(path).resolve() + payload = _read_object(source) + release_name = str(payload.get("release_name", "")).strip() + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", release_name): + raise ValueError("release_name must be a safe non-empty identifier") + raw_samples = payload.get("samples") + if not isinstance(raw_samples, list) or not raw_samples: + raise ValueError("release spec samples must be a non-empty list") + samples = [] + seen = set() + base = source.parent + for index, raw in enumerate(raw_samples, start=1): + if not isinstance(raw, dict): + raise ValueError(f"release sample {index} must be an object") + sample_id = str(raw.get("sample_id", "")).strip() + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", sample_id): + raise ValueError(f"invalid release sample_id: {sample_id!r}") + if sample_id in seen: + raise ValueError(f"duplicate release sample_id: {sample_id}") + seen.add(sample_id) + role = str(raw.get("role", "")).strip() + if role not in RELEASE_ROLES: + raise ValueError(f"invalid role for {sample_id}: {role!r}") + samples.append( + OocyteReleaseSample( + sample_id=sample_id, + role=role, # type: ignore[arg-type] + image_path=_resolve_path(base, raw.get("image"), field="image"), + antibodies_path=_resolve_path( + base, + raw.get("antibodies"), + field="antibodies", + ), + final_labels_path=_resolve_path( + base, + raw.get("final_labels"), + field="final_labels", + ), + final_mapping_path=_resolve_path( + base, + raw.get("final_mapping"), + field="final_mapping", + ), + final_candidates_path=_resolve_path( + base, + raw.get("final_candidates"), + field="final_candidates", + ), + profiling_dir=_resolve_path( + base, + raw.get("profiling_dir"), + field="profiling_dir", + ), + review_exports=_resolve_path_list( + base, + raw.get("review_exports"), + field="review_exports", + ), + provenance_files=_resolve_path_list( + base, + raw.get("provenance_files"), + field="provenance_files", + ), + detector_candidates_path=_resolve_optional_path( + base, + raw.get("detector_candidates"), + ), + rescue_diagnostics_path=_resolve_optional_path( + base, + raw.get("rescue_diagnostics"), + ), + ) + ) + image_paths = [sample.image_path for sample in samples] + if len(set(image_paths)) != len(image_paths): + raise ValueError("release samples must not reuse a source image") + return OocyteReleaseSpec( + release_name=release_name, + samples=tuple(samples), + algorithm_document=_resolve_optional_path( + base, + payload.get("algorithm_document"), + ), + ) + + +def _require_files(paths: Iterable[Path]) -> None: + for path in paths: + if not Path(path).is_file(): + raise FileNotFoundError(path) + + +def _label_summary(path: Path, *, chunk_height: int = 1024) -> Dict[str, Any]: + labels_seen: set[int] = set() + counts = np.zeros(1, dtype=np.int64) + with tifffile.TiffFile(path) as tif: + series = tif.series[0] + axes = str(series.axes) + shape = tuple(int(value) for value in series.shape) + if axes != "YX": + raise ValueError(f"release label image must have YX axes: {path} has {axes}") + if np.dtype(series.dtype) != np.dtype(np.uint16): + raise ValueError(f"release label image must be uint16: {path}") + store = series.aszarr() + try: + array = zarr.open(store, mode="r") + height, width = shape + for y0 in range(0, height, chunk_height): + block = np.asarray(array[y0 : min(height, y0 + chunk_height), :]) + block_counts = np.bincount(block.ravel()) + if len(block_counts) > len(counts): + counts = np.pad(counts, (0, len(block_counts) - len(counts))) + counts[: len(block_counts)] += block_counts + labels_seen.update(int(value) for value in np.unique(block) if value) + finally: + close = getattr(store, "close", None) + if close is not None: + close() + positive = sorted(labels_seen) + return { + "shape_yx": [int(shape[0]), int(shape[1])], + "positive_labels": positive, + "positive_label_count": len(positive), + "assigned_pixel_count": int(counts[1:].sum()), + "label_pixel_counts": { + int(label): int(counts[label]) for label in positive + }, + } + + +def _resolve_mask_path(candidates_path: Path, row: Mapping[str, Any]) -> Path: + path = Path(str(row.get("mask_path", ""))) + if path.is_absolute(): + return path.resolve() + source_dir = str(row.get("mask_source_dir", "")).strip() + if source_dir: + return (Path(source_dir) / path).resolve() + return (Path(candidates_path).parent / path).resolve() + + +def _load_packaged_mask(path: Path) -> tuple[np.ndarray, tuple[int, ...], tuple[int, ...]]: + with np.load(path, allow_pickle=False) as archive: + required = {"mask", "bbox_xyxy", "image_shape_yx"} + missing = required.difference(archive.files) + if missing: + raise ValueError(f"release mask archive missing fields: {sorted(missing)}") + mask = np.asarray(archive["mask"], dtype=np.bool_) + bbox = tuple(int(value) for value in archive["bbox_xyxy"].tolist()) + image_shape = tuple( + int(value) for value in archive["image_shape_yx"].tolist() + ) + return mask, bbox, image_shape + + +def _validate_packaged_masks( + labels_path: Path, + final_dir: Path, + mapping: pd.DataFrame, + candidates: pd.DataFrame, + image_shape_yx: Sequence[int], +) -> None: + candidate_paths = { + str(row["detector_component_id"]): str(row["mask_path"]) + for row in candidates.to_dict("records") + } + with tifffile.TiffFile(labels_path) as tif: + store = tif.series[0].aszarr() + try: + labels = zarr.open(store, mode="r") + for row in mapping.to_dict("records"): + component_id = str(row["detector_component_id"]) + relative_mask_path = str(row["mask_path"]) + if candidate_paths.get(component_id) != relative_mask_path: + raise ValueError( + f"release mask paths differ for object: {component_id}" + ) + mask_path = (final_dir / relative_mask_path).resolve() + if not mask_path.is_relative_to(final_dir) or not mask_path.is_file(): + raise ValueError(f"release mask path is invalid: {component_id}") + mask, bbox, mask_image_shape = _load_packaged_mask(mask_path) + if len(bbox) != 4 or tuple(image_shape_yx) != mask_image_shape: + raise ValueError(f"release mask geometry is invalid: {component_id}") + x0, y0, x1, y1 = bbox + if not (0 <= x0 < x1 <= image_shape_yx[1]): + raise ValueError(f"release mask X bounds are invalid: {component_id}") + if not (0 <= y0 < y1 <= image_shape_yx[0]): + raise ValueError(f"release mask Y bounds are invalid: {component_id}") + if mask.shape != (y1 - y0, x1 - x0): + raise ValueError(f"release mask shape is invalid: {component_id}") + expected_bbox = tuple( + int(row[name]) + for name in ("bbox_x0", "bbox_y0", "bbox_x1", "bbox_y1") + ) + if bbox != expected_bbox: + raise ValueError(f"release mask bbox mismatch: {component_id}") + assigned = int(row["assigned_pixel_count"]) + if int(mask.sum()) != assigned: + raise ValueError(f"release mask pixel count mismatch: {component_id}") + label = int(row["label"]) + label_patch = np.asarray(labels[y0:y1, x0:x1]) == label + if not np.array_equal(mask, label_patch): + raise ValueError(f"release mask/label pixels differ: {component_id}") + finally: + close = getattr(store, "close", None) + if close is not None: + close() + + +def _safe_mask_name(index: int, component_id: str) -> str: + safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", component_id).strip("._") + if not safe: + safe = "object" + return f"mask-{index:04d}__{safe}.npz" + + +def _validate_profile_source(sample: OocyteReleaseSample) -> Dict[str, Any]: + manifest_path = sample.profiling_dir / "profiling_manifest.json" + _require_files([manifest_path, *(sample.profiling_dir / name for name in PROFILE_ARTIFACT_NAMES)]) + manifest = _read_object(manifest_path) + if str(manifest.get("sample_id")) != sample.sample_id: + raise ValueError(f"profiling sample mismatch for {sample.sample_id}") + image = manifest.get("source_image") + if not isinstance(image, dict): + raise ValueError(f"profiling source image identity missing for {sample.sample_id}") + stat = sample.image_path.stat() + expected_image = { + "path": str(sample.image_path), + "size_bytes": int(stat.st_size), + "mtime_ns": str(stat.st_mtime_ns), + } + for field, expected in expected_image.items(): + if str(image.get(field)) != str(expected): + raise ValueError( + f"profiling source image {field} mismatch for {sample.sample_id}" + ) + antibodies = manifest.get("antibodies") + if not isinstance(antibodies, dict): + raise ValueError(f"profiling antibody identity missing for {sample.sample_id}") + if str(antibodies.get("sha256")) != _file_sha256(sample.antibodies_path): + raise ValueError(f"profiling antibody SHA mismatch for {sample.sample_id}") + for name, record in manifest.get("artifacts", {}).items(): + path = sample.profiling_dir / str(name) + if not path.is_file() or _artifact_record(path) != { + "sha256": str(record.get("sha256")), + "size_bytes": int(record.get("size_bytes", -1)), + }: + raise ValueError(f"profiling artifact mismatch for {sample.sample_id}: {name}") + return manifest + + +def _validate_negative_control_inputs(sample: OocyteReleaseSample) -> Dict[str, Any]: + if sample.role != "negative_control": + return {} + if sample.detector_candidates_path is None or sample.rescue_diagnostics_path is None: + raise ValueError( + f"negative control {sample.sample_id} requires detector and rescue diagnostics" + ) + _require_files([sample.detector_candidates_path, sample.rescue_diagnostics_path]) + detector = pd.read_csv(sample.detector_candidates_path) + rescue = pd.read_csv(sample.rescue_diagnostics_path) + accepted = int(detector["accepted"].fillna(False).astype(bool).sum()) + rescue_accepted = int(rescue["rescue_status"].astype(str).eq("accepted").sum()) + if accepted or rescue_accepted: + raise ValueError( + f"negative control {sample.sample_id} contains accepted detector objects" + ) + return { + "baseline_refined_count": len(detector), + "baseline_accepted_count": accepted, + "rescue_evaluation_count": len(rescue), + "rescue_accepted_count": rescue_accepted, + "detector_candidates_sha256": _file_sha256(sample.detector_candidates_path), + "rescue_diagnostics_sha256": _file_sha256(sample.rescue_diagnostics_path), + } + + +def _copy_evidence( + sample: OocyteReleaseSample, + review_dir: Path, + negative_evidence: Mapping[str, Any], +) -> Dict[str, Any]: + review_dir.mkdir(parents=True, exist_ok=True) + records = [] + used_names: set[str] = set() + evidence_groups = [ + ("review_export", sample.review_exports), + ("provenance", sample.provenance_files), + ] + if sample.detector_candidates_path is not None: + evidence_groups.append( + ("negative_control_detector", (sample.detector_candidates_path,)) + ) + if sample.rescue_diagnostics_path is not None: + evidence_groups.append( + ("negative_control_rescue", (sample.rescue_diagnostics_path,)) + ) + for category, paths in evidence_groups: + for index, source in enumerate(paths, start=1): + name = source.name + if name in used_names: + name = f"{category}-{index:02d}__{name}" + used_names.add(name) + destination = review_dir / name + shutil.copy2(source, destination) + records.append( + { + "category": category, + "name": name, + "source_path": str(source), + **_artifact_record(destination), + } + ) + payload: Dict[str, Any] = { + "schema_version": OOCYTE_RELEASE_SCHEMA_VERSION, + "sample_id": sample.sample_id, + "role": sample.role, + "records": records, + } + if negative_evidence: + payload["negative_control_signoff"] = { + **dict(negative_evidence), + "biological_expectation": "no_oocytes", + "status": "passed", + } + _write_json(review_dir / "review_manifest.json", payload) + return payload + + +def _release_profile_manifest( + source_manifest: Mapping[str, Any], + *, + profile_dir: Path, + final_dir: Path, + source_manifest_path: Path, +) -> Dict[str, Any]: + payload = dict(source_manifest) + payload["release_packaged"] = True + payload["source_profiling_manifest"] = { + "path": str(source_manifest_path), + **_artifact_record(source_manifest_path), + } + payload["label_image"] = { + "path": "../final/oocyte_labels.ome.tiff", + **_artifact_record(final_dir / "oocyte_labels.ome.tiff"), + } + payload["mapping"] = { + "path": "../final/oocyte_labels.csv", + **_artifact_record(final_dir / "oocyte_labels.csv"), + } + payload["candidates"] = { + "path": "../final/oocyte_candidates.csv", + **_artifact_record(final_dir / "oocyte_candidates.csv"), + } + payload["artifacts"] = { + name: _artifact_record(profile_dir / name) for name in PROFILE_ARTIFACT_NAMES + } + return payload + + +def _batch_index_html( + release_name: str, + rows: Sequence[Mapping[str, Any]], +) -> str: + cards = "".join( + f'
  • {row["sample_id"]} ' + f'{row["role"]} · {row["oocyte_count"]} oocytes
    • ' + for row in rows + ) + return f'''{release_name}
    Aegle / raw UCHL1

    {release_name}

    Immutable reviewed masks and raw within-mask marker profiles.

      {cards}
    ''' + + +def _build_sample( + sample: OocyteReleaseSample, + destination: Path, +) -> Dict[str, Any]: + required = [ + sample.image_path, + sample.antibodies_path, + sample.final_labels_path, + sample.final_mapping_path, + sample.final_candidates_path, + *sample.review_exports, + *sample.provenance_files, + ] + _require_files(required) + source_profile_manifest = _validate_profile_source(sample) + negative_evidence = _validate_negative_control_inputs(sample) + candidates = pd.read_csv(sample.final_candidates_path) + mapping = pd.read_csv(sample.final_mapping_path) + markers = pd.read_csv(sample.profiling_dir / "oocyte_by_marker.csv") + metadata = pd.read_csv(sample.profiling_dir / "oocyte_metadata.csv") + if len(candidates) != len(mapping) or len(mapping) != len(markers): + raise ValueError(f"release row count mismatch for {sample.sample_id}") + required_columns = {"detector_component_id", "mask_path"} + for name, table in (("candidates", candidates), ("mapping", mapping)): + missing = required_columns.difference(table.columns) + if missing: + raise ValueError( + f"{sample.sample_id} {name} missing columns: {sorted(missing)}" + ) + candidate_ids = candidates["detector_component_id"].astype(str) + mapping_ids = mapping["detector_component_id"].astype(str) + if candidate_ids.duplicated().any() or mapping_ids.duplicated().any(): + raise ValueError(f"duplicate final object IDs for {sample.sample_id}") + if set(candidate_ids) != set(mapping_ids): + raise ValueError(f"candidate/mapping IDs differ for {sample.sample_id}") + expected_oocyte_ids = {f"{sample.sample_id}__{value}" for value in mapping_ids} + if set(markers.get("oocyte_id", pd.Series(dtype=str)).astype(str)) != expected_oocyte_ids: + raise ValueError(f"profiling object IDs differ for {sample.sample_id}") + label_summary = _label_summary(sample.final_labels_path) + expected_labels = set(int(value) for value in mapping["label"]) + if set(label_summary["positive_labels"]) != expected_labels: + raise ValueError(f"label/mapping IDs differ for {sample.sample_id}") + if sample.role == "positive" and not len(mapping): + raise ValueError(f"positive sample has no oocytes: {sample.sample_id}") + if sample.role == "negative_control" and len(mapping): + raise ValueError(f"negative control has positive labels: {sample.sample_id}") + + final_dir = destination / "final" + masks_dir = final_dir / "masks" + profile_dir = destination / "profiling" + review_dir = destination / "review" + masks_dir.mkdir(parents=True, exist_ok=True) + profile_dir.mkdir(parents=True, exist_ok=True) + mask_names: Dict[str, str] = {} + candidate_records = candidates.to_dict("records") + for index, row in enumerate(candidate_records, start=1): + component_id = str(row["detector_component_id"]) + source_mask = _resolve_mask_path(sample.final_candidates_path, row) + if not source_mask.is_file(): + raise FileNotFoundError(source_mask) + expected_sha = str(row.get("reviewed_mask_sha256", "")).strip() + if expected_sha and _file_sha256(source_mask) != expected_sha: + raise ValueError(f"final mask SHA mismatch: {source_mask}") + mask_name = _safe_mask_name(index, component_id) + shutil.copy2(source_mask, masks_dir / mask_name) + mask_names[component_id] = mask_name + if len(mask_names) != len(candidates): + raise ValueError(f"release mask IDs are not unique for {sample.sample_id}") + candidates = candidates.copy() + mapping = mapping.copy() + candidates["mask_path"] = [ + f"masks/{mask_names[str(value)]}" for value in candidate_ids + ] + mapping["mask_path"] = [ + f"masks/{mask_names[str(value)]}" for value in mapping_ids + ] + for table in (candidates, mapping): + if "mask_source_dir" in table: + table.drop(columns=["mask_source_dir"], inplace=True) + shutil.copy2(sample.final_labels_path, final_dir / "oocyte_labels.ome.tiff") + _write_csv(final_dir / "oocyte_labels.csv", mapping) + _write_csv(final_dir / "oocyte_candidates.csv", candidates) + + for name in PROFILE_ARTIFACT_NAMES: + source = sample.profiling_dir / name + if name == "oocyte_metadata.csv": + table = metadata.copy() + if "mask_path" in table: + table["mask_path"] = [ + f"../final/masks/{mask_names[str(value)]}" + for value in table["detector_component_id"].astype(str) + ] + _write_csv(profile_dir / name, table) + else: + shutil.copy2(source, profile_dir / name) + release_profile_manifest = _release_profile_manifest( + source_profile_manifest, + profile_dir=profile_dir, + final_dir=final_dir, + source_manifest_path=sample.profiling_dir / "profiling_manifest.json", + ) + _write_json(profile_dir / "profiling_manifest.json", release_profile_manifest) + review_manifest = _copy_evidence(sample, review_dir, negative_evidence) + console_summary = write_embedded_release_console( + sample_id=sample.sample_id, + role=sample.role, + source_image_path=sample.image_path, + antibodies_path=sample.antibodies_path, + final_dir=final_dir, + profiling_dir=profile_dir, + destination=destination / "review_console.html", + ) + + sample_artifacts = { + str(path.relative_to(destination)): _artifact_record(path) + for path in sorted(destination.rglob("*")) + if path.is_file() and path.name != "sample_release_manifest.json" + } + sample_manifest = { + "schema_version": OOCYTE_RELEASE_SCHEMA_VERSION, + "implementation_version": OOCYTE_RELEASE_IMPLEMENTATION_VERSION, + "sample_id": sample.sample_id, + "role": sample.role, + "oocyte_count": len(mapping), + "channel_count": int(source_profile_manifest.get("channel_count", -1)), + "assigned_pixel_count": label_summary["assigned_pixel_count"], + "image_shape_yx": label_summary["shape_yx"], + "source_image": { + "path": str(sample.image_path), + "size_bytes": int(sample.image_path.stat().st_size), + "mtime_ns": str(sample.image_path.stat().st_mtime_ns), + }, + "antibodies": { + "path": str(sample.antibodies_path), + **_artifact_record(sample.antibodies_path), + }, + "source_final_artifacts": { + "labels": { + "path": str(sample.final_labels_path), + **_artifact_record(sample.final_labels_path), + }, + "mapping": { + "path": str(sample.final_mapping_path), + **_artifact_record(sample.final_mapping_path), + }, + "candidates": { + "path": str(sample.final_candidates_path), + **_artifact_record(sample.final_candidates_path), + }, + }, + "review_record_count": len(review_manifest["records"]), + "embedded_console": console_summary, + "artifacts": sample_artifacts, + } + _write_json(destination / "sample_release_manifest.json", sample_manifest) + return sample_manifest + + +def _readme_text(release_name: str, rows: Sequence[Mapping[str, Any]]) -> str: + positives = sum(int(row["oocyte_count"]) for row in rows if row["role"] == "positive") + channel_counts = sorted({int(row["channel_count"]) for row in rows}) + channels = str(channel_counts[0]) if len(channel_counts) == 1 else str(channel_counts) + return f"""# {release_name} + +Reviewed raw-UCHL1 oocyte segmentation release for panel1. + +- Positive oocytes: {positives} +- Samples: {len(rows)} +- Measurement: raw within-mask mean fluorescence +- Channels: {channels}, including DAPI for acquisition traceability +- DeepCell dependency: none + +`batch_oocyte_by_marker.csv` and `batch_oocyte_metadata.csv` are the cohort-level tables. Each sample directory contains exact masks, a whole-slide label image, mapping, candidates, profiling outputs, review evidence, and a checksum manifest. Raw OME-TIFFs are referenced by immutable file identity and are not copied into this release. + +Run `python src/run_oocyte_release.py validate --release-dir ` from the Aegle repository to verify hashes and cross-file invariants. +""" + + +def build_oocyte_release(spec_path: Path, out_dir: Path) -> OocyteReleaseResult: + """Build one atomic immutable release and validate it before publication.""" + + spec_source = Path(spec_path).resolve() + spec = load_oocyte_release_spec(spec_source) + destination = Path(out_dir).resolve() + if destination.exists(): + raise FileExistsError(f"release output is immutable and already exists: {destination}") + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = Path( + tempfile.mkdtemp(prefix=f".{destination.name}.tmp-", dir=destination.parent) + ) + try: + packaged_spec = temporary / f"release_spec{spec_source.suffix.casefold()}" + shutil.copy2(spec_source, packaged_spec) + sample_rows = [] + for sample in spec.samples: + sample_destination = temporary / "samples" / sample.sample_id + manifest = _build_sample(sample, sample_destination) + sample_rows.append( + { + "sample_id": sample.sample_id, + "role": sample.role, + "oocyte_count": int(manifest["oocyte_count"]), + "channel_count": int(manifest["channel_count"]), + "assigned_pixel_count": int(manifest["assigned_pixel_count"]), + "source_image": str(sample.image_path), + "sample_manifest_sha256": _file_sha256( + sample_destination / "sample_release_manifest.json" + ), + } + ) + marker_tables = [ + pd.read_csv(temporary / "samples" / row["sample_id"] / "profiling" / "oocyte_by_marker.csv") + for row in sample_rows + ] + metadata_tables = [ + pd.read_csv(temporary / "samples" / row["sample_id"] / "profiling" / "oocyte_metadata.csv") + for row in sample_rows + ] + marker_columns = [tuple(table.columns) for table in marker_tables] + metadata_columns = [tuple(table.columns) for table in metadata_tables] + if len(set(marker_columns)) != 1 or len(set(metadata_columns)) != 1: + raise ValueError("sample profiling schemas differ within release") + batch_markers = _concat_tables(marker_tables) + batch_metadata = _concat_tables(metadata_tables) + _write_csv(temporary / "batch_summary.csv", pd.DataFrame(sample_rows)) + _write_csv(temporary / "batch_oocyte_by_marker.csv", batch_markers) + _write_csv(temporary / "batch_oocyte_metadata.csv", batch_metadata) + if spec.algorithm_document is not None: + _require_files([spec.algorithm_document]) + shutil.copy2( + spec.algorithm_document, + temporary / "oocyte_detection_algorithm.html", + ) + _atomic_write_text( + temporary / "oocyte_review_index.html", + _batch_index_html(spec.release_name, sample_rows), + ) + _atomic_write_text( + temporary / "README.md", + _readme_text(spec.release_name, sample_rows), + ) + package_artifacts = { + str(path.relative_to(temporary)): _artifact_record(path) + for path in sorted(temporary.rglob("*")) + if path.is_file() and path.name != "release_manifest.json" + } + manifest = { + "schema_version": OOCYTE_RELEASE_SCHEMA_VERSION, + "implementation_version": OOCYTE_RELEASE_IMPLEMENTATION_VERSION, + "release_name": spec.release_name, + "built_at": datetime.now(timezone.utc).isoformat(), + "spec": { + "source_path": str(spec_source), + "path": packaged_spec.name, + **_artifact_record(packaged_spec), + }, + "sample_count": len(sample_rows), + "positive_oocyte_count": int( + sum(row["oocyte_count"] for row in sample_rows if row["role"] == "positive") + ), + "negative_control_count": int( + sum(row["role"] == "negative_control" for row in sample_rows) + ), + "samples": sample_rows, + "artifacts": package_artifacts, + } + _write_json(temporary / "release_manifest.json", manifest) + validate_oocyte_release(temporary) + temporary.replace(destination) + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + validation = validate_oocyte_release(destination) + return OocyteReleaseResult( + release_dir=destination, + manifest_path=destination / "release_manifest.json", + sample_count=int(validation["sample_count"]), + oocyte_count=int(validation["positive_oocyte_count"]), + validation=validation, + ) + + +def _verify_artifact_index(root: Path, records: Mapping[str, Any]) -> None: + for relative, raw in records.items(): + if not isinstance(raw, Mapping): + raise ValueError(f"invalid artifact record: {relative}") + path = (root / relative).resolve() + if not path.is_relative_to(root) or not path.is_file(): + raise FileNotFoundError(f"release artifact is missing: {relative}") + expected = { + "sha256": str(raw.get("sha256", "")), + "size_bytes": int(raw.get("size_bytes", -1)), + } + if _artifact_record(path) != expected: + raise ValueError(f"release artifact mismatch: {relative}") + + +def validate_oocyte_release(release_dir: Path) -> Dict[str, Any]: + """Validate every checksum and cross-file invariant in a release.""" + + root = Path(release_dir).resolve() + manifest_path = root / "release_manifest.json" + if not manifest_path.is_file(): + raise FileNotFoundError(manifest_path) + manifest = _read_object(manifest_path) + if manifest.get("schema_version") != OOCYTE_RELEASE_SCHEMA_VERSION: + raise ValueError("unsupported oocyte release schema_version") + implementation_version = str(manifest.get("implementation_version")) + embedded_console_required = implementation_version != "oocyte_release_v1" + overview_console_required = implementation_version in { + "oocyte_release_v3", + "oocyte_release_v4", + "oocyte_release_v5", + } + nearest_hotspot_required = implementation_version in { + "oocyte_release_v4", + "oocyte_release_v5", + } + hotspot_toggle_required = implementation_version == "oocyte_release_v5" + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping): + raise ValueError("release manifest is missing artifacts") + actual_files = { + str(path.relative_to(root)) + for path in root.rglob("*") + if path.is_file() and path.name != "release_manifest.json" + } + if set(artifacts) != actual_files: + raise ValueError("release artifact index does not match package files") + _verify_artifact_index(root, artifacts) + sample_rows = manifest.get("samples") + if not isinstance(sample_rows, list) or not sample_rows: + raise ValueError("release manifest has no samples") + seen_samples = set() + marker_columns: tuple[str, ...] | None = None + total_positive = 0 + all_marker_ids: set[str] = set() + per_sample = [] + for row in sample_rows: + sample_id = str(row.get("sample_id", "")) + role = str(row.get("role", "")) + if sample_id in seen_samples or role not in RELEASE_ROLES: + raise ValueError("invalid or duplicate sample in release manifest") + seen_samples.add(sample_id) + sample_root = root / "samples" / sample_id + sample_manifest_path = sample_root / "sample_release_manifest.json" + sample_manifest = _read_object(sample_manifest_path) + if sample_manifest.get("sample_id") != sample_id or sample_manifest.get("role") != role: + raise ValueError(f"sample manifest identity mismatch: {sample_id}") + sample_artifacts = sample_manifest.get("artifacts") + if not isinstance(sample_artifacts, Mapping): + raise ValueError(f"sample artifact index is missing: {sample_id}") + _verify_artifact_index(sample_root, sample_artifacts) + labels_path = sample_root / "final" / "oocyte_labels.ome.tiff" + mapping = pd.read_csv(sample_root / "final" / "oocyte_labels.csv") + candidates = pd.read_csv(sample_root / "final" / "oocyte_candidates.csv") + markers = pd.read_csv(sample_root / "profiling" / "oocyte_by_marker.csv") + metadata = pd.read_csv(sample_root / "profiling" / "oocyte_metadata.csv") + label_summary = _label_summary(labels_path) + count = int(sample_manifest.get("oocyte_count", -1)) + if not ( + count == len(mapping) == len(candidates) == len(markers) == len(metadata) + == label_summary["positive_label_count"] + ): + raise ValueError(f"release row/label count mismatch: {sample_id}") + if embedded_console_required: + console_path = sample_root / "review_console.html" + console_text = console_path.read_text(encoding="utf-8") + console_summary = sample_manifest.get("embedded_console") + if not isinstance(console_summary, Mapping): + raise ValueError( + f"release embedded console summary missing: {sample_id}" + ) + embedded_card_count = len( + re.findall( + r"]*\bdata-embedded-review-card(?:\s|>)", + console_text, + ) + ) + embedded_webp_count = console_text.count("data:image/webp;base64,") + overview_webp_count = 2 if count else 1 + expected_webp_count = count * 2 + if overview_console_required: + expected_webp_count += overview_webp_count + if ( + int(console_summary.get("embedded_card_count", -1)) != count + or int(console_summary.get("embedded_webp_count", -1)) + != expected_webp_count + or embedded_card_count != count + or embedded_webp_count != expected_webp_count + or f'data-embedded-card-count="{count}"' not in console_text + ): + raise ValueError( + f"release embedded console count mismatch: {sample_id}" + ) + if overview_console_required: + hotspot_count = len( + re.findall(r"]*\bdata-global-hotspot(?:\s|>)", console_text) + ) + if ( + int(console_summary.get("overview_webp_count", -1)) + != overview_webp_count + or int(console_summary.get("global_hotspot_count", -1)) + != count + or int(console_summary.get("overview_downsample", -1)) <= 1 + or console_text.count("data-global-overview") != 1 + or hotspot_count != count + or console_text.count('href="#oocyte-') != count + or console_text.count('id="oocyte-') != count + or "{overview_html}" in console_text + ): + raise ValueError( + f"release whole-slide console mismatch: {sample_id}" + ) + if nearest_hotspot_required and ( + console_text.count("data-card-target=") != count + or "best>Number(nearest.getAttribute('r'))**2" not in console_text + ): + raise ValueError( + f"release nearest-hotspot navigation mismatch: {sample_id}" + ) + if hotspot_toggle_required and ( + "hotspots.toggleAttribute('hidden',showRaw)" not in console_text + ): + raise ValueError( + f"release whole-slide mask toggle mismatch: {sample_id}" + ) + required_console_links = ( + "profiling/oocyte_by_marker.csv", + "profiling/oocyte_metadata.csv", + "final/oocyte_labels.ome.tiff", + "final/oocyte_labels.csv", + "review/review_manifest.json", + "sample_release_manifest.json", + ) + if any(link not in console_text for link in required_console_links): + raise ValueError(f"release embedded console links missing: {sample_id}") + if "/api/" in console_text or "http://" in console_text: + raise ValueError(f"release console requires a server: {sample_id}") + labels = set(int(value) for value in mapping["label"]) + if labels != set(label_summary["positive_labels"]): + raise ValueError(f"release mapping labels mismatch: {sample_id}") + if labels != set(range(1, count + 1)): + raise ValueError(f"release labels are not contiguous: {sample_id}") + if mapping["detector_component_id"].astype(str).duplicated().any(): + raise ValueError(f"release object IDs are duplicated: {sample_id}") + if set(mapping["detector_component_id"].astype(str)) != set( + candidates["detector_component_id"].astype(str) + ): + raise ValueError(f"release candidate IDs mismatch: {sample_id}") + expected_oocyte_ids = { + f"{sample_id}__{value}" + for value in mapping["detector_component_id"].astype(str) + } + marker_ids = set(markers["oocyte_id"].astype(str)) + if marker_ids != expected_oocyte_ids or set(metadata["oocyte_id"].astype(str)) != expected_oocyte_ids: + raise ValueError(f"release profiling IDs mismatch: {sample_id}") + if all_marker_ids.intersection(marker_ids): + raise ValueError("release oocyte IDs are not unique across samples") + all_marker_ids.update(marker_ids) + for mapping_row in mapping.to_dict("records"): + label = int(mapping_row["label"]) + expected_pixels = int(mapping_row["assigned_pixel_count"]) + if expected_pixels <= 0: + raise ValueError(f"release label is empty: {sample_id} label {label}") + if label_summary["label_pixel_counts"].get(label) != expected_pixels: + raise ValueError(f"release assigned pixel mismatch: {sample_id} label {label}") + if not ( + float(mapping_row["bbox_x0"]) + <= float(mapping_row["center_x"]) + < float(mapping_row["bbox_x1"]) + and float(mapping_row["bbox_y0"]) + <= float(mapping_row["center_y"]) + < float(mapping_row["bbox_y1"]) + ): + raise ValueError(f"release centroid is outside bbox: {sample_id}") + _validate_packaged_masks( + labels_path, + sample_root / "final", + mapping, + candidates, + label_summary["shape_yx"], + ) + current_columns = tuple(markers.columns) + if marker_columns is None: + marker_columns = current_columns + elif current_columns != marker_columns: + raise ValueError("release marker schemas differ") + if role == "negative_control" and count != 0: + raise ValueError(f"negative control is nonzero: {sample_id}") + if role == "positive" and count <= 0: + raise ValueError(f"positive sample is empty: {sample_id}") + if role == "positive": + total_positive += count + profile_manifest = _read_object(sample_root / "profiling" / "profiling_manifest.json") + profile_artifacts = profile_manifest.get("artifacts") + if not isinstance(profile_artifacts, Mapping): + raise ValueError(f"release profiling manifest missing artifacts: {sample_id}") + _verify_artifact_index(sample_root / "profiling", profile_artifacts) + if role == "negative_control": + review_manifest = _read_object( + sample_root / "review" / "review_manifest.json" + ) + signoff = review_manifest.get("negative_control_signoff") + if not isinstance(signoff, Mapping): + raise ValueError(f"negative control signoff is missing: {sample_id}") + if ( + signoff.get("status") != "passed" + or int(signoff.get("baseline_accepted_count", -1)) != 0 + or int(signoff.get("rescue_accepted_count", -1)) != 0 + ): + raise ValueError(f"negative control signoff failed: {sample_id}") + per_sample.append({"sample_id": sample_id, "role": role, "oocyte_count": count}) + batch_summary = pd.read_csv(root / "batch_summary.csv") + if len(batch_summary) != len(per_sample): + raise ValueError("release batch summary row count mismatch") + expected_summary = { + (row["sample_id"], row["role"], row["oocyte_count"]) + for row in per_sample + } + actual_summary = { + (str(row.sample_id), str(row.role), int(row.oocyte_count)) + for row in batch_summary.itertuples(index=False) + } + if actual_summary != expected_summary: + raise ValueError("release batch summary values mismatch") + for row in batch_summary.itertuples(index=False): + sample_manifest_path = ( + root / "samples" / str(row.sample_id) / "sample_release_manifest.json" + ) + if str(row.sample_manifest_sha256) != _file_sha256(sample_manifest_path): + raise ValueError(f"release sample manifest SHA mismatch: {row.sample_id}") + batch_markers = pd.read_csv(root / "batch_oocyte_by_marker.csv") + batch_metadata = pd.read_csv(root / "batch_oocyte_metadata.csv") + if len(batch_markers) != total_positive or len(batch_metadata) != total_positive: + raise ValueError("release cohort table row count mismatch") + if set(batch_markers["oocyte_id"].astype(str)) != all_marker_ids: + raise ValueError("release cohort marker IDs mismatch") + if set(batch_metadata["oocyte_id"].astype(str)) != all_marker_ids: + raise ValueError("release cohort metadata IDs mismatch") + if int(manifest.get("positive_oocyte_count", -1)) != total_positive: + raise ValueError("release positive oocyte total mismatch") + if int(manifest.get("sample_count", -1)) != len(sample_rows): + raise ValueError("release sample count mismatch") + return { + "status": "valid", + "release_name": str(manifest.get("release_name", "")), + "sample_count": len(sample_rows), + "positive_oocyte_count": total_positive, + "negative_control_count": sum(row["role"] == "negative_control" for row in per_sample), + "channel_count": 0 if marker_columns is None else len(marker_columns) - 3, + "artifact_count": len(artifacts), + "samples": per_sample, + } + + +__all__ = [ + "OOCYTE_RELEASE_IMPLEMENTATION_VERSION", + "OOCYTE_RELEASE_SCHEMA_VERSION", + "OocyteReleaseResult", + "OocyteReleaseSample", + "OocyteReleaseSpec", + "build_oocyte_release", + "load_oocyte_release_spec", + "validate_oocyte_release", +] diff --git a/aegle/oocyte/release_console.py b/aegle/oocyte/release_console.py new file mode 100644 index 0000000..8fd90da --- /dev/null +++ b/aegle/oocyte/release_console.py @@ -0,0 +1,475 @@ +"""Self-contained static review pages for packaged oocyte releases.""" + +from __future__ import annotations + +import base64 +import html +import math +import tempfile +from io import BytesIO +from pathlib import Path +from typing import Any, Dict, Mapping + +from matplotlib import colormaps +import numpy as np +import pandas as pd +import tifffile +import zarr +from PIL import Image +from scipy import ndimage as ndi + +from .io import extract_cyx_channel_patch, find_channel_index, load_candidate_mask +from .report import render_raw_mask_thumbnail_bytes + + +EMBEDDED_CONSOLE_SCHEMA_VERSION = 1 +EMBEDDED_CONSOLE_RENDER_VERSION = "release-embedded-uchl1-v4" +OVERVIEW_DOWNSAMPLE = 16 + + +def _atomic_write_text(path: Path, content: str) -> None: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{destination.name}.", + suffix=".tmp", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + handle.write(content) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _number(value: Any, digits: int = 1) -> str: + try: + number = float(value) + except (TypeError, ValueError): + return "n/a" + if not np.isfinite(number): + return "n/a" + return f"{number:.{digits}f}" + + +def _data_uri(image_bytes: bytes) -> str: + encoded = base64.b64encode(image_bytes).decode("ascii") + return f"data:image/webp;base64,{encoded}" + + +def _image_data_uri(image: Image.Image, *, quality: int = 84) -> str: + buffer = BytesIO() + image.save(buffer, format="WEBP", quality=quality, method=6) + return _data_uri(buffer.getvalue()) + + +def _reduce_strip(strip: np.ndarray, factor: int) -> np.ndarray: + pad_h = (-strip.shape[0]) % factor + pad_w = (-strip.shape[1]) % factor + if pad_h or pad_w: + strip = np.pad(strip, ((0, pad_h), (0, pad_w)), mode="edge") + return strip.reshape( + strip.shape[0] // factor, + factor, + strip.shape[1] // factor, + factor, + ).mean(axis=(1, 3)) + + +def _raw_overview_image( + raw_array: Any, + channel_index: int, + *, + downsample: int = OVERVIEW_DOWNSAMPLE, + strip_height: int = 1024, +) -> Image.Image: + channel_count, image_h, image_w = (int(value) for value in raw_array.shape) + if not 0 <= channel_index < channel_count: + raise IndexError("overview channel index is outside the source image") + overview = np.zeros( + (math.ceil(image_h / downsample), math.ceil(image_w / downsample)), + dtype=np.float32, + ) + output_y = 0 + for y0 in range(0, image_h, strip_height): + y1 = min(image_h, y0 + strip_height) + strip = np.asarray(raw_array[channel_index, y0:y1, :], dtype=np.float32) + reduced = _reduce_strip(strip, downsample) + overview[ + output_y : output_y + reduced.shape[0], + : reduced.shape[1], + ] = reduced + output_y += reduced.shape[0] + transformed = np.log1p(np.maximum(overview, 0.0)) + finite = transformed[np.isfinite(transformed)] + low, high = ( + (float(value) for value in np.percentile(finite, [1.0, 99.9])) + if finite.size + else (0.0, 1.0) + ) + normalized = np.clip( + (transformed - low) / max(float(high - low), 1e-6), + 0.0, + 1.0, + ) + rgb = np.asarray(colormaps["magma"](normalized)[..., :3] * 255.0, dtype=np.uint8) + return Image.fromarray(rgb) + + +def _masked_overview_image( + raw_overview: Image.Image, + persisted_masks: Mapping[str, Any], + *, + downsample: int = OVERVIEW_DOWNSAMPLE, +) -> Image.Image: + boundary_map = np.zeros( + (raw_overview.height, raw_overview.width), + dtype=np.bool_, + ) + for persisted in persisted_masks.values(): + boundary = np.logical_xor( + persisted.mask, + ndi.binary_erosion(persisted.mask), + ) + yy, xx = np.nonzero(boundary) + if not len(xx): + continue + overview_y = (persisted.bbox.y0 + yy) // downsample + overview_x = (persisted.bbox.x0 + xx) // downsample + valid = ( + (overview_y >= 0) + & (overview_y < raw_overview.height) + & (overview_x >= 0) + & (overview_x < raw_overview.width) + ) + boundary_map[overview_y[valid], overview_x[valid]] = True + if boundary_map.any(): + boundary_map = ndi.binary_dilation(boundary_map, iterations=1) + rgb = np.asarray(raw_overview.convert("RGB")).copy() + rgb[boundary_map] = np.array([0, 255, 246], dtype=np.uint8) + return Image.fromarray(rgb) + + +def _provenance_label(row: Mapping[str, Any]) -> str: + detection_pass = str(row.get("detection_pass", "")).strip() + manual_choice = str(row.get("manual_mask_choice", "")).strip() + if "manual_contour" in detection_pass or "contour" in manual_choice: + return "manual contour" + if detection_pass.startswith("manual_seed"): + return "reviewed recall" + if detection_pass.startswith("rescue"): + return "rescue" + return "automatic" + + +def _artifact_links() -> str: + links = ( + ("Batch index", "../../oocyte_review_index.html"), + ("Batch expression", "../../batch_oocyte_by_marker.csv"), + ("Sample expression", "profiling/oocyte_by_marker.csv"), + ("Metadata", "profiling/oocyte_metadata.csv"), + ("Label OME-TIFF", "final/oocyte_labels.ome.tiff"), + ("Label mapping", "final/oocyte_labels.csv"), + ("Candidates", "final/oocyte_candidates.csv"), + ("Review evidence", "review/review_manifest.json"), + ("Checksums", "sample_release_manifest.json"), + ) + return "".join( + f'{html.escape(label)}' for label, href in links + ) + + +def _card_html( + row: Mapping[str, Any], + *, + raw_uri: str, + masked_uri: str, +) -> str: + label_id = int(row["label_id"]) + oocyte_id = html.escape(str(row["oocyte_id"])) + component_id = html.escape(str(row["detector_component_id"])) + provenance = _provenance_label(row) + warning = bool(row.get("boundary_warning", False)) + warning_html = 'boundary note' if warning else "" + return f""" +
    +
    + Raw UCHL1 with final mask for {oocyte_id} + + +
    +
    +

    #{label_id:03d}

    {html.escape(provenance)}
    + {oocyte_id}{component_id} +
    + {_number(row.get('equivalent_diameter_um'))} um diameter + {_number(row.get('area_um2'), 0)} um2 area + {_number(row.get('circularity'), 2)} circularity + {_number(row.get('solidity'), 2)} solidity + {_number(row.get('center_x'), 0)} x + {_number(row.get('center_y'), 0)} y +
    + {warning_html} +
    +
    + """ + + +def _overview_html( + *, + rows: list[Mapping[str, Any]], + image_shape_yx: tuple[int, int], + raw_uri: str, + masked_uri: str | None, +) -> str: + image_h, image_w = image_shape_yx + hotspots = [] + for row in rows: + label_id = int(row["label_id"]) + center_x = float(row["center_x"]) + center_y = float(row["center_y"]) + try: + bbox_width = float(row["bbox_x1"]) - float(row["bbox_x0"]) + bbox_height = float(row["bbox_y1"]) - float(row["bbox_y0"]) + except (TypeError, ValueError): + bbox_width = bbox_height = 0.0 + radius = max(70.0, min(180.0, max(bbox_width, bbox_height) * 0.75)) + title = html.escape(f"#{label_id:03d} {row['oocyte_id']}") + hotspots.append( + f'' + f'' + f'{title}' + ) + if masked_uri is None: + images = ( + f'' + ) + controls = ( + 'No final masks in this negative control' + ) + svg = "" + else: + images = ( + f'' + f'' + ) + controls = ( + '' + 'Click a mask location to jump to its card' + ) + svg = ( + f'' + f'{"".join(hotspots)}' + ) + return f""" +
    +
    Whole-slide navigator

    Final masks in tissue context

    {controls}
    +

    Downsampled raw UCHL1 across the complete registered section. Cyan boundaries are the exact final NPZ masks projected at overview scale.

    +
    {images}{svg}
    +
    + """ + + +def _page_html( + *, + sample_id: str, + role: str, + cards: list[str], + provenance_counts: Mapping[str, int], + overview_html: str, +) -> str: + count = len(cards) + filters = "".join( + f'" + for name, value in sorted(provenance_counts.items()) + ) + if count: + content = f""" + {overview_html} +
    {filters}
    +
    {''.join(cards)}
    + """ + description = ( + "Each card embeds the registered raw UCHL1 patch and the exact final " + "release mask. Cyan is the delivered boundary; the white cross marks " + "the recorded center. No server or raw-image file is needed to view " + "these cards." + ) + else: + content = f""" + {overview_html} +

    No final oocytes

    This sample is a validated no-oocyte negative control. Its label image, mapping, candidate table, and expression tables contain zero positive rows.

    + """ + description = ( + "The frozen detector and rescue diagnostics contain zero accepted " + "objects, and the final whole-slide label contains background only." + ) + return f""" + + + + + {html.escape(sample_id)} final oocytes + + +
    +
    Aegle / raw UCHL1 / final reviewed release / {html.escape(role)}

    {html.escape(sample_id)}

    {count} final reviewed oocyte labels. {html.escape(description)}

    + {content} +
    Static release console. All review images are embedded in this HTML; artifact links are relative to the unpacked release directory.
    +
    +""" + + +def write_embedded_release_console( + *, + sample_id: str, + role: str, + source_image_path: Path, + antibodies_path: Path, + final_dir: Path, + profiling_dir: Path, + destination: Path, + output_size_px: int = 360, + webp_quality: int = 82, +) -> Dict[str, Any]: + """Write a server-free release page containing exact raw/mask image pairs.""" + + metadata = pd.read_csv(Path(profiling_dir) / "oocyte_metadata.csv") + mapping = pd.read_csv(Path(final_dir) / "oocyte_labels.csv") + if len(metadata) != len(mapping): + raise ValueError(f"release console row count mismatch for {sample_id}") + joined = metadata.merge( + mapping[["detector_component_id", "mask_path"]], + on="detector_component_id", + how="left", + validate="one_to_one", + suffixes=("", "_final"), + ) + if len(joined) and joined["mask_path_final"].isna().any(): + raise ValueError(f"release console mask mapping is incomplete for {sample_id}") + joined = joined.sort_values("label_id", kind="stable") + + rows = joined.to_dict("records") + persisted_masks = { + str(row["detector_component_id"]): load_candidate_mask( + (Path(final_dir) / str(row["mask_path_final"])).resolve() + ) + for row in rows + } + cards: list[str] = [] + provenance_counts: Dict[str, int] = {} + channel_index = find_channel_index(antibodies_path, "UCHL1") + with tifffile.TiffFile(source_image_path) as tif: + series = tif.series[0] + if str(series.axes) != "CYX": + raise ValueError( + f"embedded release console requires CYX source axes: {series.axes}" + ) + store = series.aszarr() + try: + raw_array = zarr.open(store, mode="r") + raw_overview = _raw_overview_image(raw_array, channel_index) + raw_overview_uri = _image_data_uri(raw_overview) + masked_overview_uri = None + if persisted_masks: + masked_overview_uri = _image_data_uri( + _masked_overview_image(raw_overview, persisted_masks) + ) + for row in rows: + persisted = persisted_masks[str(row["detector_component_id"])] + max_mask_dimension = max(persisted.mask.shape) + radius = max(96, min(240, math.ceil(max_mask_dimension * 1.35))) + patch = extract_cyx_channel_patch( + raw_array, + channel_index, + ( + int(round(float(row["center_x"]))), + int(round(float(row["center_y"]))), + ), + radius, + ) + raw_uri = _data_uri( + render_raw_mask_thumbnail_bytes( + patch, + persisted, + show_mask=False, + output_size_px=output_size_px, + quality=webp_quality, + ) + ) + masked_uri = _data_uri( + render_raw_mask_thumbnail_bytes( + patch, + persisted, + show_mask=True, + output_size_px=output_size_px, + quality=webp_quality, + ) + ) + cards.append(_card_html(row, raw_uri=raw_uri, masked_uri=masked_uri)) + provenance = _provenance_label(row) + provenance_counts[provenance] = ( + provenance_counts.get(provenance, 0) + 1 + ) + image_shape_yx = (int(raw_array.shape[1]), int(raw_array.shape[2])) + finally: + close = getattr(store, "close", None) + if close is not None: + close() + + overview = _overview_html( + rows=rows, + image_shape_yx=image_shape_yx, + raw_uri=raw_overview_uri, + masked_uri=masked_overview_uri, + ) + + page = _page_html( + sample_id=sample_id, + role=role, + cards=cards, + provenance_counts=provenance_counts, + overview_html=overview, + ) + _atomic_write_text(destination, page) + overview_webp_count = 2 if masked_overview_uri is not None else 1 + return { + "schema_version": EMBEDDED_CONSOLE_SCHEMA_VERSION, + "renderer": EMBEDDED_CONSOLE_RENDER_VERSION, + "embedded_card_count": len(cards), + "embedded_webp_count": len(cards) * 2 + overview_webp_count, + "overview_webp_count": overview_webp_count, + "overview_downsample": OVERVIEW_DOWNSAMPLE, + "global_hotspot_count": len(rows), + "html_size_bytes": len(page.encode("utf-8")), + "provenance_counts": provenance_counts, + } + + +__all__ = [ + "EMBEDDED_CONSOLE_RENDER_VERSION", + "EMBEDDED_CONSOLE_SCHEMA_VERSION", + "write_embedded_release_console", +] diff --git a/aegle/oocyte/report.py b/aegle/oocyte/report.py new file mode 100644 index 0000000..9d4c9d3 --- /dev/null +++ b/aegle/oocyte/report.py @@ -0,0 +1,781 @@ +"""Self-contained algorithm documentation and per-sample HTML review pages.""" + +from __future__ import annotations + +import hashlib +import html +import json +import logging +import tempfile +from dataclasses import dataclass +from io import BytesIO +from pathlib import Path +from typing import Any, Dict, List, Mapping, Tuple + +import matplotlib + +matplotlib.use("Agg") +import numpy as np +import pandas as pd +import tifffile +import zarr +from matplotlib import colormaps +from PIL import Image +from scipy import ndimage as ndi + +from .config import DONOR13_V6, DONOR13_V6_RESCUE_V1, OocyteDetectionConfig +from .export import export_whole_slide_labels +from .io import extract_cyx_channel_patch, load_candidate_mask +from .models import ExtractedPatch, PersistedMask +from .rescue import suppress_accepted_mask_duplicates + + +LOGGER = logging.getLogger(__name__) +THUMBNAIL_CACHE_SCHEMA_VERSION = 1 +THUMBNAIL_RENDER_VERSION = "raw-mask-thumbnail-v2" +PRECISION_REVIEW_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class HtmlReportResult: + batch_dir: Path + algorithm_document: Path + batch_index: Path + sample_pages: Dict[str, Path] + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _atomic_write_json(path: Path, payload: Mapping[str, Any]) -> None: + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{destination.name}.", + suffix=".tmp", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + json.dump(payload, handle, sort_keys=True, separators=(",", ":")) + handle.write("\n") + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _thumbnail_fingerprint( + row: Mapping[str, Any], + *, + source_image: Path, + source_size_bytes: int, + source_mtime_ns: int, + channel_index: int, + patch_radius_px: int, + mask_path: Path, +) -> str: + payload = { + "renderer": THUMBNAIL_RENDER_VERSION, + "source_image": str(Path(source_image).resolve()), + "source_size_bytes": int(source_size_bytes), + "source_mtime_ns": str(source_mtime_ns), + "channel_index": int(channel_index), + "patch_radius_px": int(patch_radius_px), + "candidate_id": str(row["detector_component_id"]), + "detection_pass": str(row.get("detection_pass", "baseline_v6")), + "center_x": int(round(float(row["center_x"]))), + "center_y": int(round(float(row["center_y"]))), + "mask_path": str(Path(mask_path).resolve()), + "mask_sha256": _file_sha256(mask_path), + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _load_thumbnail_manifest(path: Path) -> Dict[str, Any]: + manifest_path = Path(path) + if not manifest_path.is_file(): + return {} + try: + payload = json.loads(manifest_path.read_text()) + except (OSError, json.JSONDecodeError): + return {} + if ( + payload.get("schema_version") != THUMBNAIL_CACHE_SCHEMA_VERSION + or payload.get("renderer") != THUMBNAIL_RENDER_VERSION + or not isinstance(payload.get("entries"), dict) + ): + return {} + return payload + + +def _thumbnail_is_current( + asset_path: Path, + entry: Any, + expected_fingerprint: str, +) -> bool: + if not asset_path.is_file() or not isinstance(entry, dict): + return False + if entry.get("render_fingerprint") != expected_fingerprint: + return False + expected_sha = entry.get("asset_sha256") + return isinstance(expected_sha, str) and _file_sha256(asset_path) == expected_sha + + +def _format_number(value: Any, digits: int = 2) -> str: + try: + number = float(value) + except (TypeError, ValueError): + return "n/a" + if not np.isfinite(number): + return "n/a" + return f"{number:.{digits}f}" + + +def _json_safe(value: Any) -> Any: + if value is None: + return None + if isinstance(value, (np.bool_, bool)): + return bool(value) + if isinstance(value, (np.integer, int)): + return int(value) + if isinstance(value, (np.floating, float)): + return None if not np.isfinite(float(value)) else float(value) + return str(value) + + +def _place_mask(mask: PersistedMask, patch: ExtractedPatch) -> np.ndarray: + placed = np.zeros(patch.image.shape, dtype=np.bool_) + x0 = max(mask.bbox.x0, patch.bbox.x0) + y0 = max(mask.bbox.y0, patch.bbox.y0) + x1 = min(mask.bbox.x1, patch.bbox.x1) + y1 = min(mask.bbox.y1, patch.bbox.y1) + if x0 >= x1 or y0 >= y1: + return placed + top, _, left, _ = patch.padding_tblr + source = mask.mask[ + y0 - mask.bbox.y0 : y1 - mask.bbox.y0, + x0 - mask.bbox.x0 : x1 - mask.bbox.x0, + ] + target_y = top + y0 - patch.bbox.y0 + target_x = left + x0 - patch.bbox.x0 + placed[target_y : target_y + source.shape[0], target_x : target_x + source.shape[1]] = source + return placed + + +def _raw_mask_thumbnail_image( + patch: ExtractedPatch, + persisted_mask: PersistedMask, + *, + show_mask: bool = True, + output_size_px: int = 420, +) -> Image.Image: + raw = np.asarray(patch.image, dtype=np.float32) + transformed = np.log1p(np.maximum(raw, 0.0)) + finite = transformed[np.isfinite(transformed)] + lo, hi = np.percentile(finite, [2.0, 99.8]) if finite.size else (0.0, 1.0) + normalized = np.clip((transformed - lo) / max(float(hi - lo), 1e-6), 0.0, 1.0) + rgb = np.asarray(colormaps["magma"](normalized)[..., :3] * 255.0, dtype=np.uint8) + mask = _place_mask(persisted_mask, patch) + if show_mask and mask.any(): + rgb[mask] = np.asarray( + 0.82 * rgb[mask] + 0.18 * np.array([0, 245, 236]), + dtype=np.uint8, + ) + boundary = np.logical_xor(mask, ndi.binary_erosion(mask)) + rgb[ndi.binary_dilation(boundary, iterations=1)] = np.array([0, 255, 246]) + center_y = rgb.shape[0] // 2 + center_x = rgb.shape[1] // 2 + rgb[max(0, center_y - 6) : center_y + 7, center_x] = 255 + rgb[center_y, max(0, center_x - 6) : center_x + 7] = 255 + image = Image.fromarray(rgb) + return image.resize((output_size_px, output_size_px), Image.Resampling.LANCZOS) + + +def render_raw_mask_thumbnail_bytes( + patch: ExtractedPatch, + persisted_mask: PersistedMask, + *, + show_mask: bool = True, + output_size_px: int = 420, + quality: int = 88, +) -> bytes: + """Render one raw-UCHL1 patch as self-contained WebP bytes.""" + + image = _raw_mask_thumbnail_image( + patch, + persisted_mask, + show_mask=show_mask, + output_size_px=output_size_px, + ) + buffer = BytesIO() + image.save(buffer, format="WEBP", quality=quality, method=6) + return buffer.getvalue() + + +def _raw_mask_thumbnail( + patch: ExtractedPatch, + persisted_mask: PersistedMask, + destination: Path, + *, + output_size_px: int = 420, +) -> None: + image_bytes = render_raw_mask_thumbnail_bytes( + patch, + persisted_mask, + output_size_px=output_size_px, + ) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + prefix=f".{destination.name}.", + suffix=".webp", + dir=destination.parent, + delete=False, + ) as handle: + temporary_path = Path(handle.name) + temporary_path.write_bytes(image_bytes) + temporary_path.replace(destination) + finally: + if temporary_path is not None and temporary_path.exists(): + temporary_path.unlink() + + +def _load_references(path: Path | None, sample_id: str) -> np.ndarray: + if path is None: + return np.empty((0, 2), dtype=np.float64) + coordinates = [] + with Path(path).open() as handle: + for line in handle: + if not line.strip(): + continue + record = json.loads(line) + if str(record.get("sample_id", "")) != sample_id: + continue + if float(record.get("final_score", 0.0)) < 0.35: + continue + center = record.get("center") + if center is not None and len(center) == 2: + coordinates.append((float(center[0]), float(center[1]))) + return np.asarray(coordinates, dtype=np.float64).reshape(-1, 2) + + +def _annotate_reference_distance(table: pd.DataFrame, reference_xy: np.ndarray) -> pd.DataFrame: + output = table.copy() + if output.empty or reference_xy.size == 0: + output["nearest_reference_distance_px"] = np.nan + output["reference_class"] = "unavailable" if reference_xy.size == 0 else "novel" + return output + candidate_xy = output[["center_x", "center_y"]].to_numpy(dtype=np.float64) + distances = np.sqrt( + ((candidate_xy[:, None, :] - reference_xy[None, :, :]) ** 2).sum(axis=2) + ) + nearest = distances.min(axis=1) + output["nearest_reference_distance_px"] = nearest + output["reference_class"] = np.where(nearest <= 100.0, "reference-linked", "detector-only") + return output + + +def _candidate_quality(row: Mapping[str, Any]) -> str: + score = float(row.get("detector_score", 0.0)) + diameter = float(row.get("local_equivalent_diameter_um", 0.0)) + circularity = float(row.get("local_circularity", 0.0)) + if ( + str(row.get("rescue_acceptance_rule", "")) + in {"bright_irregular", "bright_fragment", "baseline_shape_fallback"} + or score < 0.50 + or diameter < 20.0 + or circularity < 0.70 + ): + return "review-priority" + return "standard" + + +def _precision_review_key(row: Mapping[str, Any]) -> str: + return ( + f"{str(row.get('detection_pass', 'baseline_v6'))}:" + f"{str(row['detector_component_id'])}" + ) + + +def _precision_review_rows(candidates: pd.DataFrame) -> List[Dict[str, Any]]: + payload_columns = [ + "html_id", + "display_id", + "detector_component_id", + "detection_pass", + "detector_score", + "center_x", + "center_y", + "local_equivalent_diameter_um", + "local_circularity", + "local_solidity", + "reference_class", + "nearest_reference_distance_px", + "quality_class", + "acceptance_mode", + "rescue_acceptance_rule", + ] + rows = [] + for row in candidates.reindex(columns=payload_columns).to_dict("records"): + safe_row = {key: _json_safe(value) for key, value in row.items()} + safe_row["review_key"] = _precision_review_key(safe_row) + rows.append(safe_row) + keys = [str(row["review_key"]) for row in rows] + if len(keys) != len(set(keys)): + raise ValueError("precision review keys must be unique within a sample") + return rows + + +def _precision_review_identity( + *, + sample_id: str, + source_image: Path, + manifest: Mapping[str, Any], + sample_summary: Mapping[str, Any], + candidate_table_path: Path, + candidate_count: int, +) -> Dict[str, Any]: + source_path = Path(source_image).resolve() + source_stat = source_path.stat() + return { + "sample_id": str(sample_id), + "source_image": str(source_path), + "source_image_size_bytes": int(source_stat.st_size), + "source_image_mtime_ns": str(source_stat.st_mtime_ns), + "profile_name": str( + manifest.get("profile_name", sample_summary.get("profile_name", "")) + ), + "profile_fingerprint": str(manifest.get("profile_fingerprint", "")), + "implementation_version": str(manifest.get("implementation_version", "")), + "candidate_table_sha256": _file_sha256(candidate_table_path), + "combined_candidate_count": int(candidate_count), + } + + +def _spatial_svg( + candidates: pd.DataFrame, + image_shape_yx: Tuple[int, int], +) -> str: + height, width = image_shape_yx + view_width = 1000.0 + view_height = max(280.0, view_width * height / max(width, 1)) + circles = [] + for row in candidates.to_dict("records"): + x = float(row["center_x"]) / max(width, 1) * view_width + y = float(row["center_y"]) / max(height, 1) * view_height + css = "rescue-dot" if row["detection_pass"] == "secondary_rescue" else "baseline-dot" + candidate_id = html.escape(str(row["html_id"]), quote=True) + label = html.escape(str(row["display_id"]), quote=True) + circles.append( + f'{label}' + ) + return ( + f'' + '' + '' + '' + f'' + f'' + + "".join(circles) + + "" + ) + + +_REPORT_CSS = r""" +:root{--ink:#172522;--paper:#f2ecdf;--panel:#fffaf0;--teal:#087b78;--cyan:#00d9cf;--amber:#d77a1f;--red:#a53b2d;--muted:#6e756f;--line:#d8d0bf;--shadow:0 16px 38px rgba(41,51,46,.12)} +*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;color:var(--ink);background:radial-gradient(circle at 15% 0,#fff9e9 0,transparent 32%),linear-gradient(135deg,#eee5d5,#f7f2e8 65%,#e7eee7);font-family:"Iowan Old Style","Palatino Linotype",Palatino,serif}.shell{width:min(1480px,calc(100% - 32px));margin:auto}.hero{margin:20px 0;padding:30px 34px;border:1px solid #c9c0ad;border-radius:24px;background:linear-gradient(120deg,rgba(255,250,240,.97),rgba(229,244,238,.92));box-shadow:var(--shadow);position:relative;overflow:hidden}.hero:after{content:"";position:absolute;width:260px;height:260px;border:40px solid rgba(8,123,120,.08);border-radius:50%;right:-90px;top:-110px}.eyebrow,.mono{font-family:"IBM Plex Mono","Aptos Mono","Courier New",monospace}.eyebrow{text-transform:uppercase;letter-spacing:.14em;font-size:.76rem;color:var(--teal);font-weight:700}.hero h1{font-size:clamp(2.1rem,5vw,4.6rem);line-height:.94;margin:.25em 0 .18em;max-width:930px}.hero p{max-width:850px;font-size:1.06rem;line-height:1.55}.stats{display:flex;gap:12px;flex-wrap:wrap;margin-top:20px}.stat{background:#fffaf0;border:1px solid var(--line);border-radius:14px;padding:12px 16px;min-width:145px}.stat strong{font-size:1.55rem;display:block}.toolbar{position:sticky;top:8px;z-index:20;margin:18px 0;padding:12px;display:flex;gap:10px;align-items:center;flex-wrap:wrap;border:1px solid var(--line);border-radius:16px;background:rgba(255,250,240,.94);backdrop-filter:blur(10px);box-shadow:0 8px 20px rgba(30,40,35,.1)}button,select,input{font:inherit}button,.button,select,input[type=search]{border:1px solid #bdb5a6;background:#fffaf0;color:var(--ink);border-radius:999px;padding:9px 14px;text-decoration:none}button{cursor:pointer}button:hover,.button:hover,.filter.active{border-color:var(--teal);background:#e2f2ed}.toolbar .spacer{flex:1}.review-message{width:100%;min-height:1em;color:var(--teal);font-size:.75rem}.review-message.error{color:var(--red)}.map-panel{background:var(--panel);border:1px solid var(--line);border-radius:20px;padding:18px;box-shadow:var(--shadow);margin:18px 0}.map-panel h2{margin:0 0 8px}.spatial-map{width:100%;max-height:520px}.map-dot{cursor:pointer;stroke:#fffaf0;stroke-width:2;transition:r .18s,opacity .18s}.map-dot:hover,.map-dot:focus{r:12;outline:none}.baseline-dot{fill:var(--teal)}.rescue-dot{fill:var(--amber)}.legend{display:flex;gap:18px;align-items:center;font-family:"IBM Plex Mono","Aptos Mono","Courier New",monospace;font-size:.8rem}.legend i{display:inline-block;width:11px;height:11px;border-radius:50%;margin-right:6px}.cards{display:grid;grid-template-columns:repeat(auto-fill,minmax(285px,1fr));gap:16px;margin:18px 0 60px}.card{background:var(--panel);border:1px solid var(--line);border-radius:18px;overflow:hidden;box-shadow:0 8px 22px rgba(30,40,35,.08);animation:rise .45s both;transition:transform .18s,border-color .18s}.card:hover{transform:translateY(-3px);border-color:#9b968b}.card.review-priority{border-color:#d69a66}.card.hidden{display:none}.card-image{position:relative;aspect-ratio:1;background:#180c21}.card-image img{position:absolute;inset:0;width:100%;height:100%;object-fit:cover;display:block}.card-image img[hidden]{display:none!important}.mask-toggle{position:absolute;right:9px;bottom:9px;z-index:3;padding:6px 10px;border-radius:9px;background:rgba(255,250,240,.92);box-shadow:0 3px 10px rgba(20,22,20,.28);font:700 .7rem "IBM Plex Mono","Aptos Mono","Courier New",monospace}.mask-toggle.raw-visible{background:rgba(8,123,120,.94);border-color:#8ee6df;color:white}.mask-toggle:disabled{cursor:not-allowed;opacity:.82}.card-body{padding:14px}.card-title{display:flex;justify-content:space-between;gap:8px;align-items:flex-start}.card h3{margin:0;font-size:1.15rem}.badge{font-family:"IBM Plex Mono","Aptos Mono","Courier New",monospace;font-size:.69rem;padding:4px 7px;border-radius:999px;background:#dceeea;color:#075c5a}.badge.rescue{background:#f7dfbf;color:#8a4712}.metrics{display:grid;grid-template-columns:repeat(2,1fr);gap:7px;margin:12px 0;font-family:"IBM Plex Mono","Aptos Mono","Courier New",monospace;font-size:.73rem}.metrics span{background:#f3ecde;border-radius:8px;padding:6px}.review-actions{display:grid;grid-template-columns:repeat(3,1fr);gap:6px}.review-actions button{padding:7px 4px;font-size:.78rem}.review-actions button.selected[data-status=accept]{background:#d8eee5;border-color:#4b9b7d}.review-actions button.selected[data-status=reject]{background:#f3d9d2;border-color:#b65343}.review-actions button.selected[data-status=unsure]{background:#f6e8bd;border-color:#c18a2d}.note-presets{display:flex;flex-wrap:wrap;gap:5px;margin-top:8px}.note-preset{padding:5px 8px;border-radius:8px;color:#59645f;background:#f3ecde;font:600 .63rem "IBM Plex Mono","Aptos Mono","Courier New",monospace}.note-preset.selected{color:#fff;background:var(--teal);border-color:var(--teal)}.notes{width:100%;border:1px solid var(--line);border-radius:9px;margin-top:8px;padding:7px;background:#fffdf7}.empty{padding:60px;text-align:center;border:1px dashed #bdb5a6;border-radius:18px;background:#fffaf0}.footer{padding:30px 0 60px;color:var(--muted)}@keyframes rise{from{opacity:0;transform:translateY(12px)}to{opacity:1;transform:none}}@media(max-width:700px){.shell{width:min(100% - 18px,1480px)}.hero{padding:24px 20px}.toolbar{position:static}.toolbar .spacer{display:none}.cards{grid-template-columns:1fr}.card{animation:none}} +""" + + +_REPORT_JS = r""" +const PACKAGE=JSON.parse(document.getElementById('candidate-data').textContent),DATA=PACKAGE.rows,IDENTITY=PACKAGE.identity,KEY='aegle-oocyte-precision-review:'+IDENTITY.sample_id+':'+IDENTITY.candidate_table_sha256;let state={};try{state=JSON.parse(localStorage.getItem(KEY)||'{}')}catch(_error){state={}}let filter='all';const cards=[...document.querySelectorAll('.card')],allowedStatuses=new Set(['accept','reject','unsure']);function save(){localStorage.setItem(KEY,JSON.stringify(state));progress()}function paint(card){const key=card.dataset.reviewKey,s=state[key]||{},notes=s.notes||'';card.dataset.review=s.status||'unreviewed';card.querySelectorAll('.review-actions button').forEach(b=>b.classList.toggle('selected',b.dataset.status===s.status));card.querySelectorAll('.note-preset').forEach(b=>b.classList.toggle('selected',b.dataset.note===notes));card.querySelector('.notes').value=notes}function progress(){const reviewed=DATA.filter(row=>allowedStatuses.has((state[row.review_key]||{}).status)).length;document.getElementById('review-progress').textContent=reviewed+' / '+DATA.length+' reviewed'}function apply(){const q=document.getElementById('search').value.toLowerCase();cards.forEach(c=>{const matchFilter=filter==='all'||(filter==='unreviewed'&&c.dataset.review==='unreviewed')||(filter==='flagged'&&c.dataset.quality==='review-priority')||c.dataset.pass===filter||c.dataset.rule===filter;const matchSearch=!q||c.dataset.search.includes(q);c.classList.toggle('hidden',!(matchFilter&&matchSearch))})}function message(text,isError=false){const node=document.getElementById('review-message');node.textContent=text;node.classList.toggle('error',isError)}cards.forEach(paint);document.querySelectorAll('.review-actions button').forEach(b=>b.addEventListener('click',()=>{const card=b.closest('.card'),key=card.dataset.reviewKey;state[key]={...(state[key]||{}),status:b.dataset.status};paint(card);save();apply()}));document.querySelectorAll('.note-preset').forEach(b=>b.addEventListener('click',()=>{const card=b.closest('.card'),key=card.dataset.reviewKey,current=(state[key]||{}).notes||'',notes=current===b.dataset.note?'':b.dataset.note;state[key]={...(state[key]||{}),notes};paint(card);save()}));document.querySelectorAll('.notes').forEach(n=>n.addEventListener('input',()=>{const card=n.closest('.card'),key=card.dataset.reviewKey;state[key]={...(state[key]||{}),notes:n.value};card.querySelectorAll('.note-preset').forEach(b=>b.classList.toggle('selected',b.dataset.note===n.value));save()}));document.querySelectorAll('.filter').forEach(b=>b.addEventListener('click',()=>{filter=b.dataset.filter;document.querySelectorAll('.filter').forEach(x=>x.classList.toggle('active',x===b));apply()}));document.getElementById('search').addEventListener('input',apply);document.getElementById('sort').addEventListener('change',e=>{const mode=e.target.value,grid=document.querySelector('.cards');cards.sort((a,b)=>mode==='score'?+b.dataset.score-+a.dataset.score:mode==='position'?+a.dataset.y-+b.dataset.y:+a.dataset.index-+b.dataset.index).forEach(c=>grid.appendChild(c))});document.querySelectorAll('.map-dot').forEach(dot=>dot.addEventListener('click',()=>document.getElementById(dot.dataset.card).scrollIntoView({behavior:'smooth',block:'center'})));function reviewedRows(){return DATA.map(row=>({...row,manual_status:(state[row.review_key]||{}).status||'',manual_notes:(state[row.review_key]||{}).notes||''}))}function exportRows(type){const rows=reviewedRows();let blob,name;if(type==='json'){const payload={schema_version:1,review_type:'oocyte_precision_review',identity:IDENTITY,exported_at:new Date().toISOString(),rows};blob=new Blob([JSON.stringify(payload,null,2)],{type:'application/json'});name=IDENTITY.sample_id+'_oocyte_review.json'}else{const keys=Object.keys(rows[0]||{}),esc=v=>'"'+String(v??'').replaceAll('"','""')+'"';blob=new Blob([[keys.join(','),...rows.map(r=>keys.map(k=>esc(r[k])).join(','))].join('\n')],{type:'text/csv'});name=IDENTITY.sample_id+'_oocyte_review.csv'}const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download=name;a.click();setTimeout(()=>URL.revokeObjectURL(a.href),0)}function validateImport(payload){if(!payload||Array.isArray(payload)||payload.schema_version!==1||payload.review_type!=='oocyte_precision_review')throw new Error('Expected an identity-bound oocyte_precision_review JSON export');for(const field of ['sample_id','source_image','source_image_size_bytes','source_image_mtime_ns','profile_fingerprint','implementation_version','candidate_table_sha256','combined_candidate_count'])if(String((payload.identity||{})[field])!==String(IDENTITY[field]))throw new Error('Review identity mismatch: '+field);if(!Array.isArray(payload.rows))throw new Error('Review rows are missing');const expected=new Map(DATA.map(row=>[row.review_key,row])),next={},seen=new Set();for(const row of payload.rows){const key=row.review_key||String(row.detection_pass||'baseline_v6')+':'+String(row.detector_component_id||'');if(seen.has(key))throw new Error('Duplicate review row: '+key);seen.add(key);const current=expected.get(key);if(!current)throw new Error('Unknown candidate: '+key);if(String(row.detector_component_id)!==String(current.detector_component_id)||String(row.detection_pass)!==String(current.detection_pass)||Number(row.center_x)!==Number(current.center_x)||Number(row.center_y)!==Number(current.center_y))throw new Error('Candidate metadata mismatch: '+key);const status=String(row.manual_status||'').toLowerCase();if(status&&!allowedStatuses.has(status))throw new Error('Invalid review status for '+key);next[key]={status,notes:String(row.manual_notes||'')}}return next}document.getElementById('import-json').onclick=()=>document.getElementById('import-json-file').click();document.getElementById('import-json-file').onchange=async event=>{const file=event.target.files[0];if(!file)return;try{state=validateImport(JSON.parse(await file.text()));cards.forEach(paint);save();apply();message('Imported '+progressCount()+' reviewed decisions from '+file.name)}catch(error){message(error.message,true)}finally{event.target.value=''}};function progressCount(){return DATA.filter(row=>allowedStatuses.has((state[row.review_key]||{}).status)).length}document.getElementById('export-csv').onclick=()=>exportRows('csv');document.getElementById('export-json').onclick=()=>exportRows('json');progress();apply(); +document.querySelectorAll('.mask-toggle').forEach(button=>button.addEventListener('click',()=>{const host=button.closest('.card-image'),masked=host.querySelector('.masked-thumbnail'),raw=host.querySelector('.raw-thumbnail');if(button.dataset.rawVisible==='true'){raw.hidden=true;masked.hidden=false;button.dataset.rawVisible='false';button.textContent='Hide mask';button.classList.remove('raw-visible');return}const reveal=()=>{masked.hidden=true;raw.hidden=false;button.dataset.rawVisible='true';button.textContent='Show mask';button.classList.add('raw-visible');button.disabled=false};if(raw.dataset.loaded==='true'){reveal();return}button.disabled=true;button.textContent='Loading raw';raw.onload=()=>{raw.dataset.loaded='true';reveal()};raw.onerror=()=>{button.textContent='Raw requires review server';button.title='Open this page through the dynamic review server';button.disabled=true};raw.src=raw.dataset.src})); +""" + + +_PRECISION_NOTE_PRESETS = ( + ("Halo artifact", "halo_artifact"), + ("True oocyte, bad mask", "true_oocyte; mask_truncated; mask_off_target"), + ("Non-oocyte tissue", "non_oocyte_tissue; irregular_bright_patch"), + ("Mask off target", "mask_off_target"), +) + + +def _candidate_card( + row: Mapping[str, Any], + order: int, + *, + patch_radius_px: int, +) -> str: + card_id = html.escape(str(row["html_id"]), quote=True) + display_id = html.escape(str(row["display_id"])) + candidate_id = html.escape(str(row["detector_component_id"])) + detection_pass = str(row["detection_pass"]) + rescue_rule = str(row.get("rescue_acceptance_rule", "")) + rescue_labels = { + "bright_irregular": "Rescue irregular", + "bright_fragment": "Rescue fragment", + "baseline_shape_fallback": "P99 fallback", + } + pass_label = rescue_labels.get( + rescue_rule, + "Rescue P95" if detection_pass == "secondary_rescue" else "Baseline v6", + ) + badge_class = "rescue" if detection_pass == "secondary_rescue" else "" + quality = str(row["quality_class"]) + search = html.escape( + f"{display_id} {candidate_id} {detection_pass} {row.get('reference_class', '')}".lower(), + quote=True, + ) + image_path = html.escape(str(row["thumbnail_path"]), quote=True) + raw_path = html.escape( + "api/patch.webp?" + f"x={int(round(float(row['center_x'])))}&" + f"y={int(round(float(row['center_y'])))}&" + f"radius={int(patch_radius_px)}&contrast=local", + quote=True, + ) + review_key = html.escape(_precision_review_key(row), quote=True) + note_presets = "".join( + '" + for label, value in _PRECISION_NOTE_PRESETS + ) + return f""" +
    +
    + Raw UCHL1 and exact mask for {display_id} + + +
    +
    +

    {display_id}

    {pass_label}
    +
    {candidate_id}
    +
    + score {_format_number(row.get('detector_score'),3)}d {_format_number(row.get('local_equivalent_diameter_um'),1)} um + circ {_format_number(row.get('local_circularity'),2)}solid {_format_number(row.get('local_solidity'),2)} + x {int(round(float(row['center_x'])))}y {int(round(float(row['center_y'])))} + {html.escape(str(row.get('reference_class','unavailable')))}ref {_format_number(row.get('nearest_reference_distance_px'),0)} px +
    +
    +
    {note_presets}
    + +
    +
    """ + + +def _sample_html( + sample_id: str, + candidates: pd.DataFrame, + image_shape_yx: Tuple[int, int], + *, + combined_labels_available: bool, + patch_radius_px: int, + review_identity: Mapping[str, Any], +) -> str: + baseline_count = int((candidates["detection_pass"] == "baseline_v6").sum()) if not candidates.empty else 0 + rescue_count = int((candidates["detection_pass"] == "secondary_rescue").sum()) if not candidates.empty else 0 + flagged_count = int((candidates["quality_class"] == "review-priority").sum()) if not candidates.empty else 0 + cards = "".join( + _candidate_card(row, index, patch_radius_px=patch_radius_px) + for index, row in enumerate(candidates.to_dict("records"), start=1) + ) + if not cards: + cards = '

    No machine-accepted oocytes

    This is a detector result, not evidence that the tissue contains no oocytes.

    ' + payload = { + "schema_version": PRECISION_REVIEW_SCHEMA_VERSION, + "review_type": "oocyte_precision_review_workspace", + "identity": dict(review_identity), + "rows": _precision_review_rows(candidates), + } + payload_json = json.dumps(payload, separators=(",", ":")).replace("Combined OME labels' + if combined_labels_available + else "" + ) + return f"""{html.escape(sample_id)} oocyte review +
    Aegle / raw UCHL1 / biological review

    {html.escape(sample_id)} oocyte atlas

    Every cyan contour is loaded from the exact persisted mask used by the detector. Baseline v6 candidates are retained unchanged; secondary-rescue candidates recover crowded fields using robust P95 background estimation and mask-overlap deduplication.

    {labels_link}
    {len(candidates)}combined
    {baseline_count}baseline v6
    {rescue_count}rescue delta
    {flagged_count}review priority
    + +

    Whole-slide position

    baseline v6secondary rescue
    {spatial}
    {cards}
    Review state is stored in this browser only until exported. Coordinates are full-resolution image pixels; physical scale is 0.5 um/px.
    """ + + +def _config_table(config: OocyteDetectionConfig) -> str: + rows = [] + for section, values in config.to_dict().items(): + if isinstance(values, dict): + for key, value in values.items(): + rows.append( + f"{html.escape(section)}{html.escape(str(key))}{html.escape(str(value))}" + ) + return "".join(rows) + + +def algorithm_document_html() -> str: + """Return the complete standalone detector algorithm document.""" + + v6_rows = _config_table(DONOR13_V6) + rescue_rows = _config_table(DONOR13_V6_RESCUE_V1) + return f"""Aegle raw-UCHL1 oocyte detector
    Aegle standalone scientific imaging module

    Raw-UCHL1 oocyte detection

    A detector and mask generator designed for ovarian oocytes that are fragmented by general-purpose DeepCell/Mesmer segmentation. It requires the registered raw fluorescence image and UCHL1 channel metadata, not DeepCell labels, nuclei, cell tables, or a legacy candidate list.

    UCHL1signal substrate
    2-passv6 + rescue
    exactpersisted masks
    +

    1. Contract and rationale

    Oocytes are much larger than surrounding cells and can contain a dark germinal vesicle. DeepCell therefore splits one biological oocyte into several labels or leaves a nuclear hole. The standalone module instead uses the roughly 300-fold UCHL1 contrast, detects candidate locations over the whole slide, then performs local intensity segmentation at native resolution.

    • Input: one CYX registered OME-TIFF, resolved UCHL1 channel index, pixel size, sample ID, and named profile.
    • Required external segmentation: none.
    • Output: candidate CSV, exact cropped NPZ masks, tiled uint16 whole-slide OME label image, mapping table, review/QC artifacts, and HTML review page.
    • Legacy references affect comparison displays only and never affect detection, masks, scores, or acceptance.
    +

    2. End-to-end flow

    Raw OME-TIFFresolve UCHL1Strip mean map8x downsampleCoarse proposalscomponents + peaks + LoGNative local refinementmulti-seed + compact retryScore + v6 acceptanceshape / size / contrastSecondary rescueP95 + all componentsMask-overlap dedupactual component centroidPersistNPZ + OMEMontage + spatial QC + filterable per-sample HTML biological reviewall visualizations load the persisted scored mask
    +

    3. Whole-slide proposal generation

    The reader streams horizontal strips from the selected CYX channel. Each 8 x 8 block is reduced to its mean, avoiding a full-resolution in-memory copy. A log-transformed difference of Gaussians separates compact bright objects from broad tissue background. Candidate seeds are the union of thresholded connected components, component-local intensity peaks, globally separated peaks, and Laplacian-of-Gaussian blobs. Physical diameter limits and 60 px seed merging suppress puncta and redundant proposals while retaining several peaks in crowded follicles.

    +

    4. Native-resolution v6 segmentation

    patch + annulusGaussian smoothtriangle thresholdmax(base, annulus P99)remove smallclose + fill holesselect componentexact boolean mask

    Each coarse proposal generates a coarse seed, sharp local peaks, broad peaks, and offset-ring seeds. Default and compact contexts are evaluated, with one component-centroid reseed when appropriate. The center component wins; when a dark germinal vesicle leaves the seed in background, an area-over-distance fallback selects the nearby large component. Morphology removes small objects, closes short gaps, and fills enclosed nuclear holes.

    score = 0.35 brightness + 0.30 shape + 0.20 size + 0.15 centeredness

    Shape combines circularity and solidity. Size uses a soft physical range with a 25–80 um core. Acceptance retains the original v6 strict threshold and its conservative morphology rescue. Global v6 deduplication remains frozen for numerical reproducibility.

    +

    5. Crowded-field secondary rescue

    P99 is precise for isolated oocytes, but a neighboring oocyte inside the background annulus can raise the floor until the target becomes a small fragment. The separate donor13_v6_rescue_v1 profile never alters a v6 acceptance. It retries rejected v6 seeds in their original default or compact context at annulus P95, and independently enumerates all plausible P95 components in a 240 px discovery window around each coarse proposal. Every discovery is recentered and segmented again at native resolution.

    If both default and compact P95 select a distant fallback component, one compact P80 evaluation is permitted. This is a targeted crowded-field escape hatch, not a global lower threshold. The rescue can also retain the original P99 mask when P95 merges neighbors. High-intensity irregular and bright-fragment rules are separately named and always marked review-priority.

    Rescue candidates must satisfy physical size, circularity, solidity, score, local contrast, native intensity, and centroid-offset gates. Deduplication compares the actual persisted component masks: a rescue is suppressed when at least 25% of the smaller mask overlaps an accepted v6 or higher-ranked rescue mask. Seed coordinates are not used as the biological object center.

    +

    6. Persistence and review invariants

    • Each candidate NPZ stores a boolean cropped mask, image-space bounding box, source shape, candidate ID, profile fingerprint, implementation version, and metrics.
    • The OME label image is recomposed only from accepted NPZ masks in descending score order; overlap pixels retain the higher-scoring label.
    • Montages and HTML thumbnails load persisted masks. A display can never silently rerun thresholding with newer code.
    • Thumbnail cache reuse requires a matching raw-source stat, channel, center, radius, renderer version, exact NPZ SHA-256, and output WebP SHA-256; display-rank changes therefore invalidate stale assets.
    • Precision browser state uses stable detector keys plus the candidate-table SHA-256. Identity-bound JSON can be imported or exported, and never mutates detector output.
    • Reference-linked and detector-only labels are review metadata, not training truth.
    +

    7. Validation status and limitations

    The frozen v6 profile reproduces donor13 sample counts and numerical rows, including the representative 13-23/#680 mask. Rescue is an explicitly review-gated extension: the first-cohort delta was inspected card by card and reference-backed donor13 failures were used only as diagnostic evidence, never as proposal input. Donor11 executes safely but has not yet received biological examples or a validated donor-specific intensity regime. Open-ring oocytes with a very large dark germinal vesicle can still yield an incomplete crescent mask when the fluorescent rim is not closed; these remain review-priority rather than being silently completed by an assumed circle.

    +

    8. Frozen profile parameters

    donor13_v6

    {v6_rows}
    sectionparametervalue

    donor13_v6_rescue_v1

    {rescue_rows}
    sectionparametervalue
    """ + + +def _load_combined_candidates( + sample_id: str, + sample_dir: Path, + rescue_delta_dir: Path | None, +) -> pd.DataFrame: + baseline = pd.read_csv(sample_dir / "candidates.csv") + baseline = baseline[baseline["accepted"].astype(bool)].copy() + if "segmentation_pass" in baseline.columns: + baseline["detection_pass"] = np.where( + baseline["segmentation_pass"].astype(str) == "secondary_rescue", + "secondary_rescue", + "baseline_v6", + ) + else: + baseline["detection_pass"] = "baseline_v6" + baseline["mask_source_dir"] = str(sample_dir) + tables = [baseline] + if rescue_delta_dir is not None: + rescue_path = rescue_delta_dir / sample_id / "candidates.csv" + if rescue_path.is_file(): + rescue = pd.read_csv(rescue_path) + rescue = rescue[rescue["accepted"].astype(bool)].copy() + rescue["detection_pass"] = "secondary_rescue" + rescue["mask_source_dir"] = str(rescue_path.parent) + tables.append(rescue) + combined = pd.concat(tables, ignore_index=True, sort=False) + persisted_masks = {} + for row in combined.to_dict("records"): + candidate_id = str(row["detector_component_id"]) + persisted_masks[candidate_id] = load_candidate_mask( + Path(str(row["mask_source_dir"])) / str(row["mask_path"]) + ) + combined, suppressed = suppress_accepted_mask_duplicates( + combined, + persisted_masks, + overlap_fraction=0.25, + max_centroid_distance_px=125.0, + ) + suppressed.to_csv(sample_dir / "combined_duplicate_suppressed.csv", index=False) + combined = combined[combined["accepted"].astype(bool)].copy() + combined = combined.sort_values( + ["detector_score", "center_y", "center_x"], + ascending=[False, True, True], + ).reset_index(drop=True) + combined["display_id"] = [f"#{index:03d}" for index in range(1, len(combined) + 1)] + combined["html_id"] = [f"oocyte-{index:04d}" for index in range(1, len(combined) + 1)] + combined["quality_class"] = [ + _candidate_quality(row) for row in combined.to_dict("records") + ] + return combined + + +def generate_html_reports( + batch_dir: Path, + *, + rescue_delta_dir: Path | None = None, + references_path: Path | None = None, + patch_radius_px: int = 180, + export_combined_labels: bool = True, +) -> HtmlReportResult: + """Generate the algorithm document, batch index, and one review page per sample.""" + + root = Path(batch_dir).resolve() + rescue_root = None if rescue_delta_dir is None else Path(rescue_delta_dir).resolve() + summary = pd.read_csv(root / "batch_summary.csv") + algorithm_path = root / "oocyte_detection_algorithm.html" + algorithm_path.write_text(algorithm_document_html()) + sample_pages: Dict[str, Path] = {} + index_rows = [] + for summary_row in summary.to_dict("records"): + if str(summary_row["status"]) != "complete": + continue + sample_id = str(summary_row["sample_id"]) + LOGGER.info("HTML report starting sample %s", sample_id) + sample_dir = root / sample_id + manifest = json.loads((sample_dir / "run_manifest.json").read_text()) + image_path = Path(manifest["source_image"]) + channel_index = int(manifest["resolved_channel_index"]) + sample_summary = json.loads((sample_dir / "summary.json").read_text()) + image_shape = tuple(int(value) for value in sample_summary["image_shape_yx"]) + candidates = _load_combined_candidates(sample_id, sample_dir, rescue_root) + candidates = _annotate_reference_distance( + candidates, + _load_references(references_path, sample_id), + ) + rescue_count = int( + (candidates["detection_pass"] == "secondary_rescue").sum() + ) if not candidates.empty else 0 + combined_label_path = sample_dir / "oocyte_labels_rescue_v1.ome.tiff" + should_export_combined_labels = bool( + export_combined_labels and rescue_root is not None and rescue_count > 0 + ) + combined_labels_available = bool( + rescue_root is not None + and rescue_count > 0 + and (should_export_combined_labels or combined_label_path.is_file()) + ) + if should_export_combined_labels: + combined_export = candidates.copy() + combined_export["mask_path"] = [ + str(Path(str(row["mask_source_dir"])) / str(row["mask_path"])) + for row in combined_export.to_dict("records") + ] + combined_export.to_csv( + sample_dir / "candidates_rescue_v1_combined.csv", + index=False, + ) + export_whole_slide_labels( + combined_export, + sample_dir=sample_dir, + image_shape_yx=image_shape, + image_path=combined_label_path, + mapping_path=sample_dir / "oocyte_labels_rescue_v1.csv", + ) + assets_dir = sample_dir / "html_assets" + assets_dir.mkdir(parents=True, exist_ok=True) + thumbnail_manifest_path = assets_dir / "manifest.json" + thumbnail_manifest = _load_thumbnail_manifest(thumbnail_manifest_path) + previous_entries = thumbnail_manifest.get("entries", {}) + expected_assets = { + f"{html_id}.webp" for html_id in candidates["html_id"].astype(str) + } + for existing_asset in assets_dir.glob("*.webp"): + if existing_asset.name not in expected_assets: + existing_asset.unlink() + thumbnail_paths = [ + str(Path("html_assets") / f"{html_id}.webp") + for html_id in candidates["html_id"].astype(str) + ] + source_stat = image_path.stat() + current_entries: Dict[str, Dict[str, str]] = {} + render_jobs = [] + if not candidates.empty: + for row in candidates.to_dict("records"): + asset_name = f"{row['html_id']}.webp" + asset_path = assets_dir / asset_name + source_dir = Path(str(row["mask_source_dir"])) + mask_path = source_dir / str(row["mask_path"]) + fingerprint = _thumbnail_fingerprint( + row, + source_image=image_path, + source_size_bytes=int(source_stat.st_size), + source_mtime_ns=int(source_stat.st_mtime_ns), + channel_index=channel_index, + patch_radius_px=patch_radius_px, + mask_path=mask_path, + ) + previous_entry = previous_entries.get(asset_name) + if _thumbnail_is_current(asset_path, previous_entry, fingerprint): + current_entries[asset_name] = { + "render_fingerprint": fingerprint, + "asset_sha256": str(previous_entry["asset_sha256"]), + } + else: + render_jobs.append((row, asset_name, mask_path, fingerprint)) + if render_jobs: + with tifffile.TiffFile(image_path) as tif: + array = zarr.open(tif.series[0].aszarr(), mode="r") + for row, asset_name, mask_path, fingerprint in render_jobs: + destination = assets_dir / asset_name + persisted = load_candidate_mask(mask_path) + patch = extract_cyx_channel_patch( + array, + channel_index, + ( + int(round(float(row["center_x"]))), + int(round(float(row["center_y"]))), + ), + patch_radius_px, + ) + _raw_mask_thumbnail(patch, persisted, destination) + current_entries[asset_name] = { + "render_fingerprint": fingerprint, + "asset_sha256": _file_sha256(destination), + } + _atomic_write_json( + thumbnail_manifest_path, + { + "schema_version": THUMBNAIL_CACHE_SCHEMA_VERSION, + "renderer": THUMBNAIL_RENDER_VERSION, + "entries": current_entries, + }, + ) + candidates["thumbnail_path"] = thumbnail_paths + candidate_table_path = sample_dir / "html_candidates.csv" + candidates.to_csv(candidate_table_path, index=False) + precision_identity = _precision_review_identity( + sample_id=sample_id, + source_image=image_path, + manifest=manifest, + sample_summary=sample_summary, + candidate_table_path=candidate_table_path, + candidate_count=len(candidates), + ) + page_path = sample_dir / "oocytes.html" + page_path.write_text( + _sample_html( + sample_id, + candidates, + image_shape, + combined_labels_available=combined_labels_available, + patch_radius_px=patch_radius_px, + review_identity=precision_identity, + ) + ) + sample_pages[sample_id] = page_path + baseline_count = int((candidates["detection_pass"] == "baseline_v6").sum()) if not candidates.empty else 0 + index_rows.append((sample_id, len(candidates), baseline_count, rescue_count, page_path)) + LOGGER.info( + "HTML report completed sample %s: %s candidates", + sample_id, + len(candidates), + ) + + cards = "".join( + f'sample{html.escape(sample_id)}{total} total / {baseline} baseline / {rescue} rescue' + for sample_id, total, baseline, rescue, path in index_rows + ) + index_path = root / "oocyte_review_index.html" + index_path.write_text( + f'Oocyte review index
    Aegle raw-UCHL1 cohort review

    Oocyte review index

    Choose a sample to review exact detector masks in full-resolution image coordinates.

    Read algorithm
    {cards}
    Machine counts require biological review; zero candidates do not prove biological absence.
    ' + ) + return HtmlReportResult( + batch_dir=root, + algorithm_document=algorithm_path, + batch_index=index_path, + sample_pages=sample_pages, + ) + + +__all__ = [ + "HtmlReportResult", + "algorithm_document_html", + "generate_html_reports", +] diff --git a/aegle/oocyte/rescue.py b/aegle/oocyte/rescue.py new file mode 100644 index 0000000..e86b546 --- /dev/null +++ b/aegle/oocyte/rescue.py @@ -0,0 +1,801 @@ +"""Secondary crowded-field recovery for the raw-UCHL1 detector.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Mapping, Sequence, Tuple + +import numpy as np +import pandas as pd + +from .config import OocyteDetectionConfig, SecondaryRescueConfig +from .models import ExtractedPatch, ScoredCandidateMask +from .segmentation import ( + _segment_oocyte_patch, + _segment_oocyte_patch_components, +) + + +ScoreFunction = Callable[..., float] +PatchReader = Callable[[int, int, int], ExtractedPatch] + + +@dataclass(frozen=True) +class SecondaryRescueResult: + candidates: pd.DataFrame + candidate_masks: Dict[str, ScoredCandidateMask] + diagnostics: pd.DataFrame + + +def _annulus_percentiles( + image: np.ndarray, + inner_px: int, + outer_px: int, + brightness_percentile: float, +) -> Tuple[float, float, float]: + center_y = image.shape[0] // 2 + center_x = image.shape[1] // 2 + yy, xx = np.ogrid[: image.shape[0], : image.shape[1]] + distance = np.sqrt((yy - center_y) ** 2 + (xx - center_x) ** 2) + values = image[(distance >= inner_px) & (distance <= outer_px)] + if values.size == 0: + raise ValueError("rescue background annulus does not intersect the patch") + return ( + float(np.percentile(values, 95)), + float(np.percentile(values, 99)), + float(np.percentile(values, brightness_percentile)), + ) + + +def _mask_overlap_fraction( + left: Any, + right: Any, +) -> float: + x0 = max(left.bbox.x0, right.bbox.x0) + y0 = max(left.bbox.y0, right.bbox.y0) + x1 = min(left.bbox.x1, right.bbox.x1) + y1 = min(left.bbox.y1, right.bbox.y1) + if x0 >= x1 or y0 >= y1: + return 0.0 + left_crop = left.mask[ + y0 - left.bbox.y0 : y1 - left.bbox.y0, + x0 - left.bbox.x0 : x1 - left.bbox.x0, + ] + right_crop = right.mask[ + y0 - right.bbox.y0 : y1 - right.bbox.y0, + x0 - right.bbox.x0 : x1 - right.bbox.x0, + ] + intersection = int(np.logical_and(left_crop, right_crop).sum()) + if intersection == 0: + return 0.0 + denominator = min(int(left.mask.sum()), int(right.mask.sum())) + return float(intersection / max(denominator, 1)) + + +def suppress_accepted_mask_duplicates( + candidates: pd.DataFrame, + candidate_masks: Mapping[str, Any], + *, + overlap_fraction: float, + max_centroid_distance_px: float, +) -> Tuple[pd.DataFrame, pd.DataFrame]: + """Mark lower-scoring accepted masks as duplicates using actual mask overlap.""" + + diagnostic_columns = [ + "detector_component_id", + "duplicate_of", + "mask_overlap_fraction_smaller", + "detector_score", + "status", + ] + output = candidates.copy() + if output.empty: + return output, pd.DataFrame(columns=diagnostic_columns) + accepted = output[output["accepted"].astype(bool)].sort_values( + ["detector_score", "detector_component_id"], + ascending=[False, True], + kind="stable", + ) + kept: List[Tuple[int, Mapping[str, Any], Any]] = [] + diagnostics = [] + + def centroid(record: Mapping[str, Any], axis: str) -> float: + value = record.get(f"component_centroid_{axis}") + if value is None or not np.isfinite(float(value)): + value = record[f"center_{axis}"] + return float(value) + + for index, row in accepted.iterrows(): + record = row.to_dict() + candidate_id = str(record["detector_component_id"]) + candidate_mask = candidate_masks[candidate_id] + duplicate_of = "" + duplicate_overlap = 0.0 + for _, kept_record, kept_mask in kept: + distance = float( + np.hypot( + centroid(record, "x") - centroid(kept_record, "x"), + centroid(record, "y") - centroid(kept_record, "y"), + ) + ) + if distance > max_centroid_distance_px: + continue + overlap = _mask_overlap_fraction(candidate_mask, kept_mask) + if overlap > duplicate_overlap: + duplicate_overlap = overlap + duplicate_of = str(kept_record["detector_component_id"]) + if duplicate_overlap >= overlap_fraction: + output.at[index, "accepted"] = False + output.at[index, "acceptance_mode"] = "mask_duplicate_suppressed" + output.at[index, "duplicate_suppressed"] = True + diagnostics.append( + { + "detector_component_id": candidate_id, + "duplicate_of": duplicate_of, + "mask_overlap_fraction_smaller": duplicate_overlap, + "detector_score": float(record["detector_score"]), + "status": "mask_duplicate_suppressed", + } + ) + continue + output.at[index, "duplicate_suppressed"] = False + kept.append((index, record, candidate_mask)) + return output, pd.DataFrame(diagnostics, columns=diagnostic_columns) + + +def _discovery_component_is_plausible( + row: Mapping[str, Any], + rescue: SecondaryRescueConfig, +) -> bool: + return bool( + rescue.discovery_min_diameter_um + <= float(row["diameter_um"]) + <= rescue.discovery_max_diameter_um + and float(row["circularity"]) >= rescue.discovery_min_circularity + and float(row["solidity"]) >= rescue.discovery_min_solidity + and float(row["offset_px"]) <= rescue.max_component_offset_px + ) + + +def _final_acceptance_reason( + row: Mapping[str, Any], + rescue: SecondaryRescueConfig, +) -> str: + checks = ( + ( + rescue.final_min_diameter_um + <= float(row["local_equivalent_diameter_um"]) + <= rescue.final_max_diameter_um, + "diameter", + ), + (float(row["local_circularity"]) >= rescue.final_min_circularity, "circularity"), + (float(row["local_solidity"]) >= rescue.final_min_solidity, "solidity"), + ( + float(row["mean_to_background_ratio"]) + >= rescue.final_min_brightness_ratio, + "brightness", + ), + ( + float(row["local_max_intensity"]) + >= rescue.final_min_max_intensity, + "absolute_intensity", + ), + (float(row["detector_score"]) >= rescue.final_score_threshold, "score"), + ( + float(row["local_centroid_offset_px"]) + <= rescue.final_max_centroid_offset_px, + "centroid_offset", + ), + ( + not ( + float(row["local_max_intensity"]) + < rescue.low_intensity_shape_max_intensity + and float(row["local_eccentricity"]) + > rescue.low_intensity_shape_max_eccentricity + ), + "low_intensity_elongation", + ), + ( + not ( + float(row["local_equivalent_diameter_um"]) + < rescue.small_candidate_diameter_um + and float(row["detector_score"]) + < rescue.small_candidate_min_score + and float(row["local_max_intensity"]) + < rescue.small_candidate_min_max_intensity + ), + "small_low_confidence", + ), + ) + failed = [name for passed, name in checks if not passed] + if not failed: + return "accepted_standard_pre_dedup" + bright_irregular = bool( + rescue.bright_irregular_min_diameter_um + <= float(row["local_equivalent_diameter_um"]) + <= rescue.bright_irregular_max_diameter_um + and float(row["local_circularity"]) + >= rescue.bright_irregular_min_circularity + and float(row["local_solidity"]) >= rescue.bright_irregular_min_solidity + and float(row["local_max_intensity"]) + >= rescue.bright_irregular_min_max_intensity + and float(row["detector_score"]) >= rescue.bright_irregular_min_score + and float(row["local_centroid_offset_px"]) + <= rescue.bright_irregular_max_centroid_offset_px + ) + if bright_irregular: + return "accepted_bright_irregular_pre_dedup" + bright_fragment = bool( + rescue.bright_fragment_min_diameter_um + <= float(row["local_equivalent_diameter_um"]) + <= rescue.bright_fragment_max_diameter_um + and float(row["local_circularity"]) + >= rescue.bright_fragment_min_circularity + and float(row["local_solidity"]) >= rescue.bright_fragment_min_solidity + and float(row["local_max_intensity"]) + >= rescue.bright_fragment_min_max_intensity + and float(row["mean_to_background_ratio"]) + >= rescue.bright_fragment_min_brightness_ratio + and float(row["detector_score"]) >= rescue.bright_fragment_min_score + and float(row["local_centroid_offset_px"]) + <= rescue.bright_fragment_max_centroid_offset_px + ) + return ( + "accepted_bright_fragment_pre_dedup" + if bright_fragment + else "failed_" + "+".join(failed) + ) + + +def _baseline_fallback_is_eligible( + row: Mapping[str, Any], + rescue: SecondaryRescueConfig, +) -> bool: + return bool( + rescue.baseline_fallback_min_diameter_um + <= float(row["local_equivalent_diameter_um"]) + <= rescue.baseline_fallback_max_diameter_um + and float(row["local_circularity"]) + >= rescue.baseline_fallback_min_circularity + and float(row["local_solidity"]) >= rescue.baseline_fallback_min_solidity + and float(row["local_max_intensity"]) + >= rescue.baseline_fallback_min_max_intensity + and float(row["local_centroid_offset_px"]) + <= rescue.baseline_fallback_max_centroid_offset_px + ) + + +def _evaluate_recentered_component( + *, + patch_reader: PatchReader, + image_shape_yx: Tuple[int, int], + source_record: Mapping[str, Any], + discovery_rank: int, + seed_source: str, + center_x: int, + center_y: int, + config: OocyteDetectionConfig, + score_candidate: ScoreFunction, +) -> Tuple[Dict[str, Any], ScoredCandidateMask] | None: + rescue = config.secondary_rescue + if rescue is None: + return None + local = config.local + if seed_source == "rejected_v6_seed": + window_radius_px = int(source_record["evaluation_window_radius_px"]) + annulus_inner_px = int(source_record["evaluation_annulus_inner_px"]) + annulus_outer_px = int(source_record["evaluation_annulus_outer_px"]) + inherited_context = str(source_record["local_context_mode"]) + elif seed_source == "coarse_component_discovery_compact": + window_radius_px = int(local.compact_window_radius_px) + annulus_inner_px = int(local.compact_annulus_inner_px) + annulus_outer_px = int(local.compact_annulus_outer_px) + inherited_context = "compact" + elif seed_source == "coarse_component_discovery_compact_relaxed": + window_radius_px = int(local.compact_window_radius_px) + annulus_inner_px = int(local.compact_annulus_inner_px) + annulus_outer_px = int(local.compact_annulus_outer_px) + inherited_context = "compact_relaxed" + else: + window_radius_px = int(local.window_radius_px) + annulus_inner_px = int(local.annulus_inner_px) + annulus_outer_px = int(local.annulus_outer_px) + inherited_context = "default" + floor_percentile = ( + rescue.compact_relaxed_annulus_floor_percentile + if inherited_context == "compact_relaxed" + else rescue.annulus_floor_percentile + ) + brightness_percentile = ( + floor_percentile + if inherited_context == "compact_relaxed" + else rescue.brightness_reference_percentile + ) + best: Tuple[Dict[str, Any], ScoredCandidateMask] | None = None + current_x = int(center_x) + current_y = int(center_y) + for reseed_iteration in range(local.max_centroid_reseed_iterations + 1): + try: + patch = patch_reader(current_x, current_y, window_radius_px) + smooth, segmentation = _segment_oocyte_patch( + patch.image, + config, + annulus_inner_px=annulus_inner_px, + annulus_outer_px=annulus_outer_px, + annulus_floor_percentile=floor_percentile, + ) + annulus_p95, annulus_p99, brightness_reference = _annulus_percentiles( + smooth, + annulus_inner_px, + annulus_outer_px, + brightness_percentile, + ) + except (IndexError, ValueError): + break + + metrics = segmentation.metrics + brightness_ratio = float( + metrics.mean_intensity / max(brightness_reference, 1.0) + ) + score = score_candidate( + equivalent_diameter_um=metrics.equivalent_diameter_um, + circularity=metrics.circularity, + solidity=metrics.solidity, + brightness_ratio=brightness_ratio, + offset_px=metrics.centroid_offset_px, + config=config, + ) + patch_center_y = smooth.shape[0] // 2 + patch_center_x = smooth.shape[1] // 2 + component_centroid_x = float( + current_x + metrics.centroid_x_px - patch_center_x + ) + component_centroid_y = float( + current_y + metrics.centroid_y_px - patch_center_y + ) + row = { + **source_record, + "source_detector_component_id": str( + source_record["detector_component_id"] + ), + "seed_center_x": int(center_x), + "seed_center_y": int(center_y), + "local_context_mode": ( + f"secondary_rescue_p{floor_percentile:g}_{inherited_context}" + ), + "evaluation_window_radius_px": window_radius_px, + "evaluation_annulus_inner_px": annulus_inner_px, + "evaluation_annulus_outer_px": annulus_outer_px, + "center_x": int(round(component_centroid_x)), + "center_y": int(round(component_centroid_y)), + "component_centroid_x": component_centroid_x, + "component_centroid_y": component_centroid_y, + "component_centroid_shift_px": float(metrics.centroid_offset_px), + "bbox_x0": patch.bbox.x0, + "bbox_y0": patch.bbox.y0, + "bbox_x1": patch.bbox.x1, + "bbox_y1": patch.bbox.y1, + "threshold_method": metrics.threshold_method, + "threshold": float(metrics.threshold), + "selection_mode": metrics.selection_mode, + "local_area_px": int(metrics.area_px), + "local_equivalent_diameter_um": float(metrics.equivalent_diameter_um), + "local_major_axis_um": float(metrics.major_axis_um), + "local_minor_axis_um": float(metrics.minor_axis_um), + "local_eccentricity": float(metrics.eccentricity), + "local_solidity": float(metrics.solidity), + "local_circularity": float(metrics.circularity), + "local_centroid_offset_px": float(metrics.centroid_offset_px), + "local_mean_intensity": float(metrics.mean_intensity), + "local_max_intensity": float(metrics.max_intensity), + "annulus_p95": annulus_p95, + "annulus_p99": annulus_p99, + "mean_to_annulus_p99_ratio": float( + metrics.mean_intensity / max(annulus_p99, 1.0) + ), + "detector_score": float(score), + "accepted_strict": False, + "accepted_rescue": True, + "acceptance_mode": "secondary_rescue", + "accepted": True, + "seed_source": "secondary_rescue_component", + "seed_rank": int(discovery_rank), + "seed_shift_px": float( + np.hypot( + center_x - float(source_record["coarse_center_x"]), + center_y - float(source_record["coarse_center_y"]), + ) + ), + "local_reseed_iteration": int(reseed_iteration), + "local_instance_rank": int(discovery_rank), + "local_instance_count_from_coarse": 0, + "segmentation_pass": "secondary_rescue", + "score_background_percentile": float( + brightness_percentile + ), + "mean_to_background_ratio": brightness_ratio, + "rescue_discovery_center_x": int(center_x), + "rescue_discovery_center_y": int(center_y), + "rescue_discovery_rank": int(discovery_rank), + "rescue_seed_source": str(seed_source), + } + cropped_mask = patch.crop_to_image_bounds(segmentation.mask) + candidate_mask = ScoredCandidateMask( + mask=np.asarray(cropped_mask, dtype=np.bool_).copy(), + bbox=patch.bbox, + image_shape_yx=image_shape_yx, + metrics=metrics, + ) + evaluated = (row, candidate_mask) + if best is None or ( + float(row["detector_score"]), + float(row["local_circularity"]), + -float(row["local_centroid_offset_px"]), + ) > ( + float(best[0]["detector_score"]), + float(best[0]["local_circularity"]), + -float(best[0]["local_centroid_offset_px"]), + ): + best = evaluated + + shift = float(metrics.centroid_offset_px) + if reseed_iteration >= local.max_centroid_reseed_iterations: + break + if not local.centroid_reseed_min_shift_px <= shift <= ( + local.centroid_reseed_max_shift_px + ): + break + next_x = int(round(component_centroid_x)) + next_y = int(round(component_centroid_y)) + if np.hypot(next_x - current_x, next_y - current_y) <= 1.0: + break + current_x = next_x + current_y = next_y + return best + + +def run_secondary_rescue( + *, + patch_reader: PatchReader, + image_shape_yx: Tuple[int, int], + coarse_candidates: pd.DataFrame, + baseline_candidates: pd.DataFrame, + baseline_masks: Mapping[str, ScoredCandidateMask], + config: OocyteDetectionConfig, + score_candidate: ScoreFunction, +) -> SecondaryRescueResult: + """Discover extra components, recenter them, and retain only nonduplicate rescues.""" + + rescue = config.secondary_rescue + if rescue is None: + return SecondaryRescueResult( + candidates=baseline_candidates.copy(), + candidate_masks=dict(baseline_masks), + diagnostics=pd.DataFrame(), + ) + + baseline_candidates, baseline_duplicate_diagnostics = ( + suppress_accepted_mask_duplicates( + baseline_candidates, + baseline_masks, + overlap_fraction=rescue.duplicate_mask_overlap_fraction, + max_centroid_distance_px=2.5 * rescue.duplicate_centroid_distance_px, + ) + ) + accepted_existing: List[Tuple[str, Mapping[str, Any], ScoredCandidateMask]] = [] + for row in baseline_candidates.to_dict("records"): + if bool(row.get("accepted", False)): + candidate_id = str(row["detector_component_id"]) + accepted_existing.append((candidate_id, row, baseline_masks[candidate_id])) + + evaluated_centers: List[Tuple[float, float]] = [] + evaluated: List[Tuple[Dict[str, Any], ScoredCandidateMask]] = [] + diagnostics: List[Dict[str, Any]] = baseline_duplicate_diagnostics.to_dict( + "records" + ) + + def evaluate_seed( + source_record: Mapping[str, Any], + *, + center_x: float, + center_y: float, + discovery_rank: int, + seed_source: str, + ) -> bool: + result = _evaluate_recentered_component( + patch_reader=patch_reader, + image_shape_yx=image_shape_yx, + source_record=source_record, + discovery_rank=discovery_rank, + seed_source=seed_source, + center_x=int(round(center_x)), + center_y=int(round(center_y)), + config=config, + score_candidate=score_candidate, + ) + if result is None: + return False + row, candidate_mask = result + row["rescue_evaluation_id"] = f"eval_{len(diagnostics):06d}" + row["rescue_status"] = _final_acceptance_reason(row, rescue) + rule_by_status = { + "accepted_bright_irregular_pre_dedup": "bright_irregular", + "accepted_bright_fragment_pre_dedup": "bright_fragment", + } + row["rescue_acceptance_rule"] = rule_by_status.get( + row["rescue_status"], + "standard", + ) + if row["rescue_acceptance_rule"] != "standard": + row["acceptance_mode"] = ( + f"secondary_rescue_{row['rescue_acceptance_rule']}" + ) + diagnostics.append(dict(row)) + if not str(row["rescue_status"]).endswith("_pre_dedup"): + return False + evaluated.append((row, candidate_mask)) + return True + + rejected = baseline_candidates[ + ~baseline_candidates["accepted"].astype(bool) + ].sort_values("detector_score", ascending=False) + for source_record in rejected.to_dict("records"): + source_id = str(source_record["detector_component_id"]) + if _baseline_fallback_is_eligible(source_record, rescue): + fallback_row = { + **source_record, + "source_detector_component_id": source_id, + "center_x": int(round(float(source_record["component_centroid_x"]))), + "center_y": int(round(float(source_record["component_centroid_y"]))), + "accepted_strict": False, + "accepted_rescue": True, + "acceptance_mode": "secondary_rescue_baseline_shape_fallback", + "accepted": True, + "seed_source": "rejected_v6_mask_fallback", + "segmentation_pass": "secondary_rescue", + "score_background_percentile": 99.0, + "mean_to_background_ratio": float( + source_record["mean_to_annulus_p99_ratio"] + ), + "rescue_discovery_center_x": int(source_record["seed_center_x"]), + "rescue_discovery_center_y": int(source_record["seed_center_y"]), + "rescue_discovery_rank": 0, + "rescue_seed_source": "rejected_v6_mask_fallback", + "rescue_evaluation_id": f"eval_{len(diagnostics):06d}", + "rescue_status": "accepted_baseline_shape_fallback_pre_dedup", + "rescue_acceptance_rule": "baseline_shape_fallback", + } + diagnostics.append(dict(fallback_row)) + evaluated.append((fallback_row, baseline_masks[source_id])) + center_x = float(source_record["seed_center_x"]) + center_y = float(source_record["seed_center_y"]) + accepted = evaluate_seed( + source_record, + center_x=center_x, + center_y=center_y, + discovery_rank=1, + seed_source="rejected_v6_seed", + ) + if accepted: + evaluated_centers.append((center_x, center_y)) + + for source_record in coarse_candidates.to_dict("records"): + coarse_x = int(source_record["coarse_center_x"]) + coarse_y = int(source_record["coarse_center_y"]) + try: + patch = patch_reader( + coarse_x, + coarse_y, + rescue.discovery_window_radius_px, + ) + _, components = _segment_oocyte_patch_components( + patch.image, + config, + annulus_inner_px=rescue.discovery_annulus_inner_px, + annulus_outer_px=rescue.discovery_annulus_outer_px, + annulus_floor_percentile=rescue.annulus_floor_percentile, + ) + except (IndexError, ValueError): + continue + + discovery_rows = [] + patch_center_y = patch.image.shape[0] // 2 + patch_center_x = patch.image.shape[1] // 2 + for component in components: + metrics = component.metrics + component_x = float(coarse_x + metrics.centroid_x_px - patch_center_x) + component_y = float(coarse_y + metrics.centroid_y_px - patch_center_y) + discovery_rows.append( + { + "component": component, + "center_x": component_x, + "center_y": component_y, + "diameter_um": float(metrics.equivalent_diameter_um), + "circularity": float(metrics.circularity), + "solidity": float(metrics.solidity), + "offset_px": float(metrics.centroid_offset_px), + "mean_intensity": float(metrics.mean_intensity), + } + ) + plausible = [ + row + for row in discovery_rows + if _discovery_component_is_plausible(row, rescue) + ] + plausible.sort( + key=lambda row: ( + float(row["mean_intensity"]), + float(row["diameter_um"]), + float(row["circularity"]), + ), + reverse=True, + ) + plausible = plausible[: rescue.max_components_per_coarse_candidate] + for discovery_rank, discovery in enumerate(plausible, start=1): + center_x = float(discovery["center_x"]) + center_y = float(discovery["center_y"]) + repeated = any( + np.hypot(center_x - prior_x, center_y - prior_y) + <= rescue.discovery_seed_merge_distance_px + for prior_x, prior_y in evaluated_centers + ) + if repeated: + diagnostics.append( + { + "source_detector_component_id": str( + source_record["detector_component_id"] + ), + "rescue_discovery_center_x": center_x, + "rescue_discovery_center_y": center_y, + "rescue_discovery_rank": discovery_rank, + "rescue_seed_source": "coarse_component_discovery", + "rescue_status": "duplicate_discovery_seed", + } + ) + continue + evaluated_centers.append((center_x, center_y)) + diagnostic_count_before = len(diagnostics) + accepted = evaluate_seed( + source_record, + center_x=center_x, + center_y=center_y, + discovery_rank=discovery_rank, + seed_source="coarse_component_discovery", + ) + latest = diagnostics[-1] if len(diagnostics) > diagnostic_count_before else None + should_retry_compact = bool( + not accepted + and ( + latest is None + or str(latest.get("selection_mode", "")) + != "center_component" + or float(latest.get("local_centroid_offset_px", 0.0)) > 80.0 + ) + ) + if should_retry_compact: + compact_diagnostic_count_before = len(diagnostics) + compact_accepted = evaluate_seed( + source_record, + center_x=center_x, + center_y=center_y, + discovery_rank=discovery_rank, + seed_source="coarse_component_discovery_compact", + ) + latest_compact = ( + diagnostics[-1] + if len(diagnostics) > compact_diagnostic_count_before + else None + ) + should_retry_relaxed = bool( + not compact_accepted + and ( + latest_compact is None + or str(latest_compact.get("selection_mode", "")) + != "center_component" + or float( + latest_compact.get("local_centroid_offset_px", 0.0) + ) + > 80.0 + ) + ) + if should_retry_relaxed: + evaluate_seed( + source_record, + center_x=center_x, + center_y=center_y, + discovery_rank=discovery_rank, + seed_source=( + "coarse_component_discovery_compact_relaxed" + ), + ) + + evaluated.sort( + key=lambda item: ( + float(item[0]["detector_score"]), + float(item[0]["local_circularity"]), + float(item[0]["local_equivalent_diameter_um"]), + ), + reverse=True, + ) + kept: List[Tuple[str, Dict[str, Any], ScoredCandidateMask]] = [] + for row, candidate_mask in evaluated: + duplicate_id = "" + duplicate_fraction = 0.0 + comparison = accepted_existing + kept + for existing_id, existing_row, existing_mask in comparison: + centroid_distance = float( + np.hypot( + float(row["component_centroid_x"]) + - float(existing_row["component_centroid_x"]), + float(row["component_centroid_y"]) + - float(existing_row["component_centroid_y"]), + ) + ) + if centroid_distance > 2.5 * rescue.duplicate_centroid_distance_px: + continue + overlap = _mask_overlap_fraction(candidate_mask, existing_mask) + if overlap > duplicate_fraction: + duplicate_fraction = overlap + duplicate_id = existing_id + is_duplicate = bool( + duplicate_fraction >= rescue.duplicate_mask_overlap_fraction + ) + if not is_duplicate: + for existing_id, existing_row, _ in comparison: + distance = float( + np.hypot( + float(row["component_centroid_x"]) + - float(existing_row["component_centroid_x"]), + float(row["component_centroid_y"]) + - float(existing_row["component_centroid_y"]), + ) + ) + if distance <= 8.0: + duplicate_id = existing_id + is_duplicate = True + break + diagnostic_match = next( + item + for item in diagnostics + if str(item.get("rescue_status", "")).endswith("_pre_dedup") + and item.get("rescue_evaluation_id") == row.get("rescue_evaluation_id") + ) + diagnostic_match["rescue_duplicate_of"] = duplicate_id + diagnostic_match["rescue_duplicate_overlap_fraction"] = duplicate_fraction + if is_duplicate: + diagnostic_match["rescue_status"] = "duplicate_existing_mask" + continue + candidate_id = f"rescue_{len(kept):04d}" + row["detector_component_id"] = candidate_id + row["rescue_status"] = "accepted" + row["rescue_duplicate_of"] = "" + row["rescue_duplicate_overlap_fraction"] = duplicate_fraction + diagnostic_match.update(row) + kept.append((candidate_id, row, candidate_mask)) + + baseline = baseline_candidates.copy() + if not baseline.empty: + baseline["segmentation_pass"] = "baseline_v6" + baseline["score_background_percentile"] = 99.0 + baseline["mean_to_background_ratio"] = baseline[ + "mean_to_annulus_p99_ratio" + ] + baseline["rescue_status"] = "not_applicable" + rescue_rows = pd.DataFrame([row for _, row, _ in kept]) + combined = pd.concat([baseline, rescue_rows], ignore_index=True, sort=False) + combined_masks = dict(baseline_masks) + combined_masks.update( + {candidate_id: mask for candidate_id, _, mask in kept} + ) + return SecondaryRescueResult( + candidates=combined, + candidate_masks=combined_masks, + diagnostics=pd.DataFrame(diagnostics), + ) + + +__all__ = [ + "SecondaryRescueResult", + "run_secondary_rescue", + "suppress_accepted_mask_duplicates", +] diff --git a/aegle/oocyte/review.py b/aegle/oocyte/review.py new file mode 100644 index 0000000..27c0d8b --- /dev/null +++ b/aegle/oocyte/review.py @@ -0,0 +1,672 @@ +"""Persisted-mask review montages for standalone oocyte detection outputs.""" + +from __future__ import annotations + +import json +import math +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import tifffile +import zarr + +from .io import extract_cyx_channel_patch, load_candidate_mask +from .models import ExtractedPatch, PersistedMask + + +@dataclass(frozen=True) +class ReviewPackResult: + review_dir: Path + accepted_count: int + novel_count: int | None + missed_reference_count: int | None + artifact_paths: Dict[str, Path] + + +@dataclass(frozen=True) +class _SampleSource: + sample_id: str + sample_dir: Path + image_path: Path + channel_index: int + + +def _load_sample_sources(batch_dir: Path) -> List[_SampleSource]: + summary_path = batch_dir / "batch_summary.csv" + if not summary_path.is_file(): + raise FileNotFoundError(f"batch summary not found: {summary_path}") + summary = pd.read_csv(summary_path) + sources = [] + for record in summary.to_dict("records"): + if str(record["status"]) != "complete": + continue + sample_id = str(record["sample_id"]) + sample_dir = batch_dir / sample_id + manifest_path = sample_dir / "run_manifest.json" + manifest = json.loads(manifest_path.read_text()) + sources.append( + _SampleSource( + sample_id=sample_id, + sample_dir=sample_dir, + image_path=Path(manifest["source_image"]), + channel_index=int(manifest["resolved_channel_index"]), + ) + ) + return sources + + +def _load_candidates(sources: List[_SampleSource]) -> pd.DataFrame: + tables = [] + for source in sources: + table = pd.read_csv(source.sample_dir / "candidates.csv") + table.insert(0, "sample_id", source.sample_id) + tables.append(table) + if not tables: + return pd.DataFrame() + return pd.concat(tables, ignore_index=True) + + +def _load_references( + path: Path, + sample_ids: set[str], + min_final_score: float, +) -> pd.DataFrame: + records = [] + with Path(path).open() as handle: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + record = json.loads(line) + sample_id = str(record.get("sample_id", "")) + if sample_id not in sample_ids: + continue + final_score = float(record.get("final_score", 0.0)) + if final_score < min_final_score: + continue + center = record.get("center") + if center is None or len(center) != 2: + raise ValueError( + f"reference line {line_number} has no two-coordinate center" + ) + records.append( + { + "sample_id": sample_id, + "reference_oocyte_id": int(record["oocyte_id"]), + "reference_center_x": float(center[0]), + "reference_center_y": float(center[1]), + "reference_final_score": final_score, + "reference_quality": str(record.get("quality", "")), + } + ) + return pd.DataFrame(records) + + +def _attach_reference_fields( + candidates: pd.DataFrame, + references: pd.DataFrame, + match_radius_px: float, +) -> pd.DataFrame: + output = candidates.copy() + output["nearest_reference_oocyte_id"] = np.nan + output["nearest_reference_distance_px"] = np.nan + output["nearest_reference_final_score"] = np.nan + output["nearest_reference_quality"] = "" + output["matches_reference_within_radius"] = False + if output.empty or references.empty: + return output + + for sample_id, index in output.groupby("sample_id").groups.items(): + sample_refs = references[references["sample_id"] == sample_id] + if sample_refs.empty: + continue + ref_xy = sample_refs[["reference_center_x", "reference_center_y"]].to_numpy() + candidate_xy = output.loc[index, ["center_x", "center_y"]].to_numpy() + distances = np.sqrt( + ((candidate_xy[:, None, :] - ref_xy[None, :, :]) ** 2).sum(axis=2) + ) + nearest_indices = distances.argmin(axis=1) + nearest_distances = distances[np.arange(len(index)), nearest_indices] + nearest = sample_refs.iloc[nearest_indices] + output.loc[index, "nearest_reference_oocyte_id"] = nearest[ + "reference_oocyte_id" + ].to_numpy() + output.loc[index, "nearest_reference_distance_px"] = nearest_distances + output.loc[index, "nearest_reference_final_score"] = nearest[ + "reference_final_score" + ].to_numpy() + output.loc[index, "nearest_reference_quality"] = nearest[ + "reference_quality" + ].to_numpy() + output.loc[index, "matches_reference_within_radius"] = ( + nearest_distances <= match_radius_px + ) + return output + + +def _reference_comparison( + references: pd.DataFrame, + candidates: pd.DataFrame, + match_radius_px: float, +) -> pd.DataFrame: + rows = [] + accepted = candidates[candidates["accepted"].astype(bool)] + for reference in references.to_dict("records"): + sample_candidates = candidates[candidates["sample_id"] == reference["sample_id"]] + sample_accepted = accepted[accepted["sample_id"] == reference["sample_id"]] + + def nearest(table: pd.DataFrame) -> Dict[str, Any] | None: + if table.empty: + return None + distances = np.hypot( + table["center_x"].to_numpy(dtype=float) + - reference["reference_center_x"], + table["center_y"].to_numpy(dtype=float) + - reference["reference_center_y"], + ) + position = int(distances.argmin()) + row = table.iloc[position] + return { + "component_id": str(row["detector_component_id"]), + "distance_px": float(distances[position]), + "detector_score": float(row["detector_score"]), + "accepted": bool(row["accepted"]), + } + + nearest_refined = nearest(sample_candidates) + nearest_accepted = nearest(sample_accepted) + rows.append( + { + **reference, + "nearest_refined_component_id": None + if nearest_refined is None + else nearest_refined["component_id"], + "nearest_refined_distance_px": None + if nearest_refined is None + else nearest_refined["distance_px"], + "nearest_refined_detector_score": None + if nearest_refined is None + else nearest_refined["detector_score"], + "nearest_refined_accepted": False + if nearest_refined is None + else nearest_refined["accepted"], + "matched_by_refined": bool( + nearest_refined is not None + and nearest_refined["distance_px"] <= match_radius_px + ), + "nearest_accepted_component_id": None + if nearest_accepted is None + else nearest_accepted["component_id"], + "nearest_accepted_distance_px": None + if nearest_accepted is None + else nearest_accepted["distance_px"], + "nearest_accepted_detector_score": None + if nearest_accepted is None + else nearest_accepted["detector_score"], + "matched_by_accepted": bool( + nearest_accepted is not None + and nearest_accepted["distance_px"] <= match_radius_px + ), + } + ) + return pd.DataFrame(rows) + + +def _candidate_review_table( + candidates: pd.DataFrame, + *, + novel_only: bool, + references_available: bool, +) -> pd.DataFrame: + table = candidates[candidates["accepted"].astype(bool)].copy() + if novel_only: + table = table[~table["matches_reference_within_radius"].astype(bool)].copy() + if table.empty: + return table + table = table.sort_values( + ["detector_score", "mean_to_annulus_p99_ratio"], + ascending=[False, False], + kind="stable", + ).reset_index(drop=True) + table.insert(0, "review_rank", np.arange(1, len(table) + 1)) + if references_available: + table["review_bucket"] = np.where( + table["matches_reference_within_radius"], + "near_reference", + "novel", + ) + else: + table["review_bucket"] = "accepted" + table["manual_is_oocyte"] = "" + table["manual_mask_quality"] = "" + table["manual_duplicate_group"] = "" + table["manual_notes"] = "" + return table + + +def _missed_review_table(reference_comparison: pd.DataFrame) -> pd.DataFrame: + table = reference_comparison[ + ~reference_comparison["matched_by_accepted"].astype(bool) + ].copy() + if table.empty: + return table + table = table.sort_values( + ["reference_final_score", "reference_quality"], + ascending=[False, True], + kind="stable", + ).reset_index(drop=True) + table.insert(0, "review_rank", np.arange(1, len(table) + 1)) + table["center_x"] = table["reference_center_x"].round().astype(int) + table["center_y"] = table["reference_center_y"].round().astype(int) + table["manual_true_oocyte_at_reference"] = "" + table["manual_failure_mode"] = "" + table["manual_duplicate_group"] = "" + table["manual_notes"] = "" + return table + + +def _place_mask_in_patch( + persisted: PersistedMask, + patch: ExtractedPatch, +) -> np.ndarray | None: + ix0 = max(persisted.bbox.x0, patch.bbox.x0) + iy0 = max(persisted.bbox.y0, patch.bbox.y0) + ix1 = min(persisted.bbox.x1, patch.bbox.x1) + iy1 = min(persisted.bbox.y1, patch.bbox.y1) + if ix0 >= ix1 or iy0 >= iy1: + return None + top, _, left, _ = patch.padding_tblr + overlay = np.zeros(patch.image.shape, dtype=np.bool_) + target_x0 = left + ix0 - patch.bbox.x0 + target_y0 = top + iy0 - patch.bbox.y0 + source_x0 = ix0 - persisted.bbox.x0 + source_y0 = iy0 - persisted.bbox.y0 + width = ix1 - ix0 + height = iy1 - iy0 + overlay[target_y0 : target_y0 + height, target_x0 : target_x0 + width] = ( + persisted.mask[ + source_y0 : source_y0 + height, + source_x0 : source_x0 + width, + ] + ) + return overlay if overlay.any() else None + + +def _format_float(value: Any, digits: int, missing: str = "n/a") -> str: + if value is None or pd.isna(value): + return missing + return f"{float(value):.{digits}f}" + + +def _render_pages( + table: pd.DataFrame, + *, + mode: str, + pages_dir: Path, + page_title: str, + columns: int, + rows: int, + sources: Dict[str, _SampleSource], + arrays: Dict[str, Any], + accepted_by_sample: Dict[str, pd.DataFrame], + mask_cache: Dict[Path, PersistedMask], + neighbor_overlays: bool, + max_neighbor_overlays: int, + neighbor_margin_px: float, + missed_patch_radius_px: int, +) -> pd.DataFrame: + output = table.copy() + if output.empty: + return output + pages_dir.mkdir(parents=True, exist_ok=True) + per_page = columns * rows + output["page_number"] = np.arange(len(output)) // per_page + 1 + output["panel_index"] = np.arange(len(output)) % per_page + 1 + output["page_path"] = output["page_number"].map( + lambda page: str(pages_dir / f"page_{int(page):02d}.png") + ) + rank_lookup = { + (str(row["sample_id"]), str(row["detector_component_id"])): int( + row["review_rank"] + ) + for row in output.to_dict("records") + if "detector_component_id" in row + } + + def persisted_mask(sample_dir: Path, value: Any) -> PersistedMask: + path = Path(str(value)) + if not path.is_absolute(): + path = sample_dir / path + path = path.resolve() + if path not in mask_cache: + mask_cache[path] = load_candidate_mask(path) + return mask_cache[path] + + for page_number in range(1, math.ceil(len(output) / per_page) + 1): + page = output[output["page_number"] == page_number] + fig, axes = plt.subplots( + rows, + columns, + figsize=(4.3 * columns, 4.4 * rows), + constrained_layout=True, + ) + axes_flat = np.atleast_1d(axes).ravel() + for ax in axes_flat: + ax.set_axis_off() + for ax, record in zip(axes_flat, page.to_dict("records")): + sample_id = str(record["sample_id"]) + source = sources[sample_id] + radius = ( + int(record.get("evaluation_window_radius_px", missed_patch_radius_px)) + if mode == "candidate" + else missed_patch_radius_px + ) + center_x = int(record["center_x"]) + center_y = int(record["center_y"]) + patch = extract_cyx_channel_patch( + arrays[sample_id], + source.channel_index, + (center_x, center_y), + radius, + ) + current_mask = None + if mode == "candidate": + current_mask = _place_mask_in_patch( + persisted_mask(source.sample_dir, record["mask_path"]), + patch, + ) + overlays = [] + if neighbor_overlays: + neighbors = accepted_by_sample.get(sample_id, pd.DataFrame()).copy() + if mode == "candidate" and not neighbors.empty: + neighbors = neighbors[ + neighbors["detector_component_id"] + != record["detector_component_id"] + ] + if not neighbors.empty: + distances = np.hypot( + neighbors["center_x"].to_numpy(dtype=float) - center_x, + neighbors["center_y"].to_numpy(dtype=float) - center_y, + ) + neighbors = neighbors.assign(_distance_px=distances) + neighbors = neighbors[ + neighbors["_distance_px"] <= radius + neighbor_margin_px + ].sort_values("_distance_px") + for neighbor in neighbors.head(max_neighbor_overlays).to_dict("records"): + overlay = _place_mask_in_patch( + persisted_mask(source.sample_dir, neighbor["mask_path"]), + patch, + ) + if overlay is None: + continue + rank = rank_lookup.get( + (sample_id, str(neighbor["detector_component_id"])) + ) + overlays.append((overlay, "" if rank is None else f"#{rank}")) + + ax.imshow(np.log1p(patch.image), cmap="magma") + ax.scatter( + [patch.image.shape[1] // 2], + [patch.image.shape[0] // 2], + s=24, + c="white", + marker="+", + linewidths=0.8, + ) + if current_mask is not None: + ax.contour( + current_mask.astype(np.uint8), + levels=[0.5], + colors=["cyan"], + linewidths=1.2, + ) + for overlay, label in overlays: + ax.contour( + overlay.astype(np.uint8), + levels=[0.5], + colors=["#a7f432"], + linewidths=0.9, + ) + if label: + label_y, label_x = np.argwhere(overlay).mean(axis=0) + ax.text( + float(label_x), + float(label_y), + label, + color="#a7f432", + fontsize=7, + ha="center", + va="center", + bbox={ + "boxstyle": "round,pad=0.12", + "fc": "#111111", + "ec": "none", + "alpha": 0.7, + }, + ) + rank = int(record["review_rank"]) + if mode == "candidate": + title_lines = [ + f"#{rank:03d} | {sample_id} {record['detector_component_id']}", + f"{record['review_bucket']} {record['acceptance_mode']}", + f"score={_format_float(record['detector_score'], 3)}", + f"d={_format_float(record['local_equivalent_diameter_um'], 1)}um " + f"circ={_format_float(record['local_circularity'], 3)}", + ] + if "nearest_reference_distance_px" in record: + title_lines.append( + "refdist=" + + _format_float( + record["nearest_reference_distance_px"], + 0, + missing="none", + ) + + ("px" if not pd.isna(record["nearest_reference_distance_px"]) else "") + ) + else: + title_lines = [ + f"#{rank:03d} | {sample_id} ref#{int(record['reference_oocyte_id'])}", + f"old={_format_float(record['reference_final_score'], 3)} " + f"{record['reference_quality']}", + "nearest_acc=" + + _format_float( + record.get("nearest_accepted_distance_px"), + 0, + missing="none", + ) + + ( + "px" + if not pd.isna(record.get("nearest_accepted_distance_px")) + else "" + ), + ] + ax.set_title("\n".join(title_lines), fontsize=9, color="#202124") + ax.set_axis_off() + fig.suptitle(f"{page_title} | page {page_number}", fontsize=13) + fig.savefig( + pages_dir / f"page_{page_number:02d}.png", + dpi=180, + bbox_inches="tight", + ) + plt.close(fig) + return output + + +def generate_review_pack( + batch_dir: Path, + *, + references_path: Path | None = None, + reference_min_final_score: float = 0.35, + match_radius_px: float = 100.0, + columns: int = 3, + rows: int = 4, + neighbor_overlays: bool = True, + max_neighbor_overlays: int = 8, + neighbor_margin_px: float = 40.0, + missed_patch_radius_px: int = 180, +) -> ReviewPackResult: + """Generate accepted and optional novel/missed review tables and pages.""" + + if columns <= 0 or rows <= 0: + raise ValueError("montage columns and rows must be positive") + batch_root = Path(batch_dir).resolve() + review_dir = batch_root / "review" + review_dir.mkdir(parents=True, exist_ok=True) + sources_list = _load_sample_sources(batch_root) + sources = {source.sample_id: source for source in sources_list} + candidates = _load_candidates(sources_list) + references_available = references_path is not None + if references_available: + references = _load_references( + Path(references_path), + set(sources), + reference_min_final_score, + ) + else: + references = pd.DataFrame() + candidates = _attach_reference_fields(candidates, references, match_radius_px) + accepted = _candidate_review_table( + candidates, + novel_only=False, + references_available=references_available, + ) + novel = ( + _candidate_review_table( + candidates, + novel_only=True, + references_available=True, + ) + if references_available + else pd.DataFrame() + ) + reference_comparison = ( + _reference_comparison(references, candidates, match_radius_px) + if references_available + else pd.DataFrame() + ) + missed = ( + _missed_review_table(reference_comparison) + if references_available + else pd.DataFrame() + ) + accepted_by_sample = { + sample_id: table.copy() + for sample_id, table in candidates[ + candidates["accepted"].astype(bool) + ].groupby("sample_id") + } + arrays = {} + mask_cache: Dict[Path, PersistedMask] = {} + with ExitStack() as stack: + for source in sources_list: + tif = stack.enter_context(tifffile.TiffFile(source.image_path)) + arrays[source.sample_id] = zarr.open(tif.series[0].aszarr(), mode="r") + accepted = _render_pages( + accepted, + mode="candidate", + pages_dir=review_dir / "accepted_pages", + page_title="Accepted raw-UCHL1 oocyte candidates", + columns=columns, + rows=rows, + sources=sources, + arrays=arrays, + accepted_by_sample=accepted_by_sample, + mask_cache=mask_cache, + neighbor_overlays=neighbor_overlays, + max_neighbor_overlays=max_neighbor_overlays, + neighbor_margin_px=neighbor_margin_px, + missed_patch_radius_px=missed_patch_radius_px, + ) + if references_available: + novel = _render_pages( + novel, + mode="candidate", + pages_dir=review_dir / "novel_pages", + page_title="Novel raw-UCHL1 oocyte candidates", + columns=columns, + rows=rows, + sources=sources, + arrays=arrays, + accepted_by_sample=accepted_by_sample, + mask_cache=mask_cache, + neighbor_overlays=neighbor_overlays, + max_neighbor_overlays=max_neighbor_overlays, + neighbor_margin_px=neighbor_margin_px, + missed_patch_radius_px=missed_patch_radius_px, + ) + missed = _render_pages( + missed, + mode="miss", + pages_dir=review_dir / "missed_pages", + page_title="Legacy references missed by accepted detector candidates", + columns=columns, + rows=rows, + sources=sources, + arrays=arrays, + accepted_by_sample=accepted_by_sample, + mask_cache=mask_cache, + neighbor_overlays=neighbor_overlays, + max_neighbor_overlays=max_neighbor_overlays, + neighbor_margin_px=neighbor_margin_px, + missed_patch_radius_px=missed_patch_radius_px, + ) + + accepted_path = review_dir / "accepted_candidates.csv" + novel_path = review_dir / "novel_candidates.csv" + missed_path = review_dir / "missed_references.csv" + comparison_path = review_dir / "reference_comparison.csv" + accepted.to_csv(accepted_path, index=False) + if references_available: + novel.to_csv(novel_path, index=False) + missed.to_csv(missed_path, index=False) + reference_comparison.to_csv(comparison_path, index=False) + summary_path = review_dir / "summary.json" + summary_path.write_text( + json.dumps( + { + "schema_version": 1, + "references_available": references_available, + "references_path": None + if references_path is None + else str(Path(references_path).resolve()), + "accepted_count": int(len(accepted)), + "novel_count": None if not references_available else int(len(novel)), + "missed_reference_count": None + if not references_available + else int(len(missed)), + "match_radius_px": float(match_radius_px), + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + artifacts = { + "accepted_candidates": accepted_path, + "summary": summary_path, + "accepted_pages": review_dir / "accepted_pages", + } + if references_available: + artifacts.update( + { + "novel_candidates": novel_path, + "missed_references": missed_path, + "reference_comparison": comparison_path, + "novel_pages": review_dir / "novel_pages", + "missed_pages": review_dir / "missed_pages", + } + ) + return ReviewPackResult( + review_dir=review_dir, + accepted_count=len(accepted), + novel_count=None if not references_available else len(novel), + missed_reference_count=None if not references_available else len(missed), + artifact_paths=artifacts, + ) diff --git a/aegle/oocyte/segmentation.py b/aegle/oocyte/segmentation.py new file mode 100644 index 0000000..250ec2c --- /dev/null +++ b/aegle/oocyte/segmentation.py @@ -0,0 +1,240 @@ +"""Local intensity-based segmentation of one raw UCHL1 patch.""" + +from __future__ import annotations + +from typing import List, Tuple + +import numpy as np +from scipy import ndimage as ndi +from skimage import filters, measure, morphology + +from .config import OocyteDetectionConfig +from .models import LocalSegmentationResult, SegmentationMetrics + + +def _threshold_value(image: np.ndarray, method: str) -> float: + if method == "triangle": + return float(filters.threshold_triangle(image)) + if method == "yen": + return float(filters.threshold_yen(image)) + raise ValueError(f"unsupported threshold method: {method}") + + +def _select_component( + labeled_mask: np.ndarray, + intensity_image: np.ndarray, + center_y: int, + center_x: int, +): + props = measure.regionprops(labeled_mask, intensity_image=intensity_image) + if not props: + raise ValueError("no connected components found after thresholding") + + center_label = int(labeled_mask[center_y, center_x]) + if center_label > 0: + prop = next(region for region in props if region.label == center_label) + return labeled_mask == prop.label, prop, "center_component" + + scored = [] + for prop in props: + distance = float( + np.hypot(prop.centroid[1] - center_x, prop.centroid[0] - center_y) + ) + scored.append((float(prop.area) / (1.0 + distance), prop)) + _, prop = max(scored, key=lambda item: item[0]) + return labeled_mask == prop.label, prop, "distance_weighted_fallback" + + +def _circularity(area: float, perimeter: float) -> float: + if perimeter <= 0: + return 0.0 + return float(4.0 * np.pi * area / (perimeter**2)) + + +def _annulus_values( + image: np.ndarray, + annulus_inner_px: int, + annulus_outer_px: int, +) -> Tuple[np.ndarray, int, int]: + center_y = image.shape[0] // 2 + center_x = image.shape[1] // 2 + yy, xx = np.ogrid[: image.shape[0], : image.shape[1]] + distance = np.sqrt((yy - center_y) ** 2 + (xx - center_x) ** 2) + annulus = image[ + (distance >= annulus_inner_px) + & (distance <= annulus_outer_px) + ] + if annulus.size == 0: + raise ValueError("local background annulus does not intersect the patch") + return annulus, center_y, center_x + + +def _segment_oocyte_patch( + patch: np.ndarray, + config: OocyteDetectionConfig, + *, + annulus_inner_px: int | None = None, + annulus_outer_px: int | None = None, + annulus_floor_percentile: float | None = None, +) -> Tuple[np.ndarray, LocalSegmentationResult]: + + patch_array = np.asarray(patch) + if patch_array.ndim != 2 or min(patch_array.shape) < 3: + raise ValueError("UCHL1 patch must be a non-empty two-dimensional image") + if not np.issubdtype(patch_array.dtype, np.number): + raise TypeError("UCHL1 patch must have a numeric dtype") + if not np.isfinite(patch_array).all(): + raise ValueError("UCHL1 patch contains non-finite values") + + local = config.local + inner_px = local.annulus_inner_px if annulus_inner_px is None else annulus_inner_px + outer_px = local.annulus_outer_px if annulus_outer_px is None else annulus_outer_px + if not 0 < inner_px < outer_px: + raise ValueError("annulus radii must satisfy 0 < inner < outer") + smooth = ndi.gaussian_filter( + patch_array.astype(np.float32), + sigma=local.gaussian_sigma, + ) + annulus, center_y, center_x = _annulus_values(smooth, inner_px, outer_px) + base_threshold = _threshold_value(smooth, local.threshold_method) + floor_percentile = ( + local.annulus_floor_percentile + if annulus_floor_percentile is None + else annulus_floor_percentile + ) + annulus_floor = float(np.percentile(annulus, floor_percentile)) * ( + local.annulus_floor_multiplier + ) + threshold = max(base_threshold, annulus_floor) + + binary = smooth >= threshold + binary = morphology.remove_small_objects( + binary, + min_size=local.min_component_size_px, + ) + if local.closing_radius_px > 0: + binary = ndi.binary_closing( + binary, + structure=morphology.disk(local.closing_radius_px), + ) + binary = ndi.binary_fill_holes(binary) + + labeled = measure.label(binary) + mask, prop, selection_mode = _select_component( + labeled, + smooth, + center_y, + center_x, + ) + pixel_size_um = config.pixel_size_um + metrics = SegmentationMetrics( + threshold_method=local.threshold_method, + base_threshold=base_threshold, + annulus_floor=annulus_floor, + threshold=threshold, + selection_mode=selection_mode, + area_px=int(prop.area), + equivalent_diameter_um=float(prop.equivalent_diameter_area * pixel_size_um), + major_axis_um=float(prop.axis_major_length * pixel_size_um), + minor_axis_um=float(prop.axis_minor_length * pixel_size_um), + eccentricity=float(prop.eccentricity), + solidity=float(prop.solidity), + circularity=_circularity(prop.area, prop.perimeter), + centroid_y_px=float(prop.centroid[0]), + centroid_x_px=float(prop.centroid[1]), + centroid_offset_px=float( + np.hypot(prop.centroid[1] - center_x, prop.centroid[0] - center_y) + ), + mean_intensity=float(prop.mean_intensity), + max_intensity=float(prop.max_intensity), + ) + result = LocalSegmentationResult( + mask=np.asarray(mask, dtype=np.bool_), + metrics=metrics, + ) + return smooth, result + + +def _segment_oocyte_patch_components( + patch: np.ndarray, + config: OocyteDetectionConfig, + *, + annulus_inner_px: int, + annulus_outer_px: int, + annulus_floor_percentile: float, +) -> Tuple[np.ndarray, List[LocalSegmentationResult]]: + """Segment every thresholded component for secondary crowded-field discovery.""" + + patch_array = np.asarray(patch) + if patch_array.ndim != 2 or min(patch_array.shape) < 3: + raise ValueError("UCHL1 patch must be a non-empty two-dimensional image") + if not np.issubdtype(patch_array.dtype, np.number): + raise TypeError("UCHL1 patch must have a numeric dtype") + if not np.isfinite(patch_array).all(): + raise ValueError("UCHL1 patch contains non-finite values") + if not 0 < annulus_inner_px < annulus_outer_px: + raise ValueError("annulus radii must satisfy 0 < inner < outer") + + local = config.local + smooth = ndi.gaussian_filter( + patch_array.astype(np.float32), + sigma=local.gaussian_sigma, + ) + annulus, center_y, center_x = _annulus_values( + smooth, + annulus_inner_px, + annulus_outer_px, + ) + base_threshold = _threshold_value(smooth, local.threshold_method) + annulus_floor = float(np.percentile(annulus, annulus_floor_percentile)) * ( + local.annulus_floor_multiplier + ) + threshold = max(base_threshold, annulus_floor) + binary = morphology.remove_small_objects( + smooth >= threshold, + min_size=local.min_component_size_px, + ) + if local.closing_radius_px > 0: + binary = ndi.binary_closing( + binary, + structure=morphology.disk(local.closing_radius_px), + ) + labeled = measure.label(ndi.binary_fill_holes(binary)) + results: List[LocalSegmentationResult] = [] + for prop in measure.regionprops(labeled, intensity_image=smooth): + mask = np.asarray(labeled == prop.label, dtype=np.bool_) + metrics = SegmentationMetrics( + threshold_method=local.threshold_method, + base_threshold=base_threshold, + annulus_floor=annulus_floor, + threshold=threshold, + selection_mode="all_components", + area_px=int(prop.area), + equivalent_diameter_um=float( + prop.equivalent_diameter_area * config.pixel_size_um + ), + major_axis_um=float(prop.axis_major_length * config.pixel_size_um), + minor_axis_um=float(prop.axis_minor_length * config.pixel_size_um), + eccentricity=float(prop.eccentricity), + solidity=float(prop.solidity), + circularity=_circularity(prop.area, prop.perimeter), + centroid_y_px=float(prop.centroid[0]), + centroid_x_px=float(prop.centroid[1]), + centroid_offset_px=float( + np.hypot(prop.centroid[1] - center_x, prop.centroid[0] - center_y) + ), + mean_intensity=float(prop.mean_intensity), + max_intensity=float(prop.max_intensity), + ) + results.append(LocalSegmentationResult(mask=mask, metrics=metrics)) + return smooth, results + + +def segment_oocyte_patch( + patch: np.ndarray, + config: OocyteDetectionConfig, +) -> LocalSegmentationResult: + """Segment the oocyte associated with the center of a raw UCHL1 patch.""" + + _, result = _segment_oocyte_patch(patch, config) + return result diff --git a/aegle/oocyte/shape_recovery_finalize.py b/aegle/oocyte/shape_recovery_finalize.py new file mode 100644 index 0000000..bd5a6bf --- /dev/null +++ b/aegle/oocyte/shape_recovery_finalize.py @@ -0,0 +1,580 @@ +"""Finalize a reviewed shape-recovery delta into a new manual-mask delivery.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Mapping, Tuple + +import numpy as np +import pandas as pd + +from .export import export_whole_slide_labels +from .io import load_candidate_mask +from .manual_seed_finalize import ( + ManualSeedFinalizeResult, + _atomic_write_csv, + _boundary_warning, + _overlap_audit, + _tight_mask, + _write_reviewed_mask, +) +from .recall_review import ( + _atomic_write_text, + _file_sha256, + _json_safe, + _load_sample, + _mask_path, + _read_json, +) + + +MANUAL_SEED_V2_PROFILE_NAME = "manual_seed_review_v2" +SHAPE_REVIEW_CHOICES = { + "keep_v4", + "accept_shape_recovery", + "exclude", + "unsure", +} + + +def _as_bool(value: Any) -> bool: + if isinstance(value, (bool, np.bool_)): + return bool(value) + return str(value).strip().casefold() in {"true", "1", "yes"} + + +def _clean_text(value: Any) -> str: + if value is None or (not isinstance(value, str) and pd.isna(value)): + return "" + return str(value).strip() + + +def _validate_shape_identity(sample_identity: Mapping[str, Any], payload: Mapping[str, Any]) -> None: + if payload.get("schema_version") != 1: + raise ValueError("shape-recovery review schema_version must be 1") + if payload.get("review_type") != "manual_seed_shape_recovery_review": + raise ValueError("review_type must be 'manual_seed_shape_recovery_review'") + identity = payload.get("identity") + if not isinstance(identity, Mapping): + raise ValueError("shape-recovery review is missing its identity object") + for field, expected in sample_identity.items(): + if identity.get(field) != expected: + raise ValueError(f"shape-recovery review identity mismatch for {field}") + for field in ( + "analysis_sha256", + "manual_review_json_sha256", + "shape_candidate_table_sha256", + ): + if not identity.get(field): + raise ValueError(f"shape-recovery review identity is missing {field}") + + +def _verify_base_delivery(base_dir: Path, expected_review_sha256: str) -> Dict[str, Any]: + manifest_path = base_dir / "manual_seed_finalize_manifest.json" + manifest = _read_json(manifest_path) + if manifest.get("delivery_name") != "reviewed_manual_seed_delta_v1": + raise ValueError("base finalize directory is not a reviewed manual-seed v1 delivery") + if manifest.get("review_json_sha256") != expected_review_sha256: + raise ValueError("base v1 review SHA-256 does not match shape-review identity") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, Mapping): + raise ValueError("base v1 manifest is missing artifacts") + for relative_path, record in artifacts.items(): + if not isinstance(record, Mapping): + raise ValueError(f"invalid base artifact record: {relative_path}") + path = Path(str(record.get("path", ""))) + if not path.is_file(): + raise FileNotFoundError(f"base v1 artifact is missing: {path}") + if not path.resolve().is_relative_to(base_dir.resolve()): + raise ValueError(f"base v1 artifact is outside its delivery: {path}") + if _file_sha256(path) != str(record.get("sha256", "")): + raise ValueError(f"base v1 artifact SHA-256 mismatch: {relative_path}") + if path.stat().st_size != int(record.get("size_bytes", -1)): + raise ValueError(f"base v1 artifact size mismatch: {relative_path}") + return manifest + + +def finalize_shape_recovery_review( + sample_dir: Path, + shape_review_json: Path, + base_finalize_dir: Path, + out_dir: Path, + *, + write_combined_labels: bool = True, + tile_shape_yx: Tuple[int, int] = (512, 512), + max_smaller_overlap_fraction: float = 0.25, +) -> ManualSeedFinalizeResult: + """Apply shape-delta decisions to v1 and write a separate v2 delivery.""" + + sample = _load_sample(sample_dir) + review_path = Path(shape_review_json).resolve() + payload = _read_json(review_path) + _validate_shape_identity(sample.review_identity, payload) + identity = payload["identity"] + candidate_table_path = Path(str(identity.get("shape_candidate_table", ""))).resolve() + if not candidate_table_path.is_file(): + raise FileNotFoundError( + f"shape-recovery candidate table does not exist: {candidate_table_path}" + ) + candidate_table_sha256 = _file_sha256(candidate_table_path) + if candidate_table_sha256 != str(identity["shape_candidate_table_sha256"]): + raise ValueError("shape-recovery candidate table SHA-256 mismatch") + shape_root = candidate_table_path.parent.resolve() + shape_candidates = pd.read_csv(candidate_table_path) + if shape_candidates["annotation_id"].duplicated().any(): + raise ValueError("shape-recovery candidate table has duplicate annotations") + shape_by_id = shape_candidates.set_index("annotation_id", drop=False) + + review_rows = payload.get("rows") + if not isinstance(review_rows, list): + raise ValueError("shape-recovery review rows must be a list") + review_ids = [str(row.get("annotation_id", "")) for row in review_rows] + if len(review_ids) != len(set(review_ids)): + raise ValueError("shape-recovery review has duplicate annotations") + if set(review_ids) != set(str(value) for value in shape_candidates["annotation_id"]): + raise ValueError("shape-recovery review rows do not match its candidate table") + shape_decisions: Dict[int, Dict[str, Any]] = {} + choice_counts: Dict[str, int] = {} + for row_number, raw_row in enumerate(review_rows, start=1): + if not isinstance(raw_row, Mapping): + raise ValueError(f"shape-recovery review row {row_number} must be an object") + annotation_id = str(raw_row["annotation_id"]) + candidate = shape_by_id.loc[annotation_id] + review_index = int(candidate["review_index"]) + if int(raw_row["review_index"]) != review_index: + raise ValueError(f"shape-recovery review index changed for {annotation_id}") + if not ( + np.isclose(float(raw_row["x"]), float(candidate["x"]), atol=0.01) + and np.isclose(float(raw_row["y"]), float(candidate["y"]), atol=0.01) + ): + raise ValueError(f"shape-recovery coordinates changed for {annotation_id}") + choice = str(raw_row.get("shape_review_choice", "")).strip() + if choice not in SHAPE_REVIEW_CHOICES: + raise ValueError(f"invalid or missing shape-review choice for {annotation_id}") + if choice == "unsure": + raise ValueError(f"shape-recovery review remains unsure for {annotation_id}") + choice_counts[choice] = choice_counts.get(choice, 0) + 1 + shape_decisions[review_index] = { + "annotation_id": annotation_id, + "choice": choice, + "notes": str(raw_row.get("shape_review_notes", "")).strip(), + "candidate": candidate, + } + + base_dir = Path(base_finalize_dir).resolve() + base_manifest = _verify_base_delivery( + base_dir, + str(identity["manual_review_json_sha256"]), + ) + if base_manifest.get("sample") != sample.review_identity: + raise ValueError("base v1 sample identity does not match shape review") + base_decisions = pd.read_csv(base_dir / "manual_seed_review_decisions.csv") + base_candidates = pd.read_csv(base_dir / "manual_seed_accepted_candidates.csv") + if base_decisions["review_index"].duplicated().any(): + raise ValueError("base v1 decisions contain duplicate review indices") + base_decision_by_index = base_decisions.set_index("review_index", drop=False) + base_candidate_by_index = base_candidates.set_index("manual_review_index", drop=False) + analysis_path = Path(str(identity.get("analysis_table", ""))).resolve() + if not analysis_path.is_file() or _file_sha256(analysis_path) != str( + identity["analysis_sha256"] + ): + raise ValueError("shape review analysis table is missing or stale") + analysis = pd.read_csv(analysis_path).set_index("annotation_id", drop=False) + + selections = [] + final_decisions = [] + replacement_count = 0 + addition_count = 0 + exclusion_count = 0 + for review_index, base in base_decision_by_index.sort_index().iterrows(): + review_index = int(review_index) + annotation_id = str(base["annotation_id"]) + base_accepted = _as_bool(base["accepted"]) + shape = shape_decisions.get(review_index) + final_accepted = base_accepted + final_source = "v1_reviewed_mask" if base_accepted else "excluded_v1" + source_path = ( + None + if not base_accepted + else Path(str(base["reviewed_mask_path"])).resolve() + ) + final_notes = _clean_text(base.get("manual_notes", "")) + boundary_warning = _as_bool(base.get("boundary_warning", False)) + shape_choice = "" + shape_notes = "" + if shape is not None: + shape_choice = str(shape["choice"]) + shape_notes = str(shape["notes"]) + if shape_choice == "accept_shape_recovery": + source_path = Path( + str(shape["candidate"]["shape_recovery_mask_path"]) + ).resolve() + if not source_path.is_relative_to(shape_root): + raise ValueError( + f"shape-recovery mask is outside its review directory: {source_path}" + ) + final_accepted = True + final_source = "shape_recovery" + final_notes = shape_notes + boundary_warning = _boundary_warning(shape_notes) + if base_accepted: + replacement_count += 1 + else: + addition_count += 1 + elif shape_choice == "exclude": + source_path = None + final_accepted = False + final_source = "excluded_shape_review" + final_notes = shape_notes + boundary_warning = False + if base_accepted: + exclusion_count += 1 + elif shape_choice == "keep_v4": + final_source = "v1_reviewed_mask" if base_accepted else "excluded_v1" + decision = { + "review_index": review_index, + "annotation_id": annotation_id, + "x": float(base["x"]), + "y": float(base["y"]), + "failure_class": str(base["failure_class"]), + "base_manual_mask_choice": str(base["manual_mask_choice"]), + "shape_review_choice": shape_choice, + "base_manual_notes": _clean_text(base.get("manual_notes", "")), + "shape_review_notes": shape_notes, + "final_accepted": final_accepted, + "final_source": final_source, + "final_notes": final_notes, + "boundary_warning": boundary_warning, + "source_mask_path": "" if source_path is None else str(source_path), + "source_mask_sha256": "", + "reviewed_mask_path": "", + "reviewed_mask_sha256": "", + } + if final_accepted: + if source_path is None or not source_path.is_file(): + raise FileNotFoundError(f"final reviewed mask is missing: {source_path}") + if final_source == "v1_reviewed_mask" and not source_path.is_relative_to( + base_dir + ): + raise ValueError(f"base reviewed mask is outside v1 delivery: {source_path}") + persisted = load_candidate_mask(source_path) + if persisted.image_shape_yx != sample.image_shape_yx: + raise ValueError(f"final mask image shape mismatch: {source_path}") + source_sha256 = _file_sha256(source_path) + decision["source_mask_sha256"] = source_sha256 + selections.append( + { + "review_index": review_index, + "annotation_id": annotation_id, + "source": final_source, + "source_path": source_path, + "source_sha256": source_sha256, + "mask": _tight_mask(persisted), + "notes": final_notes, + "boundary_warning": boundary_warning, + "base_candidate": ( + None + if review_index not in base_candidate_by_index.index + else base_candidate_by_index.loc[review_index] + ), + "analysis": analysis.loc[annotation_id], + "shape_choice": shape_choice, + } + ) + final_decisions.append(decision) + + production_masks = [] + for record in sample.candidates.to_dict("records"): + production_masks.append( + ( + str(record["detector_component_id"]), + load_candidate_mask(_mask_path(sample.sample_dir, record)), + ) + ) + audit = _overlap_audit( + [ + (f"manual_seed_{int(item['review_index']):03d}", item["mask"]) + for item in selections + ], + production_masks, + ) + blocking = audit[ + audit["smaller_mask_overlap_fraction"] >= max_smaller_overlap_fraction + ] + if not blocking.empty: + pairs = ", ".join( + f"{row.left_id}/{row.right_id}={row.smaller_mask_overlap_fraction:.3f}" + for row in blocking.itertuples() + ) + raise ValueError(f"shape-finalized masks have blocking overlap: {pairs}") + + destination = Path(out_dir).resolve() + destination.mkdir(parents=True, exist_ok=True) + manifest_path = destination / "manual_seed_finalize_manifest.json" + if manifest_path.exists(): + manifest_path.unlink() + masks_dir = destination / "reviewed_masks" + masks_dir.mkdir(parents=True, exist_ok=True) + expected_names = { + f"manual_seed_{int(item['review_index']):03d}.npz" for item in selections + } + for existing in masks_dir.glob("*.npz"): + if existing.name not in expected_names: + existing.unlink() + + review_sha256 = _file_sha256(review_path) + base_manifest_sha256 = _file_sha256( + base_dir / "manual_seed_finalize_manifest.json" + ) + decision_by_index = { + int(row["review_index"]): row for row in final_decisions + } + candidate_rows = [] + for item in selections: + review_index = int(item["review_index"]) + candidate_id = f"manual_seed_{review_index:03d}" + reviewed_path = masks_dir / f"{candidate_id}.npz" + source_metadata = dict(item["mask"].metadata) + metrics = dict(source_metadata.get("metrics", {})) + metadata = { + **source_metadata, + "schema_version": 1, + "sample_id": sample.sample_id, + "candidate_id": candidate_id, + "profile_name": MANUAL_SEED_V2_PROFILE_NAME, + "base_profile_name": sample.profile_name, + "base_profile_fingerprint": sample.profile_fingerprint, + "implementation_version": sample.implementation_version, + "reviewed_manual_seed": True, + "provisional_only": False, + "review_generation": 2, + "source_annotation_id": item["annotation_id"], + "final_source": item["source"], + "shape_review_choice": item["shape_choice"], + "final_notes": item["notes"], + "boundary_warning": item["boundary_warning"], + "shape_review_json": str(review_path), + "shape_review_json_sha256": review_sha256, + "shape_review_exported_at": payload.get("exported_at"), + "base_delivery_manifest": str( + base_dir / "manual_seed_finalize_manifest.json" + ), + "base_delivery_manifest_sha256": base_manifest_sha256, + "source_mask_path": str(item["source_path"]), + "source_mask_sha256": item["source_sha256"], + } + _write_reviewed_mask(reviewed_path, mask=item["mask"], metadata=metadata) + reviewed_sha256 = _file_sha256(reviewed_path) + decision = decision_by_index[review_index] + decision["reviewed_mask_path"] = str(reviewed_path) + decision["reviewed_mask_sha256"] = reviewed_sha256 + ys, xs = np.nonzero(item["mask"].mask) + centroid_x = item["mask"].bbox.x0 + float(xs.mean()) + centroid_y = item["mask"].bbox.y0 + float(ys.mean()) + row = ( + {} + if item["base_candidate"] is None + else item["base_candidate"].to_dict() + ) + row.update( + { + "detector_component_id": candidate_id, + "source_annotation_id": item["annotation_id"], + "display_id": f"#R{review_index:03d}", + "html_id": f"manual-seed-{review_index:03d}", + "accepted": True, + "accepted_strict": False, + "accepted_rescue": False, + "detector_score": 1.0, + "acceptance_mode": ( + "manual_seed_shape_recovery_reviewed" + if item["source"] == "shape_recovery" + else "manual_seed_reviewed_v2_carry_forward" + ), + "detection_pass": MANUAL_SEED_V2_PROFILE_NAME, + "segmentation_pass": ( + "manual_shape_recovery" + if item["source"] == "shape_recovery" + else "manual_v1_carry_forward" + ), + "center_x": int(round(centroid_x)), + "center_y": int(round(centroid_y)), + "component_centroid_x": centroid_x, + "component_centroid_y": centroid_y, + "bbox_x0": item["mask"].bbox.x0, + "bbox_y0": item["mask"].bbox.y0, + "bbox_x1": item["mask"].bbox.x1, + "bbox_y1": item["mask"].bbox.y1, + "local_area_px": int(item["mask"].mask.sum()), + "local_equivalent_diameter_um": metrics.get( + "equivalent_diameter_um" + ), + "local_major_axis_um": metrics.get("major_axis_um"), + "local_minor_axis_um": metrics.get("minor_axis_um"), + "local_eccentricity": metrics.get("eccentricity"), + "local_solidity": metrics.get("solidity"), + "local_circularity": metrics.get("circularity"), + "local_centroid_offset_px": metrics.get("centroid_offset_px"), + "local_mean_intensity": metrics.get("mean_intensity"), + "local_max_intensity": metrics.get("max_intensity"), + "threshold_method": metrics.get("threshold_method"), + "threshold": metrics.get("threshold"), + "selection_mode": metrics.get("selection_mode"), + "score_background_percentile": source_metadata.get( + "annulus_floor_percentile" + ), + "failure_class": item["analysis"]["failure_class"], + "manual_review_index": review_index, + "base_manual_mask_choice": decision["base_manual_mask_choice"], + "shape_review_choice": decision["shape_review_choice"], + "manual_notes": item["notes"], + "boundary_warning": bool(item["boundary_warning"]), + "quality_class": ( + "reviewed_boundary_warning" + if item["boundary_warning"] + else "reviewed_manual_seed_v2" + ), + "mask_path": str(reviewed_path.relative_to(destination)), + "mask_source_dir": str(destination), + "source_reviewed_mask_path": str(item["source_path"]), + "source_reviewed_mask_sha256": item["source_sha256"], + "reviewed_mask_sha256": reviewed_sha256, + "shape_review_json_sha256": review_sha256, + "duplicate_suppressed": False, + } + ) + candidate_rows.append(row) + + decisions_path = destination / "manual_seed_review_decisions_v2.csv" + candidates_path = destination / "manual_seed_accepted_candidates_v2.csv" + overlap_audit_path = destination / "mask_overlap_audit.csv" + _atomic_write_csv(pd.DataFrame(final_decisions), decisions_path) + candidates = pd.DataFrame(candidate_rows) + _atomic_write_csv(candidates, candidates_path) + _atomic_write_csv(audit, overlap_audit_path) + delta_labels = export_whole_slide_labels( + candidates, + sample_dir=destination, + image_shape_yx=sample.image_shape_yx, + image_path=destination / "oocyte_labels_manual_seed_delta_v2.ome.tiff", + mapping_path=destination / "oocyte_labels_manual_seed_delta_v2_mapping.csv", + tile_shape_yx=tile_shape_yx, + ) + + combined_labels = None + combined_candidates_path = None + if write_combined_labels: + production = sample.candidates.copy() + production["mask_path"] = [ + str(_mask_path(sample.sample_dir, record).resolve()) + for record in sample.candidates.to_dict("records") + ] + production["mask_source_dir"] = "" + combined = pd.concat([production, candidates], ignore_index=True, sort=False) + combined_candidates_path = ( + destination / "oocyte_candidates_rescue_v1_plus_manual_seed_v2.csv" + ) + _atomic_write_csv(combined, combined_candidates_path) + combined_labels = export_whole_slide_labels( + combined, + sample_dir=destination, + image_shape_yx=sample.image_shape_yx, + image_path=( + destination + / "oocyte_labels_rescue_v1_plus_manual_seed_v2.ome.tiff" + ), + mapping_path=( + destination + / "oocyte_labels_rescue_v1_plus_manual_seed_v2_mapping.csv" + ), + tile_shape_yx=tile_shape_yx, + ) + + artifact_paths = [ + decisions_path, + candidates_path, + overlap_audit_path, + delta_labels.image_path, + delta_labels.mapping_path, + *(Path(row["reviewed_mask_path"]) for row in final_decisions if row["reviewed_mask_path"]), + ] + if combined_labels is not None and combined_candidates_path is not None: + artifact_paths.extend( + [ + combined_candidates_path, + combined_labels.image_path, + combined_labels.mapping_path, + ] + ) + manifest = { + "schema_version": 1, + "delivery_name": "reviewed_manual_seed_delta_v2", + "sample": sample.review_identity, + "shape_review_json": str(review_path), + "shape_review_json_sha256": review_sha256, + "shape_review_exported_at": payload.get("exported_at"), + "shape_choice_counts": choice_counts, + "shape_candidate_table": str(candidate_table_path), + "shape_candidate_table_sha256": candidate_table_sha256, + "base_delivery": str(base_dir), + "base_delivery_manifest_sha256": base_manifest_sha256, + "reviewed_row_count": len(final_decisions), + "accepted_manual_mask_count": len(candidates), + "shape_replacement_count": replacement_count, + "shape_addition_count": addition_count, + "shape_exclusion_count": exclusion_count, + "boundary_warning_count": int(candidates["boundary_warning"].sum()), + "production_candidate_count": int(sample.candidates["accepted"].astype(bool).sum()), + "combined_label_count": ( + None if combined_labels is None else combined_labels.label_count + ), + "manual_overlap_audit_row_count": len(audit), + "max_smaller_overlap_fraction": ( + 0.0 if audit.empty else float(audit["smaller_mask_overlap_fraction"].max()) + ), + "label_export": { + "delta_label_count": delta_labels.label_count, + "delta_assigned_pixel_count": delta_labels.assigned_pixel_count, + "delta_overlap_pixel_count": delta_labels.overlap_pixel_count, + "combined_assigned_pixel_count": ( + None if combined_labels is None else combined_labels.assigned_pixel_count + ), + "combined_overlap_pixel_count": ( + None if combined_labels is None else combined_labels.overlap_pixel_count + ), + }, + "production_outputs_modified": False, + "base_v1_outputs_modified": False, + "artifacts": { + str(path.relative_to(destination)): { + "path": str(path), + "sha256": _file_sha256(path), + "size_bytes": path.stat().st_size, + } + for path in artifact_paths + }, + } + _atomic_write_text( + manifest_path, + json.dumps(_json_safe(manifest), indent=2, sort_keys=True, allow_nan=False), + ) + return ManualSeedFinalizeResult( + out_dir=destination, + decisions_path=decisions_path, + candidates_path=candidates_path, + overlap_audit_path=overlap_audit_path, + delta_labels=delta_labels, + combined_labels=combined_labels, + combined_candidates_path=combined_candidates_path, + manifest_path=manifest_path, + accepted_count=len(candidates), + boundary_warning_count=int(candidates["boundary_warning"].sum()), + ) + + +__all__ = [ + "MANUAL_SEED_V2_PROFILE_NAME", + "SHAPE_REVIEW_CHOICES", + "finalize_shape_recovery_review", +] diff --git a/aegle/oocyte/shape_recovery_review.py b/aegle/oocyte/shape_recovery_review.py new file mode 100644 index 0000000..16dc0c1 --- /dev/null +++ b/aegle/oocyte/shape_recovery_review.py @@ -0,0 +1,333 @@ +"""Targeted review pack for shape-gated manual-seed mask expansion.""" + +from __future__ import annotations + +import html +import io +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Mapping + +import numpy as np +import pandas as pd +from PIL import Image + +from .manual_seed_finalize import _atomic_write_csv, _validate_review_identity +from .manual_seed_review import _crosshair, _draw_mask, _load_provisional, _panel_label +from .recall_review import ( + RecallReviewRuntime, + _atomic_write_text, + _file_sha256, + _json_safe, + _load_sample, + _read_json, + _save_provisional_mask, + _validate_review_payload, +) + + +SHAPE_RECOVERY_PARAMETERS = { + "max_area_ratio": 6.0, + "min_equivalent_diameter_um": 20.0, + "min_circularity": 0.80, + "min_solidity": 0.90, + "max_centroid_offset_px": 25.0, + "min_conservative_overlap": 0.70, +} + + +@dataclass(frozen=True) +class ShapeRecoveryReviewResult: + page_path: Path + candidates_path: Path + assets_dir: Path + card_count: int + + +def _render_shape_card( + runtime: RecallReviewRuntime, + row: Mapping[str, Any], + destination: Path, + *, + radius: int, +) -> None: + center = (int(round(float(row["x"]))), int(round(float(row["y"])))) + patch = runtime.source.read_patch(center, radius) + raw = Image.open(io.BytesIO(runtime.render_patch(center, radius, "local"))).convert( + "RGBA" + ) + existing = Image.open(io.BytesIO(runtime.render_overlay(center, radius))).convert( + "RGBA" + ) + base = Image.alpha_composite(raw, existing) + click_x = int(round(float(row["x"]))) - (center[0] - radius) + click_y = int(round(float(row["y"]))) - (center[1] - radius) + first = base.copy() + _crosshair(first, click_x, click_y) + panels = [ + _panel_label( + first, + f"#{int(row['review_index']):03d} RAW + EXISTING MASKS + CENTER", + color=(0, 255, 242), + ) + ] + for path_key, color, label in ( + ( + "current_mask_path", + (255, 211, 72), + f"V4 REVIEWED / P{float(row['current_percentile']):.0f} / " + f"d {float(row['current_equivalent_diameter_um']):.1f} um", + ), + ( + "shape_recovery_mask_path", + (255, 132, 67), + f"SHAPE RECOVERY / P{float(row['shape_recovery_percentile']):.0f} / " + f"d {float(row['shape_recovery_equivalent_diameter_um']):.1f} um", + ), + ): + persisted = _load_provisional(row[path_key]) + if persisted is None: + raise FileNotFoundError(f"shape-review mask is unavailable: {row[path_key]}") + panel = _draw_mask( + base.copy(), + runtime._place_mask(persisted, patch), + color=color, + ) + _crosshair(panel, click_x, click_y) + panels.append(_panel_label(panel, label, color=color)) + gutter = 5 + canvas = Image.new( + "RGBA", + ( + sum(panel.width for panel in panels) + gutter * (len(panels) - 1), + panels[0].height, + ), + (245, 238, 224, 255), + ) + x_offset = 0 + for panel in panels: + canvas.paste(panel, (x_offset, 0)) + x_offset += panel.width + gutter + destination.parent.mkdir(parents=True, exist_ok=True) + canvas.convert("RGB").save(destination, format="WEBP", quality=89, method=6) + + +def _metric(value: Any, digits: int = 2) -> str: + try: + number = float(value) + except (TypeError, ValueError): + return "n/a" + return f"{number:.{digits}f}" if np.isfinite(number) else "n/a" + + +def _card_html(row: Mapping[str, Any]) -> str: + annotation_id = html.escape(str(row["annotation_id"]), quote=True) + review_index = int(row["review_index"]) + previous_note = "" if pd.isna(row.get("previous_manual_notes")) else str( + row.get("previous_manual_notes", "") + ) + previous_choice = html.escape(str(row.get("previous_manual_mask_choice", ""))) + return f'''
    + Shape recovery comparison for {annotation_id} +

    #{review_index:03d}

    {html.escape(str(row['failure_class']))}
    +
    {annotation_id} · previous {previous_choice} · area growth {_metric(row['shape_to_current_area_ratio'])}x
    +
    v4 d {_metric(row['current_equivalent_diameter_um'],1)} umv4 circ {_metric(row['current_circularity'])}v4 solid {_metric(row['current_solidity'])}recovery d {_metric(row['shape_recovery_equivalent_diameter_um'],1)} umrecovery circ {_metric(row['shape_recovery_circularity'])}recovery solid {_metric(row['shape_recovery_solidity'])}
    +
    + +
    ''' + + +_CSS = r''' +:root{--ink:#172522;--panel:#fffaf0;--line:#d1c6b3;--teal:#087b78;--orange:#d9662b;--yellow:#b58a17;--red:#a94336}*{box-sizing:border-box}body{margin:0;color:var(--ink);background:radial-gradient(circle at 10% 0,#fff8e8 0,transparent 30%),linear-gradient(135deg,#eadfcd,#f8f2e7 66%,#dcebe3);font-family:"Iowan Old Style","Palatino Linotype",Palatino,serif}.shell{width:min(1500px,calc(100% - 24px));margin:auto}.hero{margin:16px 0;padding:24px 28px;border:1px solid var(--line);border-radius:22px;background:linear-gradient(115deg,#fffaf0,#e4f3ec);box-shadow:0 15px 35px rgba(30,45,38,.12)}.hero h1{font-size:clamp(2rem,4.5vw,4.2rem);line-height:.95;margin:.2em 0}.eyebrow,.mono{font-family:"IBM Plex Mono","Aptos Mono","Courier New",monospace}.eyebrow{text-transform:uppercase;letter-spacing:.14em;color:var(--teal);font-size:.75rem;font-weight:700}.hero p{max-width:1000px;line-height:1.5}.toolbar{position:sticky;top:6px;z-index:5;display:flex;gap:8px;align-items:center;flex-wrap:wrap;padding:10px 12px;margin:14px 0;border:1px solid var(--line);border-radius:14px;background:rgba(255,250,240,.95);backdrop-filter:blur(8px)}button,.button,input{font:inherit;border:1px solid #b9ae9d;border-radius:10px;padding:8px 11px;background:#fffaf0;color:var(--ink)}button{cursor:pointer}.button{text-decoration:none}.grow{flex:1}.cards{display:grid;grid-template-columns:1fr;gap:16px;margin-bottom:50px}.card{background:var(--panel);border:1px solid var(--line);border-radius:18px;overflow:hidden;box-shadow:0 9px 24px rgba(32,45,39,.08)}.card.hidden{display:none}.card img{width:100%;display:block;background:#171f1c}.body{padding:14px}.title{display:flex;align-items:center;justify-content:space-between}.title h2{margin:0}.title span{font-family:monospace;color:var(--orange)}.metrics{display:grid;grid-template-columns:repeat(3,1fr);gap:6px;margin:10px 0;font:12px monospace}.metrics span{background:#f0e7d8;padding:7px;border-radius:7px}.actions{display:grid;grid-template-columns:repeat(4,1fr);gap:6px}.actions button.selected[data-choice=keep_v4]{background:#f6e6a9;border-color:var(--yellow)}.actions button.selected[data-choice=accept_shape_recovery]{background:#ffd7bd;border-color:var(--orange)}.actions button.selected[data-choice=exclude]{background:#f1d6cf;border-color:var(--red)}.actions button.selected[data-choice=unsure]{background:#e5e1d7;border-color:#847d70}.notes{width:100%;margin-top:8px}.footer{padding:10px 0 50px;color:#68726c}@media(max-width:720px){.shell{width:calc(100% - 10px)}.hero{padding:18px}.metrics{grid-template-columns:1fr 1fr}.actions{grid-template-columns:1fr 1fr}.toolbar{position:static}} +''' + + +_JS = r''' +const DATA=JSON.parse(document.getElementById('shape-data').textContent),KEY='aegle-oocyte-shape-review:'+DATA.identity.sample_id+':'+DATA.identity.shape_candidate_table_sha256;let state=JSON.parse(localStorage.getItem(KEY)||'{}'),filter='all';const cards=[...document.querySelectorAll('.card')];function save(){localStorage.setItem(KEY,JSON.stringify(state));paintProgress()}function paint(card){const id=card.dataset.id,s=state[id]||{};card.dataset.review=s.choice||'unreviewed';card.querySelectorAll('[data-choice]').forEach(b=>b.classList.toggle('selected',b.dataset.choice===s.choice));card.querySelector('.notes').value=s.notes??card.querySelector('.notes').value}function paintProgress(){const reviewed=Object.values(state).filter(s=>s.choice).length;document.getElementById('progress').textContent=reviewed+' / '+DATA.rows.length+' reviewed'}function apply(){cards.forEach(c=>c.classList.toggle('hidden',filter==='unreviewed'&&c.dataset.review!=='unreviewed'))}cards.forEach(paint);document.querySelectorAll('[data-choice]').forEach(b=>b.onclick=()=>{const card=b.closest('.card'),id=card.dataset.id;state[id]={...(state[id]||{}),choice:b.dataset.choice};paint(card);save();apply()});document.querySelectorAll('.notes').forEach(n=>n.onchange=()=>{const card=n.closest('.card'),id=card.dataset.id;state[id]={...(state[id]||{}),notes:n.value};save()});document.querySelectorAll('[data-filter]').forEach(b=>b.onclick=()=>{filter=b.dataset.filter;apply()});function exportData(type){const rows=DATA.rows.map(r=>({...r,shape_review_choice:(state[r.annotation_id]||{}).choice||'',shape_review_notes:(state[r.annotation_id]||{}).notes||''})),payload={schema_version:1,review_type:'manual_seed_shape_recovery_review',identity:DATA.identity,exported_at:new Date().toISOString(),rows};let blob,name;if(type==='json'){blob=new Blob([JSON.stringify(payload,null,2)],{type:'application/json'});name=DATA.identity.sample_id+'_shape_recovery_review.json'}else{const keys=Object.keys(rows[0]||{}),esc=v=>'"'+String(v??'').replaceAll('"','""')+'"';blob=new Blob([[keys.join(','),...rows.map(r=>keys.map(k=>esc(r[k])).join(','))].join('\n')],{type:'text/csv'});name=DATA.identity.sample_id+'_shape_recovery_review.csv'}const a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download=name;a.click();setTimeout(()=>URL.revokeObjectURL(a.href),0)}document.getElementById('export-json').onclick=()=>exportData('json');document.getElementById('export-csv').onclick=()=>exportData('csv');paintProgress();apply(); +''' + + +def generate_shape_recovery_review( + sample_dir: Path, + recall_review_json: Path, + manual_review_json: Path, + out_dir: Path, + *, + patch_radius_px: int = 220, +) -> ShapeRecoveryReviewResult: + """Generate a delta page only for masks changed by shape-gated expansion.""" + + sample = _load_sample(sample_dir) + recall_payload = _read_json(Path(recall_review_json)) + _validate_review_payload(sample, recall_payload) + manual_path = Path(manual_review_json).resolve() + manual_payload = _read_json(manual_path) + _validate_review_identity(sample.review_identity, manual_payload) + identity = manual_payload["identity"] + analysis_path = Path(str(identity.get("analysis_table", ""))).resolve() + if not analysis_path.is_file(): + raise FileNotFoundError(f"recall analysis table does not exist: {analysis_path}") + if _file_sha256(analysis_path) != str(identity.get("analysis_sha256", "")): + raise ValueError("recall analysis SHA-256 does not match manual review identity") + analysis = pd.read_csv(analysis_path) + analysis_by_id = analysis.set_index("annotation_id", drop=False) + manual_rows = manual_payload.get("rows") + if not isinstance(manual_rows, list): + raise ValueError("manual-seed mask review rows must be a list") + centers = [ + (float(annotation["x"]), float(annotation["y"])) + for annotation in recall_payload["missing_oocytes"] + ] + center_by_id = { + str(annotation["annotation_id"]): center + for annotation, center in zip(recall_payload["missing_oocytes"], centers) + } + root = Path(out_dir).resolve() + masks_dir = root / "shape_recovery_masks" + assets_dir = root / "review_assets" + masks_dir.mkdir(parents=True, exist_ok=True) + rows = [] + with RecallReviewRuntime(sample.sample_dir) as runtime: + for review_index, manual_row in enumerate(manual_rows, start=1): + if not isinstance(manual_row, Mapping): + raise ValueError(f"manual review row {review_index} must be an object") + annotation_id = str(manual_row["annotation_id"]) + if annotation_id not in center_by_id or annotation_id not in analysis_by_id.index: + raise ValueError(f"manual review annotation is not in recall analysis: {annotation_id}") + x, y = center_by_id[annotation_id] + other_centers = tuple( + center for other_id, center in center_by_id.items() if other_id != annotation_id + ) + standard = runtime.segment_manual_provisionals( + x, + y, + exclude_points_xy=other_centers, + ) + recovered = runtime.segment_manual_provisionals( + x, + y, + exclude_points_xy=other_centers, + allow_shape_recovery=True, + ) + if standard.expanded is None or recovered.expanded is None: + continue + if np.array_equal(standard.expanded.mask, recovered.expanded.mask): + continue + if recovered.expanded.metrics.area_px <= standard.expanded.metrics.area_px: + continue + source = analysis_by_id.loc[annotation_id] + current_path = Path(str(source["manual_expanded_mask_path"])).resolve() + if not current_path.is_file(): + raise FileNotFoundError(f"current v4 mask is missing: {current_path}") + recovery_path = masks_dir / f"shape-{review_index:03d}.npz" + _save_provisional_mask( + recovery_path, + result=recovered.expanded, + patch=recovered.patch, + annotation_id=annotation_id, + percentile=float(recovered.expanded_percentile), + ) + rows.append( + { + "review_index": review_index, + "annotation_id": annotation_id, + "x": x, + "y": y, + "failure_class": str(source["failure_class"]), + "previous_manual_mask_choice": str( + manual_row.get("manual_mask_choice", "") + ), + "previous_manual_notes": str(manual_row.get("manual_notes", "")), + "current_mask_path": str(current_path), + "current_percentile": float(source["manual_expanded_percentile"]), + "current_area_px": int(standard.expanded.metrics.area_px), + "current_equivalent_diameter_um": float( + standard.expanded.metrics.equivalent_diameter_um + ), + "current_circularity": float(standard.expanded.metrics.circularity), + "current_solidity": float(standard.expanded.metrics.solidity), + "shape_recovery_mask_path": str(recovery_path), + "shape_recovery_percentile": float(recovered.expanded_percentile), + "shape_recovery_area_px": int(recovered.expanded.metrics.area_px), + "shape_recovery_equivalent_diameter_um": float( + recovered.expanded.metrics.equivalent_diameter_um + ), + "shape_recovery_circularity": float( + recovered.expanded.metrics.circularity + ), + "shape_recovery_solidity": float(recovered.expanded.metrics.solidity), + "shape_recovery_centroid_offset_px": float( + recovered.expanded.metrics.centroid_offset_px + ), + "shape_to_current_area_ratio": float( + recovered.expanded.metrics.area_px + / max(standard.expanded.metrics.area_px, 1) + ), + } + ) + table = pd.DataFrame(rows) + candidates_path = root / "shape_recovery_candidates.csv" + _atomic_write_csv(table, candidates_path) + expected_assets = { + f"shape-{int(row['review_index']):03d}.webp" for row in rows + } + assets_dir.mkdir(parents=True, exist_ok=True) + for existing in assets_dir.glob("*.webp"): + if existing.name not in expected_assets: + existing.unlink() + for row in rows: + _render_shape_card( + runtime, + row, + assets_dir / f"shape-{int(row['review_index']):03d}.webp", + radius=patch_radius_px, + ) + if not rows: + raise ValueError("shape recovery did not change any reviewed masks") + + page_identity: Dict[str, Any] = { + **sample.review_identity, + "analysis_table": str(analysis_path), + "analysis_sha256": _file_sha256(analysis_path), + "manual_review_json": str(manual_path), + "manual_review_json_sha256": _file_sha256(manual_path), + "recall_review_json": str(Path(recall_review_json).resolve()), + "recall_review_json_sha256": _file_sha256(Path(recall_review_json)), + "shape_candidate_table": str(candidates_path), + "shape_candidate_table_sha256": _file_sha256(candidates_path), + "shape_recovery_parameters": SHAPE_RECOVERY_PARAMETERS, + "patch_radius_px": patch_radius_px, + } + safe_rows = [_json_safe(row) for row in rows] + cards = "".join(_card_html(row) for row in rows) + payload = {"identity": page_identity, "rows": safe_rows} + page = f'''{html.escape(sample.sample_id)} shape recovery review
    Aegle / manual-seed shape delta

    {html.escape(sample.sample_id)} shape recovery review

    Only masks changed by the shape-gated expansion are shown. Yellow is the frozen v4 reviewed candidate; orange is the proposed recovery. Use recovery only when its outer contour better matches the intended oocyte without absorbing a neighbor. This page does not modify the completed v1 delivery.

    {cards}
    Selections remain browser-local until exported. A new combined v2 is created only after this delta review is ingested.
    ''' + page_path = root / "shape_recovery_review.html" + _atomic_write_text(page_path, page) + summary = { + "schema_version": 1, + "sample": sample.review_identity, + "shape_recovery_card_count": len(rows), + "review_indices": [int(row["review_index"]) for row in rows], + "page": str(page_path), + "candidates": str(candidates_path), + "production_outputs_modified": False, + } + _atomic_write_text( + root / "summary.json", + json.dumps(_json_safe(summary), indent=2, sort_keys=True, allow_nan=False), + ) + return ShapeRecoveryReviewResult( + page_path=page_path, + candidates_path=candidates_path, + assets_dir=assets_dir, + card_count=len(rows), + ) + + +__all__ = [ + "SHAPE_RECOVERY_PARAMETERS", + "ShapeRecoveryReviewResult", + "generate_shape_recovery_review", +] diff --git a/docs/oocyte_detection.md b/docs/oocyte_detection.md new file mode 100644 index 0000000..0d92a8b --- /dev/null +++ b/docs/oocyte_detection.md @@ -0,0 +1,658 @@ +# Standalone raw-UCHL1 oocyte detection + +`aegle.oocyte` detects and segments oocytes directly from the registered raw +UCHL1 channel. It does not read or require DeepCell masks, nucleus masks, cell +metadata, or any other Aegle pipeline output. + +For Codex-assisted operation, start with `aegle/oocyte/AGENTS.md`. It defines +the human/agent decision boundary, the required state audit at the beginning of +each session, durable review evidence, versioning rules, and a reusable handoff +prompt. This document remains the command-level technical reference. + +## Installation + +From the repository root: + +```bash +python -m pip install -e '.[oocyte]' +``` + +The direct `src/run_oocyte.py` wrapper also works from a source checkout without +an editable install. + +## D11/D13 panel1 run + +The checked-in manifest contains the eight current ovary sections and resolves +its image paths relative to the workspace layout: + +```bash +python src/run_oocyte.py \ + --config exps/configs/oocyte_d11_d13_panel1/config.yaml \ + --manifest exps/configs/oocyte_d11_d13_panel1/samples.csv \ + --out-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1 +``` + +Completed matching samples resume by default. Pass `--no-resume` to recompute +them. `jobs` controls sample-level processes; candidate evaluation within one +sample remains deterministic and single-process. + +`donor13_v6` is the immutable numerical baseline. The separately named +`donor13_v6_rescue_v1` profile retains every v6 acceptance, retries rejected +seeds with a P95 annulus floor, discovers multiple components in crowded +coarse patches, and deduplicates the resulting persisted masks against v6. + +Required manifest columns are `sample_id`, `image_path`, and `pixel_size_um`. +Each enabled row must also provide either `channel_index` or `antibodies_path`. +Optional columns are `channel_name`, `enabled`, and `profile`. + +## Outputs + +Each completed sample writes: + +```text +/ + run_manifest.json + summary.json + runtime.json + coarse_candidates.csv + candidates.csv + masks/.npz + oocyte_labels.ome.tiff + oocyte_labels.csv + overview.png + accepted_duplicate_suspects.csv + rescue_diagnostics.csv # rescue profile only + oocytes.html + html_candidates.csv + html_assets/*.webp + html_assets/manifest.json +``` + +Every NPZ contains the image-bounded boolean mask, full-image bounding box, +source image shape, detector profile fingerprint, and metrics from the same +segmentation evaluation. `oocyte_labels.ome.tiff` is a tiled, compressed +`uint16` image composed from accepted persisted masks. Its mapping CSV records +label IDs and overlap pixels. It can be regenerated without rereading UCHL1. + +The batch root writes `batch_manifest.json`, `batch_summary.csv`, and +`batch_summary.json`, plus `spatial_qc/all_samples_overview.png`, an overview +index, and a combined duplicate-suspect table. A failed sample is recorded +without deleting successful sample outputs. + +It also writes `oocyte_detection_algorithm.html` and +`oocyte_review_index.html`. The algorithm page is self-contained and uses +inline SVG. Each sample page displays raw UCHL1 with the exact persisted cyan +mask, a whole-slide coordinate map, baseline/rescue filters, review-priority +flags, local browser review state, notes, and CSV/JSON export. + +`html_assets/manifest.json` binds every rank-named WebP to the raw source stat, +UCHL1 channel, patch center and radius, renderer version, and exact NPZ mask +SHA-256. Report regeneration reuses a thumbnail only when both that render +fingerprint and the WebP SHA-256 match. Candidate insertion or suppression can +therefore change display ranks without silently reusing another candidate's +image. + +When a rescue delta is supplied, donor13 samples also receive +`candidates_rescue_v1_combined.csv`, `oocyte_labels_rescue_v1.ome.tiff`, and +`oocyte_labels_rescue_v1.csv`. These combine baseline and rescue exact masks; +the original frozen v6 artifacts remain unchanged. + +## Final-label profiling + +Profile marker intensities only after selecting the final reviewed label image. +The profiler enumerates objects from that label image, not from a nucleus mask, +DeepCell output, or the optional candidate table. The candidate table can enrich +provenance columns but cannot add or remove expression rows. + +For example, profile the reviewed `13-23` v2 labels with: + +```bash +python src/run_oocyte_profile.py \ + --sample-id 13-23 \ + --image /path/to/registered.ome.tiff \ + --antibodies /path/to/antibodies.tsv \ + --labels /path/to/final/oocyte_labels.ome.tiff \ + --mapping /path/to/final/oocyte_labels.csv \ + --candidates /path/to/final/oocyte_candidates.csv \ + --out-dir /path/to/profiling \ + --pixel-size-um 0.5 +``` + +The output directory contains: + +```text +oocyte_by_marker.csv +oocyte_metadata.csv +oocyte_overview.csv +channel_manifest.csv +profiling_manifest.json +``` + +`oocyte_by_marker.csv` has one row per positive final label and one column per +registered channel. Values are raw within-mask means, matching the whole-cell +mean semantics of Aegle's `cell_by_marker.csv`. They are not background +subtracted, transformed, or normalized. All acquisition channels are retained; +`channel_manifest.csv` classifies DAPI as a nuclear stain rather than protein. + +The profiler validates that the label image and mapping contain exactly the same +positive labels, checks assigned pixel counts and bounding boxes, and records +source identities plus SHA-256 hashes for the label, mapping, antibody, candidate, +and output files. It scans labels and raw channels through bounded active regions +instead of materializing the full multichannel image. Empty negative-control +labels produce header-only marker and metadata tables with zero rows. + +## Review + +When review is enabled, the CLI writes: + +```text +review/ + accepted_candidates.csv + accepted_pages/ + novel_candidates.csv + novel_pages/ + missed_references.csv + missed_pages/ + reference_comparison.csv + summary.json +``` + +Every montage panel has a `#NNN` index that maps to `review_rank` in the CSV. +Cyan is the exact persisted candidate mask. Lime outlines are nearby accepted +masks and help identify crowded fields or duplicate candidates. + +In the per-sample Precision HTML, every candidate card has a `Hide mask` +button. It loads the matching raw UCHL1 patch on demand from the dynamic review +server and changes to `Show mask`; toggling affects only that card and does not +change its Accept, Reject, Unsure, or notes state. Open Precision through the +batch review console on port `8767` for this control. A plain static file server +can display the persisted-mask thumbnails but cannot provide on-demand raw +patches. + +Precision browser state is keyed by stable detection pass plus detector +component ID and is namespaced by the SHA-256 of `html_candidates.csv`; it is +not keyed by the displayed `#NNN` rank. `Export JSON` writes an identity-bound +`oocyte_precision_review` record. Use `Import JSON` to resume from that durable +record. Import rejects a different sample, raw-source stat, detector profile, +implementation version, or candidate-table SHA instead of applying decisions +to a changed queue. CSV remains a convenient flat view, but JSON is the release +input. + +Recommended manual values: + +- `manual_is_oocyte`: `yes`, `no`, or `uncertain`. +- `manual_mask_quality`: `good`, `undersegmented`, `oversegmented`, or `truncated`. +- `manual_duplicate_group`: assign the same short ID to duplicate panels. +- `manual_notes`: free text for unusual morphology or review context. + +The optional legacy JSONL only defines near-reference, novel, and missed review +buckets. It never changes detection, segmentation, scoring, acceptance, or +deduplication and must not be treated as biological ground truth. + +### Precision boundary recovery + +When Precision review marks a true oocyte as `Reject` with +`true_oocyte; mask_truncated; mask_off_target`, generate a separate replacement +review instead of lowering the frozen detector threshold globally: + +```bash +python src/run_oocyte_recall_review.py precision-boundary-review \ + --sample-dir /path/to/batch/13-21 \ + --precision-review-json /path/to/13-21_oocyte_review.json \ + --out-dir /path/to/batch/13-21/recall_analysis_precision_boundary_v1 \ + --patch-radius 220 +``` + +The command validates the Precision export against the current raw source, +profile, implementation version, and candidate-table SHA-256. It selects only +reviewed true-oocyte boundary failures, scans lower annulus percentiles, and +retains a proposal only when it contains at least 95% of the current component, +grows by a bounded amount, stays within physical and centroid gates, and avoids +other confirmed oocyte centers. It writes exact conservative/expanded NPZs, a +candidate CSV, WebP comparisons, an identity-bound HTML page, and a summary. It +does not modify current masks or label images. + +Each comparison shows raw context plus all current cyan masks, the frozen +current mask in yellow, a safe conservative proposal in green, and a safe +expanded proposal in orange. Choose `Use conservative` or `Use expanded` only +when that contour isolates the intended oocyte without absorbing a neighbor. +Use `Needs manual` when neither proposal is satisfactory, `Keep current` only +when the original mask is acceptable on reinspection, and `Not oocyte` only to +correct the Precision biological classification. Export JSON as the durable +record; final labels remain unchanged until a separate identity-validated +finalization step ingests those choices. + +Finalize a completed Precision boundary export into a self-contained, +Precision-only intermediate with: + +```bash +python src/run_oocyte_recall_review.py precision-boundary-finalize \ + --sample-dir /path/to/batch/13-21 \ + --precision-review-json /path/to/13-21_oocyte_review.json \ + --boundary-review-json /path/to/13-21_precision_boundary_review.json \ + --out-dir /path/to/batch/13-21/precision_resolved_v1 +``` + +The finalizer requires complete Precision decisions and complete, non-`Unsure` +boundary choices. It verifies both review identities and SHA-256 values, +validates every selected proposal against the bound candidate table and NPZ +metadata, and blocks masks with at least 25% smaller-object overlap. Accepted +current and replacement masks are copied into `reviewed_masks/`; source detector +and review-pack assets remain unchanged. + +The output includes `precision_resolved_candidates.csv`, all 132 decisions in +`precision_review_decisions.csv`, unresolved objects in +`manual_boundary_queue.csv`, a mask-overlap audit, an OME-TIFF plus label +mapping, exact copies of all review inputs, and +`precision_resolved_manifest.json` with artifact hashes. `Needs manual` objects +are deliberately absent from the label image until their boundaries are drawn +and reviewed. This intermediate always records `release_ready=false` and +`recall_complete=false`; do not run final expression profiling from it or call +it a sample release until the manual queue and whole-slide Recall review are +resolved. + +If `precision_resolved_v1/manual_boundary_queue.csv` is non-empty, generate a +native-resolution polygon editor without changing v1: + +```bash +python src/run_oocyte_recall_review.py precision-manual-boundary-review \ + --sample-dir /path/to/batch/13-21 \ + --base-resolved-dir /path/to/batch/13-21/precision_resolved_v1 \ + --out-dir /path/to/batch/13-21/recall_analysis_precision_manual_boundary_v1 \ + --patch-radius 220 +``` + +The page shows resolved neighbors in cyan, the rejected current target mask in +yellow, and the editable polygon in orange. Click to add vertices, drag a +vertex to refine it, use `Undo point` or `Clear` when needed, and toggle masks +to inspect raw UCHL1. The first vertex is yellow. Trace the intended outer +oocyte boundary, excluding adjacent oocytes and follicular halo, then select +`Accept contour`. Every contour edit returns the card to `Unreviewed`; accept +it again before exporting. `Export JSON` records full-image X/Y vertices and a +review identity. Browser local storage is not a final mask. + +After every card has a resolved choice, finalize the exported polygons into a +new immutable version: + +```bash +python src/run_oocyte_recall_review.py precision-manual-boundary-finalize \ + --sample-dir /path/to/batch/13-21 \ + --base-resolved-dir /path/to/batch/13-21/precision_resolved_v1 \ + --manual-review-json /path/to/13-21_precision_manual_boundary_review.json \ + --out-dir /path/to/batch/13-21/precision_resolved_v2 +``` + +The Python finalizer, not JavaScript, rasterizes the polygons. It rejects stale +base or candidate identities, missing and `Unsure` decisions, fewer than three +unique vertices, self-intersection, contours outside the reviewed patch, +contours that miss the reviewed center, equivalent diameters outside 10-100 +um, and any overlap pixel with a resolved mask. It copies every v1 mask into a +self-contained v2 directory, writes exact manual NPZs and a new whole-slide +label OME-TIFF, and records all hashes. A successful v2 sets +`manual_boundary_complete=true` but remains `release_ready=false` and +`recall_complete=false` until whole-slide Recall review is finalized. + +## Recall review and human-seeded diagnostics + +The candidate-card page is a precision workflow: it can reject false-positive +masks but cannot reveal an oocyte for which no accepted mask exists. The +separate recall reviewer covers image space with deterministic focal windows. +It displays a globally normalized whole-slide UCHL1 navigator, reads each +native-resolution focal patch on demand, overlays every intersecting persisted +mask, and records full-resolution missing-oocyte clicks. + +Generate and serve the `13-23` proof of concept with: + +```bash +python src/run_oocyte_recall_review.py \ + --sample-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23 \ + --host 0.0.0.0 \ + --port 8767 +``` + +Forward `8767` from the devcontainer in Cursor's `Ports` panel and open +`http://localhost:8767/recall_review.html`. Unlike the static precision pack, +this page requires the dedicated process because `/api/patch.webp`, +`/api/overlay.png`, and `/api/probe` read bounded raw-image regions and exact +NPZ masks on demand. + +After Precision finalization, Recall must use the immutable reviewed delivery +rather than the original `html_candidates.csv` masks. Generate a reviewed- +overlay bundle with: + +```bash +python src/run_oocyte_recall_review.py serve \ + --sample-dir /path/to/batch/13-21 \ + --overlay-dir /path/to/batch/13-21/precision_resolved_v2 \ + --generate-only +``` + +Only complete `precision_resolved_v1` or `precision_resolved_v2` deliveries +with zero unresolved manual boundaries are accepted. Startup verifies the +delivery manifest, every artifact SHA-256 and size, candidate count, NPZ hash, +sample identity, image geometry, and bounding boxes. The reviewed candidates +then drive the cyan overlays, whole-slide points, accepted counts, nearest- +accepted distance, and exact `already_covered` classification. Frozen refined, +coarse, and duplicate-suppressed detector tables remain the diagnostic source +for classifying genuine misses. + +The Recall page displays the bound overlay name and count. Its exported JSON +includes the reviewed manifest and candidate-table hashes. Browser state from +an automatic or different reviewed overlay is discarded, and analysis rejects +an export that does not match the currently generated Recall bundle. Analyze a +matching export normally; `--overlay-dir` is optional because the reviewed path +is recovered and revalidated from the bound identity: + +```bash +python src/run_oocyte_recall_review.py analyze \ + --sample-dir /path/to/batch/13-21 \ + --review-json /path/to/13-21_recall_review.json \ + --out-dir /path/to/batch/13-21/recall_analysis_v1 +``` + +For cohort review, generate the four donor13 consoles and one batch index with: + +```bash +python src/run_oocyte_recall_review.py serve-batch \ + --batch-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1 \ + --sample-id 13-21 \ + --sample-id 13-22 \ + --sample-id 13-23 \ + --sample-id 13-24 \ + --generate-only +``` + +When generating several samples together, bind reviewed overlays explicitly as +`SAMPLE_ID=PATH` entries. Samples without an entry keep their automatic masks: + +```bash +python src/run_oocyte_recall_review.py serve-batch \ + --batch-dir /path/to/batch \ + --sample-id 13-21 --sample-id 13-22 \ + --overlay 13-21=/path/to/batch/13-21/precision_resolved_v2 \ + --host 0.0.0.0 --port 8767 +``` + +With `--no-generate`, each sample recovers its previously bound overlay from +`recall_review/metadata.json`; static Recall and console pages are refreshed +without rereading raw images or changing metadata. + +Then serve the identity-matched bundles on one port without regenerating them: + +```bash +python src/run_oocyte_recall_review.py serve-batch \ + --batch-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1 \ + --sample-id 13-21 \ + --sample-id 13-22 \ + --sample-id 13-23 \ + --sample-id 13-24 \ + --host 0.0.0.0 \ + --port 8767 \ + --no-generate +``` + +Open `http://localhost:8767/` after forwarding the port. The root page links to +each sample's `review_console.html`, precision page, and recall page. Sample API +routes are isolated under `//api/...`; the server validates every +sample against its source image and candidate-table identity before accepting +requests. Page visits do not mark review work complete. Exported precision and +recall JSON remain the durable records. + +The review queue prioritizes near-threshold rejected candidates and proposal- +dense windows while still covering the entire image with overlapping windows. +For every inspected window, record `Complete`, `Has misses`, or `Unsure`. +Click `Add missed oocyte` and then the focal image to add a manual center. The +page immediately reports the nearest detector stages plus conservative and +expanded click-targeted segmentation diagnostics. A click is not an accepted +boundary and never modifies a production label image. + +Review state lives in browser local storage until exported. Export JSON for +scientific retention and offline analysis; CSV is a convenient flat audit +view. Analyze an exported JSON file with: + +```bash +python src/run_oocyte_recall_review.py analyze \ + --sample-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23 \ + --review-json /path/to/13-23_recall_review.json \ + --out-dir /path/to/13-23_recall_analysis +``` + +The analysis writes `recall_failure_analysis.csv`, `summary.json`, two +provisional masks per successful click under `provisional_masks/`, and a +`manual_seed_review.html` page. For each manual center, segmentation sweeps +annulus percentiles P95 through P60 in a compact native-resolution patch. The +highest-percentile component passing click-distance, centroid, and physical-size +gates is the conservative mask. The expanded mask is the largest lower-threshold +version that retains at least 70% overlap with the conservative component and +grows by no more than fourfold. Multi-lobed connected components are split by a +distance-transform watershed when they contain multiple cell-sized basins. In +offline analysis, every other manual center is also an exclusion point, so a +candidate cannot expand across another clicked oocyte. + +When the analysis directory is under the served sample directory, open its page +through the same dynamic server, for example +`http://localhost:8767/recall_analysis_v4/manual_seed_review.html`. Each card +shows raw UCHL1 plus existing cyan masks and the manual center on the left, the +green conservative mask in the middle, and the yellow expanded mask on the +right. Select `Use conservative`, `Use expanded`, `Neither`, `Duplicate`, or +`Unsure`, then export JSON. The selection export, not browser local storage, is +the durable review record. + +Failure classes identify the earliest actionable detector stage: +`proposal_miss`, `segmentation_miss`, `acceptance_miss`, `dedup_error`, or +`already_covered`. Both candidate masks remain diagnostic until this second +review is exported; analysis never changes a production label image. + +Finalize a completed second-stage JSON review with: + +```bash +python src/run_oocyte_recall_review.py finalize \ + --sample-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23 \ + --review-json /path/to/13-23_manual_seed_mask_review.json \ + --analysis-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23/recall_analysis_v4 \ + --out-dir /path/to/reviewed_manual_seed_delta_v1 +``` + +Finalization validates the sample identity and analysis SHA-256, joins choices +to the trusted analysis table by `annotation_id`, and ignores mask paths supplied +by the browser export. When Recall was generated against a reviewed Precision +overlay, the finalizer recovers that overlay from the export identity and carries +forward only those reviewed masks; it never falls back to rejected automatic +candidates. Accepted masks are copied into immutable reviewed NPZs. +`Neither`, `Duplicate`, and `Unsure` rows remain in the decision audit but do not +become labels. A 25% smaller-mask overlap with another reviewed or production +mask blocks finalization; smaller contacts are recorded in +`mask_overlap_audit.csv`. + +The output contains a manual-only label OME-TIFF, a separately named combined +rescue-v1-plus-manual OME-TIFF, candidate and mapping CSVs, all review decisions, +and `manual_seed_finalize_manifest.json` with hashes and sizes. Notes indicating +an incomplete or excessive boundary become `boundary_warning=true`. Existing +baseline and rescue-v1 labels are never overwritten. Use `--delta-only` when a +combined label image is not wanted. + +If `Neither` means a confirmed oocyte with two unacceptable threshold masks, +include `needs_manual_boundary` in its note. Freeze all safe choices as v1, then +generate a polygon editor only for those confirmed unresolved objects: + +```bash +python src/run_oocyte_recall_review.py recall-manual-boundary-review \ + --sample-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-21 \ + --manual-review-json /path/to/13-21_manual_seed_mask_review.json \ + --base-finalize-dir /path/to/reviewed_manual_seed_delta_v1 \ + --out-dir /path/to/recall_analysis_manual_boundary_v1 +``` + +The contour page shows all v1 masks in cyan and the rejected conservative mask +in yellow. Trace the intended boundary, choose `Accept contour`, and export the +identity-bound JSON. Finalize it into a separately named v2 delivery with: + +```bash +python src/run_oocyte_recall_review.py recall-manual-boundary-finalize \ + --sample-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-21 \ + --base-finalize-dir /path/to/reviewed_manual_seed_delta_v1 \ + --review-json /path/to/13-21_recall_manual_boundary_review.json \ + --out-dir /path/to/reviewed_manual_seed_delta_v2 +``` + +The finalizer independently validates a simple polygon, requires the reviewed +center inside it, enforces a 10-100 um equivalent-diameter range, and rejects any +overlap pixel with resolved masks or another manual contour. It verifies every +v1 artifact before composing v2 and never modifies v1. + +If review identifies a fragmented expanded mask, generate the opt-in shape +recovery delta with: + +```bash +python src/run_oocyte_recall_review.py shape-review \ + --sample-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23 \ + --recall-review-json /path/to/13-23_recall_review.json \ + --manual-review-json /path/to/13-23_manual_seed_mask_review.json \ + --out-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23/recall_analysis_v4/shape_recovery_v1 +``` + +This mode leaves the standard 4x expansion cap unchanged. Its candidate may +grow by up to 6x only when it still overlaps at least 70% of the conservative +mask, has diameter at least 20 um, circularity at least 0.80, solidity at least +0.90, centroid offset at most 25 px, and does not reach another manual center. +The generated page contains only changed masks. Yellow is the frozen v4 result; +orange is shape recovery. Export `Keep v4`, `Use recovery`, `Exclude`, or +`Unsure` decisions before creating a separately named combined v2. + +Finalize the exported shape review against the immutable v1 delivery with: + +```bash +python src/run_oocyte_recall_review.py shape-finalize \ + --sample-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23 \ + --shape-review-json /path/to/13-23_shape_recovery_review.json \ + --base-finalize-dir /path/to/reviewed_manual_seed_delta_v1 \ + --out-dir /path/to/reviewed_manual_seed_delta_v2 +``` + +Shape finalization verifies every v1 manifest artifact before reading it. `Use +recovery` replaces an accepted v1 mask or adds a mask that v1 excluded; `Keep +v4` carries the v1 decision forward; `Exclude` removes it; unresolved `Unsure` +decisions block finalization. It reruns manual/manual and manual/production +overlap checks, then writes self-contained reviewed NPZs, a manual-only v2 +OME-TIFF, and an independently named combined v2 OME-TIFF. Neither v1 nor the +frozen detector outputs are modified. + +Prioritized hard-case review can improve the detector but does not by itself +measure global recall. A global recall claim requires explicit disposition of +every relevant tissue window. Any learned profile must preserve the frozen v6 +and rescue-v1 outputs and must be checked against precision review plus donor11 +as a biologically confirmed no-oocyte negative control. + +## Reviewed release packaging + +Final delivery is separate from detector and review working directories. The +release builder copies only final reviewed labels and masks, raw within-mask +profiles, exported review evidence, and provenance manifests into a new +immutable directory. It references raw OME-TIFFs by size and modification-time +identity rather than duplicating them. + +Build the reviewed donor11/donor13 panel1 release with embedded static review +images using: + +```bash +python src/run_oocyte_release.py build \ + --spec exps/configs/oocyte_d11_d13_panel1/release_v6.yaml \ + --out-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_delivery_v6 +``` + +The build fails if the destination already exists, any source hash or profiling +identity is inconsistent, a positive sample is empty, or a donor11 negative +control has a nonzero label or accepted detector/rescue diagnostic. It validates +the temporary package before atomically publishing the destination. + +Verify a completed package independently with: + +```bash +python src/run_oocyte_release.py validate \ + --release-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_delivery_v6 +``` + +The validator checks every packaged SHA-256, label-to-mapping pixel counts, +candidate and profiling IDs, marker schemas, sample totals, and cohort table +composition. `batch_oocyte_by_marker.csv` is the cohort-level combined protein +expression matrix; `batch_oocyte_metadata.csv` contains geometry and provenance. +Per-sample deliverables are under `samples//`, and +`oocyte_review_index.html` is the release summary page. + +Every sample's `review_console.html` begins with an embedded, downsampled +whole-slide raw-UCHL1 view. Positive pages initially show the exact final NPZ +boundaries in cyan; Hide masks/Show masks switches between masked and raw +overviews, and clicking a mask location jumps to the nearest matching cell card. +Each positive card also embeds raw UCHL1 and the same patch with the exact +delivered mask, with its own Hide mask/Show mask control and provenance filters. +Negative-control pages show the raw whole-slide overview but no boundaries, +hotspots, or cards, and explicitly report zero final oocytes. All controls work +when the HTML is opened directly through `file://` and make no API or network +request. Relative links expose expression, metadata, labels, mapping, candidates, +review evidence, and checksums. Raw OME-TIFFs remain outside the package and are +not required to view any embedded image. + +## Incremental rescue validation + +The rescue profile can be reviewed without rerunning the expensive frozen v6 +refinement. The delta command loads completed v6 candidates and exact masks, +reads only local raw-UCHL1 patches, and writes a rescue-only standard review +batch: + +```bash +python src/run_oocyte_rescue_delta.py \ + --baseline-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1 \ + --out-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_rescue_v1_delta +``` + +Regenerate the combined per-sample HTML separately when needed: + +```bash +python src/run_oocyte_report.py \ + --batch-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1 \ + --rescue-delta-dir /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_rescue_v1_delta \ + --references /workspaces/1-spatial_frs_analysis/frs-atlas-phenocycler/output/ovary_panel1/v2_4/oocyte_review_final.jsonl +``` + +## Validation and limitations + +The frozen `donor13_v6` profile reproduces sample `13-23` with 237 coarse +proposals, 318 refined candidates, and 135 accepted candidates. The representative +`13-23/#680` mask matches the research mask pixel-for-pixel. The production +`13-23` run took about 15.8 minutes and used bounded proposal-local caches. + +Across donor13, frozen v6 produces `352` accepts. The rescue delta adds `99` +visually inspected candidates (`31`, `8`, `58`, and `2` by section). Actual-mask +overlap then suppresses `22` lower-score duplicate v6 rows, giving `132`, `63`, +`187`, and `47` combined review candidates, or `429` total. In the provisional +legacy-miss self-audit, rescue recovered all `21` visually definite examples and +none of `28` visually false references; this is development evidence, not +biological ground truth or a formal sensitivity/specificity estimate. + +The four combined donor13 OME label images match their raw source dimensions, +have one mapping row per review candidate, and contain no assigned overlap +pixels. All eight sample HTML pages and the batch index were checked, including +desktop/mobile rendering of the 187-card `13-23` page. The complete baseline +directory including HTML assets occupies about 434 MB; the rescue delta occupies +about 58 MB. + +Donor13 has engineering visual-review evidence. Some open-ring or crescent +rescue masks remain explicitly review-priority cases. Donor11 was confirmed as +a no-oocyte negative control. The frozen detector and rescue diagnostics yield +zero accepted objects across sections 11-21 through 11-24. Machine-accepted +counts in development directories remain review candidates, not final oocyte +counts. + +Run the fast suite with: + +```bash +python -m unittest discover -s tests/oocyte -v +``` + +Run the local donor13 parity tests only where the research fixtures and raw image +are available: + +```bash +AEGLE_RUN_OOCYTE_LOCAL_REGRESSION=1 \ + python -m unittest tests.oocyte.test_local_regression -v +``` diff --git a/exps/configs/oocyte_d11_d13_panel1/config.yaml b/exps/configs/oocyte_d11_d13_panel1/config.yaml new file mode 100644 index 0000000..7f49a9d --- /dev/null +++ b/exps/configs/oocyte_d11_d13_panel1/config.yaml @@ -0,0 +1,24 @@ +detector_profile: donor13_v6 +jobs: 4 +continue_on_error: true +resume_completed: true +channel_name: UCHL1 +comparison_references: ../../../../frs-atlas-phenocycler/output/ovary_panel1/v2_4/oocyte_review_final.jsonl + +review: + enabled: true + montage_columns: 3 + montage_rows: 4 + neighbor_overlays: true + max_neighbor_overlays: 8 + reference_min_final_score: 0.35 + match_radius_px: 100.0 + +html_report: + enabled: true + patch_radius_px: 180 + rescue_delta_dir: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_rescue_v1_delta + export_combined_labels: true + +experimental: + border_rescue_enabled: false diff --git a/exps/configs/oocyte_d11_d13_panel1/release_v6.yaml b/exps/configs/oocyte_d11_d13_panel1/release_v6.yaml new file mode 100644 index 0000000..c7dc1d4 --- /dev/null +++ b/exps/configs/oocyte_d11_d13_panel1/release_v6.yaml @@ -0,0 +1,131 @@ +release_name: d11_d13_panel1_reviewed_v6 +algorithm_document: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/oocyte_detection_algorithm.html + +samples: + - sample_id: 11-21 + role: negative_control + image: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_11-21_Ovary_Central_Superior_Mesenteric_Scan1.ome.tiff + antibodies: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/extras/antibodies.tsv + final_labels: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-21/oocyte_labels.ome.tiff + final_mapping: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-21/oocyte_labels.csv + final_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-21/oocyte_labels.csv + profiling_dir: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-21/profiling_negative_control_v1 + detector_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-21/candidates.csv + rescue_diagnostics: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_rescue_v1_delta/11-21/rescue_diagnostics.csv + provenance_files: + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-21/run_manifest.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-21/summary.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-21/profiling_negative_control_v1/profiling_manifest.json + + - sample_id: 11-22 + role: negative_control + image: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_11-22_Ovary_Central_Inferior_Mesenteric_Scan1.ome.tiff + antibodies: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/extras/antibodies.tsv + final_labels: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-22/oocyte_labels.ome.tiff + final_mapping: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-22/oocyte_labels.csv + final_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-22/oocyte_labels.csv + profiling_dir: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-22/profiling_negative_control_v1 + detector_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-22/candidates.csv + rescue_diagnostics: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_rescue_v1_delta/11-22/rescue_diagnostics.csv + provenance_files: + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-22/run_manifest.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-22/summary.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-22/profiling_negative_control_v1/profiling_manifest.json + + - sample_id: 11-23 + role: negative_control + image: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_11-23_Ovary_Central_Inferior_Antimesenteric_Scan1.ome.tiff + antibodies: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/extras/antibodies.tsv + final_labels: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-23/oocyte_labels.ome.tiff + final_mapping: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-23/oocyte_labels.csv + final_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-23/oocyte_labels.csv + profiling_dir: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-23/profiling_negative_control_v1 + detector_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-23/candidates.csv + rescue_diagnostics: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_rescue_v1_delta/11-23/rescue_diagnostics.csv + provenance_files: + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-23/run_manifest.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-23/summary.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-23/profiling_negative_control_v1/profiling_manifest.json + + - sample_id: 11-24 + role: negative_control + image: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_11-24_Ovary_Central_Superior_Antimesenteric_Scan1.ome.tiff + antibodies: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/extras/antibodies.tsv + final_labels: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-24/oocyte_labels.ome.tiff + final_mapping: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-24/oocyte_labels.csv + final_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-24/oocyte_labels.csv + profiling_dir: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-24/profiling_negative_control_v1 + detector_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-24/candidates.csv + rescue_diagnostics: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1_rescue_v1_delta/11-24/rescue_diagnostics.csv + provenance_files: + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-24/run_manifest.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-24/summary.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/11-24/profiling_negative_control_v1/profiling_manifest.json + + - sample_id: 13-21 + role: positive + image: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_13-21_Ovary_Central_Inferior_Mesenteric_Scan1.ome.tiff + antibodies: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/extras/antibodies.tsv + final_labels: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-21/reviewed_manual_seed_delta_v2/oocyte_labels_reviewed_manual_seed_v2.ome.tiff + final_mapping: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-21/reviewed_manual_seed_delta_v2/oocyte_labels_reviewed_manual_seed_v2_mapping.csv + final_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-21/reviewed_manual_seed_delta_v2/oocyte_candidates_reviewed_manual_seed_v2.csv + profiling_dir: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-21/profiling_reviewed_v2 + review_exports: + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-21_oocyte_review_1.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-21_precision_boundary_review.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-21_precision_manual_boundary_review.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-21_recall_review.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-21_manual_seed_mask_review.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-21_recall_manual_boundary_review.json + provenance_files: + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-21/reviewed_manual_seed_delta_v2/recall_manual_boundary_finalize_manifest.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-21/profiling_reviewed_v2/profiling_manifest.json + + - sample_id: 13-22 + role: positive + image: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_13-22_Ovary_Central_Inferior_Antimesenteric_Scan1.ome.tiff + antibodies: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/extras/antibodies.tsv + final_labels: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-22/reviewed_manual_seed_delta_v1/oocyte_labels_rescue_v1_plus_manual_seed_v1.ome.tiff + final_mapping: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-22/reviewed_manual_seed_delta_v1/oocyte_labels_rescue_v1_plus_manual_seed_v1_mapping.csv + final_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-22/reviewed_manual_seed_delta_v1/oocyte_candidates_rescue_v1_plus_manual_seed_v1.csv + profiling_dir: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-22/profiling_reviewed_v1 + review_exports: + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-22_oocyte_review.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-22_recall_review_1.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-22_manual_seed_mask_review.json + provenance_files: + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-22/reviewed_manual_seed_delta_v1/manual_seed_finalize_manifest.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-22/profiling_reviewed_v1/profiling_manifest.json + + - sample_id: 13-23 + role: positive + image: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_13-23_Ovary_Central_Superior_Antimesenteric_Scan1.ome.tiff + antibodies: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/extras/antibodies.tsv + final_labels: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23/reviewed_manual_seed_survey_v2/oocyte_labels_reviewed_manual_seed_v2.ome.tiff + final_mapping: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23/reviewed_manual_seed_survey_v2/oocyte_labels_reviewed_manual_seed_v2_mapping.csv + final_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23/reviewed_manual_seed_survey_v2/oocyte_candidates_reviewed_manual_seed_v2.csv + profiling_dir: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23/profiling_reviewed_survey_v2 + review_exports: + - "/workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-23_oocyte_review _1.json" + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-23_recall_review_2.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-23_manual_seed_mask_review_1.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-23_recall_manual_boundary_review_1.json + provenance_files: + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23/reviewed_manual_seed_survey_v2/recall_manual_boundary_finalize_manifest.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-23/profiling_reviewed_survey_v2/profiling_manifest.json + + - sample_id: 13-24 + role: positive + image: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_13-24_Ovary_Central_Superior_Mesenteric_Scan1.ome.tiff + antibodies: /workspaces/1-spatial_frs_analysis/data/Ovary/D11_13/Scan1/extras/antibodies.tsv + final_labels: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-24/reviewed_manual_seed_delta_v1/oocyte_labels_rescue_v1_plus_manual_seed_v1.ome.tiff + final_mapping: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-24/reviewed_manual_seed_delta_v1/oocyte_labels_rescue_v1_plus_manual_seed_v1_mapping.csv + final_candidates: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-24/reviewed_manual_seed_delta_v1/oocyte_candidates_rescue_v1_plus_manual_seed_v1.csv + profiling_dir: /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-24/profiling_reviewed_v1 + review_exports: + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-24_oocyte_review_1.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-24_recall_review.json + - /workspaces/1-spatial_frs_analysis/notes/oocytes_detection/reviews/13-24_manual_seed_mask_review.json + provenance_files: + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-24/reviewed_manual_seed_delta_v1/manual_seed_finalize_manifest.json + - /workspaces/1-spatial_frs_analysis/oocyte-output/d11_d13_panel1/13-24/profiling_reviewed_v1/profiling_manifest.json diff --git a/exps/configs/oocyte_d11_d13_panel1/samples.csv b/exps/configs/oocyte_d11_d13_panel1/samples.csv new file mode 100644 index 0000000..1295787 --- /dev/null +++ b/exps/configs/oocyte_d11_d13_panel1/samples.csv @@ -0,0 +1,9 @@ +sample_id,image_path,antibodies_path,channel_name,channel_index,pixel_size_um,enabled,profile +11-21,../../../../data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_11-21_Ovary_Central_Superior_Mesenteric_Scan1.ome.tiff,../../../../data/Ovary/D11_13/Scan1/extras/antibodies.tsv,UCHL1,,0.5,true,donor13_v6 +11-22,../../../../data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_11-22_Ovary_Central_Inferior_Mesenteric_Scan1.ome.tiff,../../../../data/Ovary/D11_13/Scan1/extras/antibodies.tsv,UCHL1,,0.5,true,donor13_v6 +11-23,../../../../data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_11-23_Ovary_Central_Inferior_Antimesenteric_Scan1.ome.tiff,../../../../data/Ovary/D11_13/Scan1/extras/antibodies.tsv,UCHL1,,0.5,true,donor13_v6 +11-24,../../../../data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_11-24_Ovary_Central_Superior_Antimesenteric_Scan1.ome.tiff,../../../../data/Ovary/D11_13/Scan1/extras/antibodies.tsv,UCHL1,,0.5,true,donor13_v6 +13-21,../../../../data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_13-21_Ovary_Central_Inferior_Mesenteric_Scan1.ome.tiff,../../../../data/Ovary/D11_13/Scan1/extras/antibodies.tsv,UCHL1,,0.5,true,donor13_v6 +13-22,../../../../data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_13-22_Ovary_Central_Inferior_Antimesenteric_Scan1.ome.tiff,../../../../data/Ovary/D11_13/Scan1/extras/antibodies.tsv,UCHL1,,0.5,true,donor13_v6 +13-23,../../../../data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_13-23_Ovary_Central_Superior_Antimesenteric_Scan1.ome.tiff,../../../../data/Ovary/D11_13/Scan1/extras/antibodies.tsv,UCHL1,,0.5,true,donor13_v6 +13-24,../../../../data/Ovary/D11_13/Scan1/processed_hubmap/D11_13_Scan1.er_manual_13-24_Ovary_Central_Superior_Mesenteric_Scan1.ome.tiff,../../../../data/Ovary/D11_13/Scan1/extras/antibodies.tsv,UCHL1,,0.5,true,donor13_v6 diff --git a/exps/templates/oocyte_samples_template.csv b/exps/templates/oocyte_samples_template.csv new file mode 100644 index 0000000..1593f33 --- /dev/null +++ b/exps/templates/oocyte_samples_template.csv @@ -0,0 +1,2 @@ +sample_id,image_path,antibodies_path,channel_name,channel_index,pixel_size_um,enabled,profile +13-23,/path/to/13-23.ome.tiff,/path/to/antibodies.tsv,UCHL1,,0.5,true,donor13_v6 diff --git a/exps/templates/oocyte_template.yaml b/exps/templates/oocyte_template.yaml new file mode 100644 index 0000000..953c278 --- /dev/null +++ b/exps/templates/oocyte_template.yaml @@ -0,0 +1,24 @@ +detector_profile: donor13_v6 +jobs: 4 +continue_on_error: true +resume_completed: true +channel_name: UCHL1 +comparison_references: null + +review: + enabled: true + montage_columns: 3 + montage_rows: 4 + neighbor_overlays: true + max_neighbor_overlays: 8 + reference_min_final_score: 0.35 + match_radius_px: 100.0 + +html_report: + enabled: true + patch_radius_px: 180 + rescue_delta_dir: null + export_combined_labels: true + +experimental: + border_rescue_enabled: false diff --git a/readme.md b/readme.md index b2ba4ae..099ffac 100644 --- a/readme.md +++ b/readme.md @@ -19,6 +19,14 @@ This pipeline is developed as part of the PennTMC project. - **Cell Segmentation** - Run segmentation with *Mesmer* using `scripts/run_main.sh` and do cell profiling to generate a cell x antibody matrix. +- **Standalone Oocyte Segmentation** + - Detect and segment ovary oocytes directly from registered raw UCHL1 without + DeepCell masks, then complete identity-bound Precision/Recall review, + expression profiling, and immutable release packaging. See + [`docs/oocyte_detection.md`](docs/oocyte_detection.md). For Codex-assisted + operation, start with + [`aegle/oocyte/AGENTS.md`](aegle/oocyte/AGENTS.md). + - **Downstream Analysis** - Perform pixel-level quality control (QC), cell-level QC, clustering, maker test using `scripts/run_analysis.sh`. diff --git a/setup.py b/setup.py index 5834afb..3d0bd3b 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,10 @@ "scipy>=1.7.0", "leidenalg>=0.9.0", # Required for Leiden clustering in scanpy ], + "oocyte": [ + "scipy>=1.7.0", + "scikit-image>=0.19.0", + ], "gpu": [ # CuPy for GPU-accelerated computing # Install the appropriate version for your CUDA: @@ -49,4 +53,4 @@ author_email="kuangda@seas.upenn.edu", description="A package for CODEX image analysis", url="https://github.com/kuang-da/aegle", -) \ No newline at end of file +) diff --git a/src/run_oocyte.py b/src/run_oocyte.py new file mode 100644 index 0000000..b2da22a --- /dev/null +++ b/src/run_oocyte.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Command-line entry point for standalone raw-UCHL1 oocyte detection.""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from aegle.oocyte.batch import detect_oocyte_batch +from aegle.oocyte.report import generate_html_reports +from aegle.oocyte.review import generate_review_pack + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Detect and segment oocytes directly from raw UCHL1 OME-TIFFs." + ) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--config", type=Path) + parser.add_argument("--jobs", type=int) + parser.add_argument( + "--fail-fast", + action="store_true", + help="Stop scheduling samples after the first failure.", + ) + parser.add_argument( + "--no-resume", + action="store_true", + help="Recompute samples even when matching complete outputs already exist.", + ) + return parser.parse_args() + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + ) + args = parse_args() + settings = {} + if args.config is not None: + with args.config.open() as handle: + settings = yaml.safe_load(handle) or {} + experimental = settings.get("experimental", {}) or {} + if bool(experimental.get("border_rescue_enabled", False)): + raise ValueError("donor13_v6 does not support experimental border rescue") + jobs = args.jobs if args.jobs is not None else int(settings.get("jobs", 1)) + continue_on_error = bool(settings.get("continue_on_error", True)) + if args.fail_fast: + continue_on_error = False + result = detect_oocyte_batch( + args.manifest, + out_dir=args.out_dir, + jobs=jobs, + continue_on_error=continue_on_error, + resume_completed=not args.no_resume + and bool(settings.get("resume_completed", True)), + default_profile=str(settings.get("detector_profile", "donor13_v6")), + default_channel_name=str(settings.get("channel_name", "UCHL1")), + ) + references_value = settings.get("comparison_references") + references_path = None + if references_value: + references_path = Path(str(references_value)).expanduser() + if not references_path.is_absolute(): + config_dir = ( + args.config.resolve().parent if args.config is not None else Path.cwd() + ) + references_path = (config_dir / references_path).resolve() + review = settings.get("review", {}) or {} + if bool(review.get("enabled", True)) and bool( + (result.summary["status"] == "complete").any() + ): + review_result = generate_review_pack( + result.out_dir, + references_path=references_path, + reference_min_final_score=float( + review.get("reference_min_final_score", 0.35) + ), + match_radius_px=float(review.get("match_radius_px", 100.0)), + columns=int(review.get("montage_columns", 3)), + rows=int(review.get("montage_rows", 4)), + neighbor_overlays=bool(review.get("neighbor_overlays", True)), + max_neighbor_overlays=int(review.get("max_neighbor_overlays", 8)), + ) + logging.info( + "Review counts: accepted=%s novel=%s missed=%s", + review_result.accepted_count, + review_result.novel_count, + review_result.missed_reference_count, + ) + html_report = settings.get("html_report", {}) or {} + if bool(html_report.get("enabled", True)) and bool( + (result.summary["status"] == "complete").any() + ): + rescue_delta_value = html_report.get("rescue_delta_dir") + rescue_delta_dir = None + if rescue_delta_value: + rescue_delta_dir = Path(str(rescue_delta_value)).expanduser().resolve() + html_result = generate_html_reports( + result.out_dir, + rescue_delta_dir=rescue_delta_dir, + references_path=references_path, + patch_radius_px=int(html_report.get("patch_radius_px", 180)), + export_combined_labels=bool( + html_report.get("export_combined_labels", True) + ), + ) + logging.info("HTML review index: %s", html_result.batch_index) + counts = result.summary["status"].value_counts().to_dict() + logging.info("Batch status counts: %s", counts) + logging.info("Batch summary: %s", result.artifact_paths["batch_summary_csv"]) + return 1 if result.failed_count else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/run_oocyte_profile.py b/src/run_oocyte_profile.py new file mode 100644 index 0000000..90b2d5e --- /dev/null +++ b/src/run_oocyte_profile.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Profile reviewed oocyte labels against every registered raw channel.""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from aegle.oocyte.profiling import profile_oocyte_labels + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Compute one raw within-mask marker-intensity row per final oocyte label." + ) + ) + parser.add_argument("--sample-id", required=True) + parser.add_argument("--image", type=Path, required=True) + parser.add_argument("--antibodies", type=Path, required=True) + parser.add_argument("--labels", type=Path, required=True) + parser.add_argument("--mapping", type=Path, required=True) + parser.add_argument("--candidates", type=Path) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--pixel-size-um", type=float, required=True) + parser.add_argument("--max-region-height-px", type=int, default=512) + parser.add_argument("--merge-gap-px", type=int, default=16) + return parser.parse_args() + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + ) + args = parse_args() + result = profile_oocyte_labels( + sample_id=args.sample_id, + image_path=args.image, + antibodies_path=args.antibodies, + label_path=args.labels, + mapping_path=args.mapping, + candidates_path=args.candidates, + out_dir=args.out_dir, + pixel_size_um=args.pixel_size_um, + max_region_height_px=args.max_region_height_px, + merge_gap_px=args.merge_gap_px, + ) + logging.info("Marker matrix: %s", result.artifact_paths["markers"]) + logging.info("Metadata: %s", result.artifact_paths["metadata"]) + logging.info("Manifest: %s", result.artifact_paths["manifest"]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/run_oocyte_recall_review.py b/src/run_oocyte_recall_review.py new file mode 100644 index 0000000..43155af --- /dev/null +++ b/src/run_oocyte_recall_review.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +"""Launch or analyze the standalone oocyte recall-review workflow.""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path +from typing import Sequence + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from aegle.oocyte.recall_review import ( + DEFAULT_OVERVIEW_DOWNSAMPLE, + DEFAULT_WINDOW_RADIUS_PX, + DEFAULT_WINDOW_STRIDE_PX, + analyze_recall_review, + generate_recall_review_bundle, + serve_recall_review, +) +from aegle.oocyte.recall_review_batch import ( + generate_batch_recall_review_bundle, + serve_batch_recall_review, +) +from aegle.oocyte.recall_manual_boundary_finalize import ( + finalize_recall_manual_boundary_review, +) +from aegle.oocyte.recall_manual_boundary_review import ( + generate_recall_manual_boundary_review, +) +from aegle.oocyte.manual_seed_finalize import finalize_manual_seed_review +from aegle.oocyte.precision_boundary_review import ( + generate_precision_boundary_review, +) +from aegle.oocyte.precision_boundary_finalize import ( + finalize_precision_boundary_review, +) +from aegle.oocyte.precision_manual_boundary_review import ( + generate_precision_manual_boundary_review, +) +from aegle.oocyte.precision_manual_boundary_finalize import ( + finalize_precision_manual_boundary_review, +) +from aegle.oocyte.shape_recovery_review import generate_shape_recovery_review +from aegle.oocyte.shape_recovery_finalize import finalize_shape_recovery_review + + +def _add_coverage_geometry_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--window-radius", + type=int, + default=DEFAULT_WINDOW_RADIUS_PX, + help="Full-resolution Recall window radius in pixels (default: %(default)s).", + ) + parser.add_argument( + "--window-stride", + type=int, + default=DEFAULT_WINDOW_STRIDE_PX, + help="Full-resolution Recall grid stride in pixels (default: %(default)s).", + ) + parser.add_argument( + "--overview-downsample", + type=int, + default=DEFAULT_OVERVIEW_DOWNSAMPLE, + help="Whole-slide navigator downsample factor (default: %(default)s).", + ) + + +def _serve_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate and serve one sample's raw-UCHL1 recall review page." + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument( + "--overlay-dir", + type=Path, + help="Immutable reviewed Precision delivery to use for Recall masks.", + ) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8767) + _add_coverage_geometry_arguments(parser) + parser.add_argument( + "--generate-only", + action="store_true", + help="Generate the review bundle without starting the HTTP server.", + ) + parser.add_argument( + "--no-generate", + action="store_true", + help="Serve an existing identity-matched bundle without regenerating it.", + ) + return parser + + +def _analyze_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Ingest an exported recall review and classify missing clicks." + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--review-json", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument( + "--overlay-dir", + type=Path, + help="Reviewed overlay bound to the exported Recall JSON; inferred by default.", + ) + return parser + + +def _serve_batch_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate and serve multiple sample review consoles on one port." + ) + parser.add_argument("--batch-dir", type=Path, required=True) + parser.add_argument( + "--sample-id", + action="append", + dest="sample_ids", + help="Sample ID to include; repeat for multiple samples. Defaults to all samples.", + ) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8767) + _add_coverage_geometry_arguments(parser) + parser.add_argument( + "--overlay", + action="append", + dest="overlay_specs", + metavar="SAMPLE_ID=PATH", + help="Reviewed Precision overlay for one sample; repeat as needed.", + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "--generate-only", + action="store_true", + help="Generate all selected sample bundles and the batch index, then exit.", + ) + mode.add_argument( + "--no-generate", + action="store_true", + help="Serve existing identity-matched bundles without regenerating them.", + ) + return parser + + +def _parse_overlay_specs(values: Sequence[str] | None) -> dict[str, Path]: + overlays: dict[str, Path] = {} + for raw_value in values or (): + sample_id, separator, raw_path = str(raw_value).partition("=") + sample_id = sample_id.strip() + raw_path = raw_path.strip() + if not separator or not sample_id or not raw_path: + raise ValueError( + f"invalid --overlay {raw_value!r}; expected SAMPLE_ID=PATH" + ) + if sample_id in overlays: + raise ValueError(f"duplicate --overlay for sample {sample_id}") + overlays[sample_id] = Path(raw_path).resolve() + return overlays + + +def _finalize_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Finalize a completed manual-seed mask review into label artifacts." + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--review-json", type=Path, required=True) + parser.add_argument("--analysis-dir", type=Path) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument( + "--delta-only", + action="store_true", + help="Write only the reviewed manual delta, without a combined label image.", + ) + return parser + + +def _shape_review_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate a delta review for shape-gated manual-seed masks." + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--recall-review-json", type=Path, required=True) + parser.add_argument("--manual-review-json", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + return parser + + +def _precision_boundary_review_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate a boundary-recovery review for Precision mask failures." + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--precision-review-json", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--patch-radius", type=int, default=220) + return parser + + +def _precision_boundary_finalize_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Finalize completed Precision and boundary reviews into an immutable " + "Precision-only intermediate." + ) + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--precision-review-json", type=Path, required=True) + parser.add_argument("--boundary-review-json", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + return parser + + +def _precision_manual_boundary_review_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate a polygon editor for unresolved Precision boundaries." + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--base-resolved-dir", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--patch-radius", type=int, default=220) + return parser + + +def _precision_manual_boundary_finalize_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Finalize reviewed Precision polygons into an immutable v2 " + "intermediate." + ) + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--base-resolved-dir", type=Path, required=True) + parser.add_argument("--manual-review-json", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + return parser + + +def _recall_manual_boundary_review_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate a polygon editor for unresolved Recall boundaries." + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--manual-review-json", type=Path, required=True) + parser.add_argument("--base-finalize-dir", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--patch-radius", type=int, default=220) + return parser + + +def _recall_manual_boundary_finalize_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Finalize reviewed Recall polygons into a manual-seed v2 delivery." + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--base-finalize-dir", type=Path, required=True) + parser.add_argument("--review-json", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + return parser + + +def _shape_finalize_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Finalize shape-recovery decisions into a new v2 label delivery." + ) + parser.add_argument("--sample-dir", type=Path, required=True) + parser.add_argument("--shape-review-json", type=Path, required=True) + parser.add_argument("--base-finalize-dir", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument( + "--delta-only", + action="store_true", + help="Write only the reviewed manual v2 delta, without a combined image.", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + if arguments and arguments[0] == "serve-batch": + args = _serve_batch_parser().parse_args(arguments[1:]) + try: + overlay_dirs = _parse_overlay_specs(args.overlay_specs) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + if args.generate_only: + bundle = generate_batch_recall_review_bundle( + args.batch_dir, + sample_ids=args.sample_ids, + overlay_dirs=overlay_dirs, + generate_samples=True, + window_radius_px=args.window_radius, + window_stride_px=args.window_stride, + overview_downsample=args.overview_downsample, + ) + logging.getLogger(__name__).info( + "Batch review bundle generated: samples=%s candidates=%s " + "windows=%s index=%s", + len(bundle.sample_ids), + bundle.total_candidate_count, + bundle.total_window_count, + bundle.index_path, + ) + return 0 + serve_batch_recall_review( + args.batch_dir, + sample_ids=args.sample_ids, + overlay_dirs=overlay_dirs, + host=args.host, + port=args.port, + generate=not args.no_generate, + window_radius_px=args.window_radius, + window_stride_px=args.window_stride, + overview_downsample=args.overview_downsample, + ) + return 0 + if arguments and arguments[0] == "analyze": + args = _analyze_parser().parse_args(arguments[1:]) + table_path = analyze_recall_review( + args.sample_dir, + args.review_json, + args.out_dir, + overlay_dir=args.overlay_dir, + ) + logging.getLogger(__name__).info("Recall analysis written to %s", table_path) + return 0 + if arguments and arguments[0] == "finalize": + args = _finalize_parser().parse_args(arguments[1:]) + result = finalize_manual_seed_review( + args.sample_dir, + args.review_json, + args.out_dir, + analysis_dir=args.analysis_dir, + write_combined_labels=not args.delta_only, + ) + logging.getLogger(__name__).info( + "Manual-seed review finalized: accepted=%s warnings=%s manifest=%s", + result.accepted_count, + result.boundary_warning_count, + result.manifest_path, + ) + return 0 + if arguments and arguments[0] == "shape-review": + args = _shape_review_parser().parse_args(arguments[1:]) + result = generate_shape_recovery_review( + args.sample_dir, + args.recall_review_json, + args.manual_review_json, + args.out_dir, + ) + logging.getLogger(__name__).info( + "Shape-recovery review generated: cards=%s page=%s", + result.card_count, + result.page_path, + ) + return 0 + if arguments and arguments[0] == "precision-boundary-review": + args = _precision_boundary_review_parser().parse_args(arguments[1:]) + result = generate_precision_boundary_review( + args.sample_dir, + args.precision_review_json, + args.out_dir, + patch_radius_px=args.patch_radius, + ) + logging.getLogger(__name__).info( + "Precision boundary review generated: cards=%s proposals=%s " + "manual_only=%s page=%s automatic_review=%s", + result.card_count, + result.proposal_count, + result.manual_only_count, + result.page_path, + result.automatic_review_path, + ) + return 0 + if arguments and arguments[0] == "precision-boundary-finalize": + args = _precision_boundary_finalize_parser().parse_args(arguments[1:]) + result = finalize_precision_boundary_review( + args.sample_dir, + args.precision_review_json, + args.boundary_review_json, + args.out_dir, + ) + logging.getLogger(__name__).info( + "Precision boundary review finalized: resolved=%s unresolved_manual=%s " + "excluded=%s manifest=%s", + result.resolved_count, + result.unresolved_manual_count, + result.excluded_count, + result.manifest_path, + ) + return 0 + if arguments and arguments[0] == "precision-manual-boundary-review": + args = _precision_manual_boundary_review_parser().parse_args(arguments[1:]) + result = generate_precision_manual_boundary_review( + args.sample_dir, + args.base_resolved_dir, + args.out_dir, + patch_radius_px=args.patch_radius, + ) + logging.getLogger(__name__).info( + "Precision manual-boundary review generated: cards=%s page=%s", + result.card_count, + result.page_path, + ) + return 0 + if arguments and arguments[0] == "precision-manual-boundary-finalize": + args = _precision_manual_boundary_finalize_parser().parse_args(arguments[1:]) + result = finalize_precision_manual_boundary_review( + args.sample_dir, + args.base_resolved_dir, + args.manual_review_json, + args.out_dir, + ) + logging.getLogger(__name__).info( + "Precision manual boundaries finalized: labels=%s added=%s " + "excluded=%s manifest=%s", + result.resolved_count, + result.manual_added_count, + result.manual_excluded_count, + result.manifest_path, + ) + return 0 + if arguments and arguments[0] == "recall-manual-boundary-review": + args = _recall_manual_boundary_review_parser().parse_args(arguments[1:]) + result = generate_recall_manual_boundary_review( + args.sample_dir, + args.manual_review_json, + args.base_finalize_dir, + args.out_dir, + patch_radius_px=args.patch_radius, + ) + logging.getLogger(__name__).info( + "Recall manual-boundary review generated: cards=%s page=%s", + result.card_count, + result.page_path, + ) + return 0 + if arguments and arguments[0] == "recall-manual-boundary-finalize": + args = _recall_manual_boundary_finalize_parser().parse_args(arguments[1:]) + result = finalize_recall_manual_boundary_review( + args.sample_dir, + args.base_finalize_dir, + args.review_json, + args.out_dir, + ) + logging.getLogger(__name__).info( + "Recall manual boundary finalized: labels=%s added=%s excluded=%s " + "manifest=%s", + result.combined_label_count, + result.manual_added_count, + result.manual_excluded_count, + result.manifest_path, + ) + return 0 + if arguments and arguments[0] == "shape-finalize": + args = _shape_finalize_parser().parse_args(arguments[1:]) + result = finalize_shape_recovery_review( + args.sample_dir, + args.shape_review_json, + args.base_finalize_dir, + args.out_dir, + write_combined_labels=not args.delta_only, + ) + logging.getLogger(__name__).info( + "Shape-recovery review finalized: accepted=%s warnings=%s manifest=%s", + result.accepted_count, + result.boundary_warning_count, + result.manifest_path, + ) + return 0 + if arguments and arguments[0] == "serve": + arguments = arguments[1:] + args = _serve_parser().parse_args(arguments) + if args.generate_only: + bundle = generate_recall_review_bundle( + args.sample_dir, + overlay_dir=args.overlay_dir, + window_radius_px=args.window_radius, + window_stride_px=args.window_stride, + overview_downsample=args.overview_downsample, + ) + logging.getLogger(__name__).info( + "Recall review bundle generated: %s (%s windows)", + bundle.page_path, + bundle.window_count, + ) + return 0 + serve_recall_review( + args.sample_dir, + overlay_dir=args.overlay_dir, + host=args.host, + port=args.port, + generate=not args.no_generate, + window_radius_px=args.window_radius, + window_stride_px=args.window_stride, + overview_downsample=args.overview_downsample, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/run_oocyte_release.py b/src/run_oocyte_release.py new file mode 100644 index 0000000..dc4b8df --- /dev/null +++ b/src/run_oocyte_release.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Build or validate an immutable reviewed oocyte release package.""" + +from __future__ import annotations + +import argparse +import json +import logging +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from aegle.oocyte.release import build_oocyte_release, validate_oocyte_release + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Build or validate reviewed oocyte delivery artifacts." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + build = subparsers.add_parser( + "build", + help="Build a new immutable release from a YAML or JSON specification.", + ) + build.add_argument("--spec", type=Path, required=True) + build.add_argument("--out-dir", type=Path, required=True) + + validate = subparsers.add_parser( + "validate", + help="Verify release checksums and cross-file invariants.", + ) + validate.add_argument("--release-dir", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + ) + args = parse_args() + if args.command == "build": + result = build_oocyte_release(args.spec, args.out_dir) + validation = result.validation + logging.info("Release: %s", result.release_dir) + logging.info("Samples: %d", result.sample_count) + logging.info("Positive oocytes: %d", result.oocyte_count) + else: + validation = validate_oocyte_release(args.release_dir) + print(json.dumps(validation, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/run_oocyte_report.py b/src/run_oocyte_report.py new file mode 100644 index 0000000..d93bbe7 --- /dev/null +++ b/src/run_oocyte_report.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Generate algorithm and per-sample biological review HTML.""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from aegle.oocyte.report import generate_html_reports + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--batch-dir", type=Path, required=True) + parser.add_argument("--rescue-delta-dir", type=Path) + parser.add_argument("--references", type=Path) + parser.add_argument("--patch-radius", type=int, default=180) + parser.add_argument("--no-label-export", action="store_true") + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") + result = generate_html_reports( + args.batch_dir, + rescue_delta_dir=args.rescue_delta_dir, + references_path=args.references, + patch_radius_px=args.patch_radius, + export_combined_labels=not args.no_label_export, + ) + logging.info("Algorithm document: %s", result.algorithm_document) + logging.info("Sample pages: %s", len(result.sample_pages)) + logging.info("Batch index: %s", result.batch_index) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/run_oocyte_rescue_delta.py b/src/run_oocyte_rescue_delta.py new file mode 100644 index 0000000..135f6c5 --- /dev/null +++ b/src/run_oocyte_rescue_delta.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Generate a reviewable secondary-rescue delta from completed v6 outputs.""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from aegle.oocyte.delta import generate_rescue_delta_batch +from aegle.oocyte.review import generate_review_pack + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run only the crowded-field rescue pass on completed v6 samples." + ) + parser.add_argument("--baseline-dir", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument( + "--profile", + default="donor13_v6_rescue_v1", + ) + parser.add_argument( + "--sample", + action="append", + dest="samples", + help="Sample ID to process; repeat the option or omit it for all samples.", + ) + parser.add_argument("--montage-columns", type=int, default=3) + parser.add_argument("--montage-rows", type=int, default=4) + return parser.parse_args() + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + ) + args = parse_args() + result = generate_rescue_delta_batch( + args.baseline_dir, + out_dir=args.out_dir, + profile_name=args.profile, + sample_ids=args.samples, + ) + review = generate_review_pack( + result.out_dir, + columns=args.montage_columns, + rows=args.montage_rows, + neighbor_overlays=True, + ) + logging.info("Rescue candidates: %s", review.accepted_count) + logging.info("Delta output: %s", result.out_dir) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/oocyte/__init__.py b/tests/oocyte/__init__.py new file mode 100644 index 0000000..b4b3f30 --- /dev/null +++ b/tests/oocyte/__init__.py @@ -0,0 +1 @@ +"""Tests for the standalone oocyte detector.""" diff --git a/tests/oocyte/test_batch.py b/tests/oocyte/test_batch.py new file mode 100644 index 0000000..2118117 --- /dev/null +++ b/tests/oocyte/test_batch.py @@ -0,0 +1,107 @@ +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import pandas as pd +import tifffile + +from aegle.oocyte import detect_oocyte_batch, load_sample_manifest + + +class TestOocyteBatch(unittest.TestCase): + def _write_ome(self, path): + yy, xx = np.mgrid[:256, :256] + uchl1 = np.full((256, 256), 100.0, dtype=np.float32) + uchl1 += ( + 12000.0 + * np.exp(-((yy - 128) ** 2 + (xx - 128) ** 2) / (2 * 22**2)) + ).astype(np.float32) + channels = np.stack([uchl1, np.zeros_like(uchl1)]) + tifffile.imwrite(path, channels, ome=True, metadata={"axes": "CYX"}) + + def test_isolates_sample_failures_and_records_disabled_rows(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + image_path = root / "valid.ome.tiff" + self._write_ome(image_path) + manifest = root / "samples.csv" + pd.DataFrame( + [ + { + "sample_id": "valid", + "image_path": image_path, + "channel_index": 0, + "pixel_size_um": 0.5, + "enabled": True, + }, + { + "sample_id": "missing", + "image_path": root / "missing.ome.tiff", + "channel_index": 0, + "pixel_size_um": 0.5, + "enabled": True, + }, + { + "sample_id": "disabled", + "image_path": root / "unused.ome.tiff", + "channel_index": None, + "pixel_size_um": 0.5, + "enabled": False, + }, + ] + ).to_csv(manifest, index=False) + + result = detect_oocyte_batch( + manifest, + out_dir=root / "output", + jobs=1, + continue_on_error=True, + ) + + statuses = dict(zip(result.summary["sample_id"], result.summary["status"])) + self.assertEqual( + statuses, + {"valid": "complete", "missing": "failed", "disabled": "skipped"}, + ) + self.assertEqual(result.failed_count, 1) + self.assertTrue(result.artifact_paths["batch_summary_csv"].is_file()) + self.assertTrue(result.artifact_paths["spatial_qc_atlas"].is_file()) + self.assertTrue(result.artifact_paths["spatial_qc_index"].is_file()) + self.assertTrue((root / "output/valid/oocyte_labels.ome.tiff").is_file()) + + resumed = detect_oocyte_batch( + manifest, + out_dir=root / "output", + jobs=1, + continue_on_error=True, + ) + valid = resumed.summary[resumed.summary["sample_id"] == "valid"].iloc[0] + self.assertTrue(bool(valid["resumed"])) + + def test_rejects_duplicate_sample_ids(self): + with tempfile.TemporaryDirectory() as tmp: + manifest = Path(tmp) / "samples.csv" + pd.DataFrame( + [ + { + "sample_id": "duplicate", + "image_path": "first.ome.tiff", + "channel_index": 0, + "pixel_size_um": 0.5, + }, + { + "sample_id": "duplicate", + "image_path": "second.ome.tiff", + "channel_index": 0, + "pixel_size_um": 0.5, + }, + ] + ).to_csv(manifest, index=False) + + with self.assertRaisesRegex(ValueError, "duplicate sample IDs"): + load_sample_manifest(manifest) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_config.py b/tests/oocyte/test_config.py new file mode 100644 index 0000000..3c45cee --- /dev/null +++ b/tests/oocyte/test_config.py @@ -0,0 +1,93 @@ +import json +import unittest +from dataclasses import FrozenInstanceError +from pathlib import Path + +from aegle.oocyte.config import ( + DONOR13_V6, + DONOR13_V6_RESCUE_V1, + ExperimentalConfig, + OocyteDetectionConfig, + available_profiles, + get_profile, +) + + +BASELINE_PATH = Path(__file__).parent / "data" / "donor13_v6_baseline.json" + + +class TestDonor13V6Config(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.baseline = json.loads(BASELINE_PATH.read_text()) + + def test_profile_is_registered_and_immutable(self): + self.assertEqual( + available_profiles(), + ("donor13_v6", "donor13_v6_rescue_v1"), + ) + self.assertIs(get_profile("donor13_v6"), DONOR13_V6) + self.assertIs( + get_profile("donor13_v6_rescue_v1"), + DONOR13_V6_RESCUE_V1, + ) + with self.assertRaises(FrozenInstanceError): + DONOR13_V6.pixel_size_um = 1.0 + + def test_profile_fingerprint_matches_frozen_baseline(self): + self.assertEqual( + DONOR13_V6.fingerprint(), + self.baseline["profile_fingerprint"], + ) + + def test_v6_enables_validated_seed_families_without_border_rescue(self): + contract = self.baseline["algorithm_contract"] + self.assertEqual(DONOR13_V6.local.max_broad_peak_seeds_per_candidate, 3) + self.assertEqual(DONOR13_V6.local.max_offset_ring_seeds, 8) + self.assertEqual(DONOR13_V6.local.max_centroid_reseed_iterations, 1) + self.assertLess( + DONOR13_V6.local.compact_window_radius_px, + DONOR13_V6.local.window_radius_px, + ) + self.assertFalse(DONOR13_V6.experimental.border_rescue_enabled) + self.assertFalse(contract["border_rescue_enabled"]) + + def test_baseline_totals_equal_sum_of_samples(self): + samples = self.baseline["samples"].values() + totals = self.baseline["expected_totals"] + count_fields = ( + "reference_count", + "coarse_candidate_count", + "refined_candidate_count", + "accepted_candidate_count", + "reference_recalled_by_refined", + "reference_recalled_by_accepted", + "accepted_candidates_near_reference", + "accepted_candidates_novel", + ) + self.assertEqual(len(self.baseline["samples"]), totals["sample_count"]) + for field in count_fields: + with self.subTest(field=field): + self.assertEqual(sum(sample[field] for sample in samples), totals[field]) + + def test_representative_case_matches_acceptance_tolerances(self): + metrics = self.baseline["representative_case"]["metrics"] + self.assertGreaterEqual(metrics["equivalent_diameter_um"], 35.0) + self.assertLessEqual(metrics["equivalent_diameter_um"], 39.0) + self.assertGreaterEqual(metrics["circularity"], 0.82) + self.assertGreaterEqual(metrics["solidity"], 0.94) + self.assertLessEqual(metrics["centroid_offset_px"], 12.0) + + def test_unknown_profile_is_rejected(self): + with self.assertRaisesRegex(ValueError, "unknown oocyte detector profile"): + get_profile("donor13_v7") + + def test_v6_rejects_experimental_border_rescue(self): + with self.assertRaisesRegex(ValueError, "does not support experimental border rescue"): + OocyteDetectionConfig( + experimental=ExperimentalConfig(border_rescue_enabled=True) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_detection.py b/tests/oocyte/test_detection.py new file mode 100644 index 0000000..7faa4c9 --- /dev/null +++ b/tests/oocyte/test_detection.py @@ -0,0 +1,117 @@ +import unittest + +import numpy as np +import pandas as pd + +from aegle.oocyte import ( + DONOR13_V6, + detect_coarse_candidates, + refine_candidates_from_array, +) +from aegle.oocyte.detection import ( + _deduplicate_coarse_rows, + build_downsampled_mean_map_from_array, +) + + +class TestCoarseDetection(unittest.TestCase): + def test_strip_downsampling_computes_block_means(self): + image = np.arange(16 * 16, dtype=np.float32).reshape(16, 16) + result = build_downsampled_mean_map_from_array(image, DONOR13_V6.coarse) + expected = image.reshape(2, 8, 2, 8).mean(axis=(1, 3)) + np.testing.assert_array_equal(result, expected) + + def test_coarse_row_dedup_prefers_peak_at_equal_intensity(self): + common = { + "coarse_max_ds": 100.0, + "coarse_area_ds": 30, + "coarse_center_x": 100, + "coarse_center_y": 100, + } + rows = [ + {**common, "coarse_seed_kind": "global_peak"}, + {**common, "coarse_seed_kind": "peak"}, + ] + result = _deduplicate_coarse_rows(rows, merge_distance_px=60.0) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["coarse_seed_kind"], "peak") + + def test_detects_bright_synthetic_regions(self): + yy, xx = np.ogrid[:256, :256] + image = np.full((256, 256), 100.0, dtype=np.float32) + image[(yy - 70) ** 2 + (xx - 80) ** 2 <= 7**2] = 8000.0 + image[(yy - 180) ** 2 + (xx - 170) ** 2 <= 8**2] = 12000.0 + + result = detect_coarse_candidates(image, DONOR13_V6) + + self.assertGreaterEqual(len(result.candidates), 2) + centers = result.candidates[["coarse_center_x", "coarse_center_y"]].to_numpy() + for expected_x, expected_y in ((80 * 8 + 4, 70 * 8 + 4), (170 * 8 + 4, 180 * 8 + 4)): + distances = np.hypot( + centers[:, 0] - expected_x, + centers[:, 1] - expected_y, + ) + self.assertLessEqual(float(distances.min()), 8.0) + self.assertEqual(result.mask.dtype, np.bool_) + self.assertEqual(result.contrast.shape, image.shape) + + def test_refines_two_instances_from_one_coarse_patch(self): + yy, xx = np.mgrid[:701, :701] + image = np.full((701, 701), 100.0, dtype=np.float32) + image += ( + 10000.0 + * np.exp(-((yy - 350) ** 2 + (xx - 280) ** 2) / (2 * 22**2)) + ).astype(np.float32) + image += ( + 12000.0 + * np.exp(-((yy - 350) ** 2 + (xx - 420) ** 2) / (2 * 22**2)) + ).astype(np.float32) + coarse = pd.DataFrame( + [ + { + "detector_component_id": "det_0000", + "coarse_center_x": 350, + "coarse_center_y": 350, + "coarse_max_ds": 12000.0, + } + ] + ) + + result = refine_candidates_from_array(image, coarse, DONOR13_V6) + + accepted = result.candidates[result.candidates["accepted"]] + self.assertEqual(len(accepted), 2) + self.assertEqual(len(result.candidate_masks), 2) + centers = accepted[["center_x", "center_y"]].to_numpy() + for expected_x in (280, 420): + distances = np.hypot(centers[:, 0] - expected_x, centers[:, 1] - 350) + self.assertLessEqual(float(distances.min()), 2.0) + for candidate_mask in result.candidate_masks.values(): + self.assertGreater(int(candidate_mask.mask.sum()), 2000) + + def test_rejects_tiny_bright_punctum_during_refinement(self): + yy, xx = np.mgrid[:401, :401] + image = np.full((401, 401), 100.0, dtype=np.float32) + image += ( + 12000.0 + * np.exp(-((yy - 200) ** 2 + (xx - 200) ** 2) / (2 * 2**2)) + ).astype(np.float32) + coarse = pd.DataFrame( + [ + { + "detector_component_id": "det_0000", + "coarse_center_x": 200, + "coarse_center_y": 200, + "coarse_max_ds": 12000.0, + } + ] + ) + + result = refine_candidates_from_array(image, coarse, DONOR13_V6) + + self.assertTrue(result.candidates.empty) + self.assertEqual(result.candidate_masks, {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_end_to_end.py b/tests/oocyte/test_end_to_end.py new file mode 100644 index 0000000..7f15d58 --- /dev/null +++ b/tests/oocyte/test_end_to_end.py @@ -0,0 +1,75 @@ +import json +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import tifffile + +from aegle.oocyte import DONOR13_V6, detect_oocytes, load_candidate_mask + + +class TestOocyteEndToEnd(unittest.TestCase): + def _write_ome(self, path, uchl1): + channels = np.stack([uchl1, np.zeros_like(uchl1)]) + tifffile.imwrite(path, channels, ome=True, metadata={"axes": "CYX"}) + + def test_writes_standalone_single_sample_deliverable(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + yy, xx = np.mgrid[:512, :512] + uchl1 = np.full((512, 512), 100.0, dtype=np.float32) + uchl1 += ( + 12000.0 + * np.exp(-((yy - 256) ** 2 + (xx - 256) ** 2) / (2 * 22**2)) + ).astype(np.float32) + image_path = root / "synthetic.ome.tiff" + self._write_ome(image_path, uchl1) + + result = detect_oocytes( + image_path, + sample_id="synthetic", + out_dir=root / "output", + config=DONOR13_V6, + channel_index=0, + ) + + self.assertEqual(len(result.coarse_candidates), 1) + self.assertEqual(len(result.candidates), 1) + self.assertTrue(bool(result.candidates.iloc[0]["accepted"])) + mask_path = root / "output" / result.candidates.iloc[0]["mask_path"] + persisted = load_candidate_mask(mask_path) + self.assertGreater(int(persisted.mask.sum()), 2000) + labels = tifffile.imread(result.artifact_paths["labels"]) + self.assertEqual(labels.shape, (512, 512)) + self.assertEqual(int(labels.max()), 1) + summary = json.loads(result.artifact_paths["summary"].read_text()) + self.assertEqual(summary["accepted_candidate_count"], 1) + self.assertTrue(result.artifact_paths["overview"].is_file()) + self.assertTrue(result.artifact_paths["duplicate_suspects"].is_file()) + + def test_blank_channel_writes_zero_label_image(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + image_path = root / "blank.ome.tiff" + self._write_ome( + image_path, + np.full((256, 256), 100.0, dtype=np.float32), + ) + + result = detect_oocytes( + image_path, + sample_id="blank", + out_dir=root / "output", + config=DONOR13_V6, + channel_index=0, + ) + + self.assertTrue(result.candidates.empty) + self.assertIn("accepted", result.candidates.columns) + labels = tifffile.imread(result.artifact_paths["labels"]) + self.assertFalse(labels.any()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_export.py b/tests/oocyte/test_export.py new file mode 100644 index 0000000..881ff18 --- /dev/null +++ b/tests/oocyte/test_export.py @@ -0,0 +1,102 @@ +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import pandas as pd +import tifffile + +from aegle.oocyte import DONOR13_V6 +from aegle.oocyte.export import export_whole_slide_labels +from aegle.oocyte.io import save_candidate_mask +from aegle.oocyte.models import BoundingBox, SegmentationMetrics + + +def _metrics(area_px): + return SegmentationMetrics( + threshold_method="triangle", + base_threshold=10.0, + annulus_floor=5.0, + threshold=10.0, + selection_mode="center_component", + area_px=area_px, + equivalent_diameter_um=20.0, + major_axis_um=20.0, + minor_axis_um=20.0, + eccentricity=0.0, + solidity=1.0, + circularity=1.0, + centroid_y_px=4.0, + centroid_x_px=4.0, + centroid_offset_px=0.0, + mean_intensity=1000.0, + max_intensity=1200.0, + ) + + +class TestWholeSlideLabelExport(unittest.TestCase): + def test_composes_persisted_masks_with_score_ordered_overlap(self): + with tempfile.TemporaryDirectory() as tmp: + sample_dir = Path(tmp) + masks_dir = sample_dir / "masks" + first = np.ones((8, 8), dtype=np.bool_) + second = np.ones((8, 8), dtype=np.bool_) + for candidate_id, mask, bbox in ( + ("det_0000", first, BoundingBox(10, 10, 18, 18)), + ("det_0001", second, BoundingBox(14, 14, 22, 22)), + ): + save_candidate_mask( + masks_dir / f"{candidate_id}.npz", + mask=mask, + bbox=bbox, + image_shape_yx=(64, 64), + sample_id="test", + candidate_id=candidate_id, + profile_name=DONOR13_V6.profile_name, + profile_fingerprint=DONOR13_V6.fingerprint(), + metrics=_metrics(int(mask.sum())), + ) + candidates = pd.DataFrame( + [ + { + "detector_component_id": "det_0001", + "accepted": True, + "detector_score": 0.7, + "acceptance_mode": "strict", + "center_x": 18, + "center_y": 18, + "mask_path": "masks/det_0001.npz", + }, + { + "detector_component_id": "det_0000", + "accepted": True, + "detector_score": 0.9, + "acceptance_mode": "strict", + "center_x": 14, + "center_y": 14, + "mask_path": "masks/det_0000.npz", + }, + ] + ) + + result = export_whole_slide_labels( + candidates, + sample_dir=sample_dir, + image_shape_yx=(64, 64), + image_path=sample_dir / "oocyte_labels.ome.tiff", + mapping_path=sample_dir / "oocyte_labels.csv", + tile_shape_yx=(16, 16), + ) + + labels = tifffile.imread(result.image_path) + mapping = pd.read_csv(result.mapping_path) + self.assertEqual(result.label_count, 2) + self.assertEqual(int((labels == 1).sum()), 64) + self.assertEqual(int((labels == 2).sum()), 48) + self.assertEqual(result.overlap_pixel_count, 16) + self.assertEqual(mapping.iloc[0]["detector_component_id"], "det_0000") + self.assertEqual(mapping.iloc[1]["overlap_pixel_count"], 16) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_io.py b/tests/oocyte/test_io.py new file mode 100644 index 0000000..5f655f3 --- /dev/null +++ b/tests/oocyte/test_io.py @@ -0,0 +1,80 @@ +import tempfile +import unittest +from pathlib import Path + +import numpy as np + +from aegle.oocyte import DONOR13_V6, find_channel_index +from aegle.oocyte.io import ( + extract_padded_patch, + load_candidate_mask, + save_candidate_mask, +) +from aegle.oocyte.models import BoundingBox +from aegle.oocyte.segmentation import segment_oocyte_patch + +from tests.oocyte.test_segmentation import circular_patch + + +class TestOocyteIO(unittest.TestCase): + def test_finds_channel_index_from_explicit_channel_id(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "antibodies.tsv" + path.write_text( + "version\tchannel_id\tantibody_name\n" + "2\tChannel:0:0\tDAPI\n" + "2\tChannel:0:29\tUCHL1\n" + ) + self.assertEqual(find_channel_index(path), 29) + + def test_extracts_and_unpads_edge_patch(self): + image = np.arange(80, dtype=np.uint16).reshape(8, 10) + extracted = extract_padded_patch(image, center_xy=(0, 0), radius=2) + + self.assertEqual(extracted.image.shape, (5, 5)) + self.assertEqual(extracted.bbox, BoundingBox(0, 0, 3, 3)) + self.assertEqual(extracted.padding_tblr, (2, 0, 2, 0)) + cropped = extracted.crop_to_image_bounds(np.ones((5, 5), dtype=np.bool_)) + self.assertEqual(cropped.shape, (3, 3)) + + def test_extracts_patch_centered_just_outside_image(self): + image = np.arange(80, dtype=np.uint16).reshape(8, 10) + extracted = extract_padded_patch(image, center_xy=(-1, 3), radius=2) + + self.assertEqual(extracted.image.shape, (5, 5)) + self.assertEqual(extracted.bbox, BoundingBox(0, 1, 2, 6)) + self.assertEqual(extracted.padding_tblr, (0, 0, 3, 0)) + cropped = extracted.crop_to_image_bounds(np.ones((5, 5), dtype=np.bool_)) + self.assertEqual(cropped.shape, (5, 2)) + + def test_candidate_mask_round_trip(self): + result = segment_oocyte_patch(circular_patch(), DONOR13_V6) + bbox = BoundingBox(100, 200, 461, 561) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "sample_candidate.npz" + save_candidate_mask( + path, + mask=result.mask, + bbox=bbox, + image_shape_yx=(1000, 1000), + sample_id="13-23", + candidate_id="13-23_det_0001", + profile_name=DONOR13_V6.profile_name, + profile_fingerprint=DONOR13_V6.fingerprint(), + metrics=result.metrics, + ) + loaded = load_candidate_mask(path) + + np.testing.assert_array_equal(loaded.mask, result.mask) + self.assertEqual(loaded.bbox, bbox) + self.assertEqual(loaded.image_shape_yx, (1000, 1000)) + self.assertEqual(loaded.metadata["sample_id"], "13-23") + self.assertEqual(loaded.metadata["candidate_id"], "13-23_det_0001") + self.assertEqual( + loaded.metadata["profile_fingerprint"], + DONOR13_V6.fingerprint(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_local_regression.py b/tests/oocyte/test_local_regression.py new file mode 100644 index 0000000..04c87fc --- /dev/null +++ b/tests/oocyte/test_local_regression.py @@ -0,0 +1,94 @@ +import os +import unittest +from pathlib import Path + +import numpy as np +import pandas as pd + +from aegle.oocyte import ( + DONOR13_V6, + read_ome_channel_patch, + scan_coarse_candidates, + scan_refined_candidates, + segment_oocyte_patch, +) + + +WORKSPACE_ROOT = Path(__file__).resolve().parents[3] +RAW_IMAGE = ( + WORKSPACE_ROOT + / "data/Ovary/D11_13/Scan1/processed_hubmap" + / "D11_13_Scan1.er_manual_13-23_Ovary_Central_Superior_Antimesenteric_Scan1.ome.tiff" +) +EXPECTED_MASK = ( + WORKSPACE_ROOT + / "frs-atlas-phenocycler/output/ovary_panel1/v2_4/oocyte_segmentation_poc" + / "13-23_oocyte_680.mask.npz" +) +EXPECTED_COARSE = ( + WORKSPACE_ROOT + / "frs-atlas-phenocycler/output/ovary_panel1/v2_4" + / "oocyte_raw_detector_peak_seed_v6_experimental/13-23" + / "13-23_coarse_candidates.csv" +) +EXPECTED_REFINED = EXPECTED_COARSE.with_name("13-23_refined_candidates.csv") +RUN_LOCAL = os.environ.get("AEGLE_RUN_OOCYTE_LOCAL_REGRESSION") == "1" + + +@unittest.skipUnless( + RUN_LOCAL and RAW_IMAGE.exists() and EXPECTED_MASK.exists(), + "set AEGLE_RUN_OOCYTE_LOCAL_REGRESSION=1 with local donor13 data", +) +class TestDonor13LocalRegression(unittest.TestCase): + def test_oocyte_680_matches_research_mask(self): + patch = read_ome_channel_patch( + RAW_IMAGE, + channel_index=27, + center_xy=(11701, 20077), + radius=DONOR13_V6.local.window_radius_px, + ) + result = segment_oocyte_patch(patch.image, DONOR13_V6) + with np.load(EXPECTED_MASK, allow_pickle=False) as archive: + expected_mask = np.asarray(archive["oocyte_mask"], dtype=np.bool_) + + np.testing.assert_array_equal(result.mask, expected_mask) + self.assertAlmostEqual(result.metrics.equivalent_diameter_um, 36.91886960131623) + self.assertAlmostEqual(result.metrics.circularity, 0.8751328034565493) + self.assertAlmostEqual(result.metrics.solidity, 0.9758432087511395) + self.assertLessEqual(result.metrics.centroid_offset_px, 12.0) + + def test_13_23_coarse_proposals_match_research_output(self): + result = scan_coarse_candidates(RAW_IMAGE, 27, DONOR13_V6) + expected = pd.read_csv(EXPECTED_COARSE) + + pd.testing.assert_frame_equal( + result.candidates, + expected, + check_dtype=False, + check_exact=False, + rtol=1e-12, + atol=1e-12, + ) + self.assertEqual(result.image_shape_yx, (26849, 17783)) + + def test_13_23_refinement_matches_research_output(self): + coarse = pd.read_csv(EXPECTED_COARSE) + result = scan_refined_candidates(RAW_IMAGE, 27, coarse, DONOR13_V6) + expected = pd.read_csv(EXPECTED_REFINED) + expected = expected[result.candidates.columns] + + pd.testing.assert_frame_equal( + result.candidates, + expected, + check_dtype=False, + check_exact=False, + rtol=1e-12, + atol=1e-12, + ) + self.assertEqual(len(result.candidates), 318) + self.assertEqual(int(result.candidates["accepted"].sum()), 135) + self.assertEqual(len(result.candidate_masks), len(result.candidates)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_manual_seed_finalize.py b/tests/oocyte/test_manual_seed_finalize.py new file mode 100644 index 0000000..86d8fd2 --- /dev/null +++ b/tests/oocyte/test_manual_seed_finalize.py @@ -0,0 +1,179 @@ +import json +import re +import tempfile +import unittest +from pathlib import Path + +import pandas as pd +import tifffile + +from aegle.oocyte.io import load_candidate_mask +from aegle.oocyte.manual_seed_finalize import finalize_manual_seed_review +from aegle.oocyte.recall_review import analyze_recall_review +from tests.oocyte.test_recall_review import RecallReviewFixture + + +def _manual_review_payload(page_path: Path) -> dict: + match = re.search( + r'', + page_path.read_text(), + flags=re.DOTALL, + ) + if match is None: + raise AssertionError("manual-seed page is missing its JSON payload") + embedded = json.loads(match.group(1)) + return { + "schema_version": 1, + "review_type": "manual_seed_mask_review", + "identity": embedded["identity"], + "exported_at": "2026-07-11T12:00:00.000Z", + "rows": embedded["rows"], + } + + +def _prepare_review( + root: Path, + *, + click_xy: tuple[int, int] = (160, 160), +) -> tuple[RecallReviewFixture, Path, dict]: + fixture = RecallReviewFixture(root) + bundle = fixture.generate() + metadata = json.loads(bundle.metadata_path.read_text()) + recall_review_path = root / "recall-review.json" + recall_review_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall", + "sample": metadata["review_identity"], + "windows": [ + { + "window_id": metadata["windows"][0]["window_id"], + "status": "has_misses", + } + ], + "missing_oocytes": [ + { + "annotation_id": "manual-1", + "window_id": metadata["windows"][0]["window_id"], + "x": click_xy[0], + "y": click_xy[1], + "notes": "synthetic missed circle", + } + ], + } + ) + ) + analysis_dir = root / "analysis" + analyze_recall_review(fixture.sample_dir, recall_review_path, analysis_dir) + payload = _manual_review_payload(analysis_dir / "manual_seed_review.html") + return fixture, analysis_dir, payload + + +class TestManualSeedFinalize(unittest.TestCase): + def test_writes_reviewed_delta_and_versioned_combined_labels(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, analysis_dir, payload = _prepare_review(root) + payload["rows"][0]["manual_mask_choice"] = "accept_manual_expanded" + payload["rows"][0]["manual_notes"] = ( + "The expanded mask still misses part of the cell boundary" + ) + review_path = root / "manual-review.json" + review_path.write_text(json.dumps(payload)) + candidates_before = (fixture.sample_dir / "html_candidates.csv").read_bytes() + analysis_before = (analysis_dir / "recall_failure_analysis.csv").read_bytes() + + result = finalize_manual_seed_review( + fixture.sample_dir, + review_path, + root / "finalized", + analysis_dir=analysis_dir, + tile_shape_yx=(16, 16), + ) + + self.assertEqual(result.accepted_count, 1) + self.assertEqual(result.boundary_warning_count, 1) + self.assertEqual(result.delta_labels.label_count, 1) + self.assertIsNotNone(result.combined_labels) + self.assertEqual(result.combined_labels.label_count, 2) + self.assertEqual(tifffile.imread(result.delta_labels.image_path).shape, (768, 768)) + self.assertEqual( + set(tifffile.imread(result.combined_labels.image_path).ravel()), + {0, 1, 2}, + ) + decisions = pd.read_csv(result.decisions_path) + candidates = pd.read_csv(result.candidates_path) + self.assertTrue(bool(decisions.loc[0, "boundary_warning"])) + self.assertEqual(candidates.loc[0, "display_id"], "#R001") + mask_path = result.out_dir / candidates.loc[0, "mask_path"] + reviewed = load_candidate_mask(mask_path) + self.assertTrue(reviewed.metadata["reviewed_manual_seed"]) + self.assertFalse(reviewed.metadata["provisional_only"]) + manifest = json.loads(result.manifest_path.read_text()) + self.assertEqual(manifest["accepted_manual_mask_count"], 1) + self.assertEqual(manifest["combined_label_count"], 2) + self.assertFalse(manifest["production_outputs_modified"]) + self.assertEqual( + candidates_before, + (fixture.sample_dir / "html_candidates.csv").read_bytes(), + ) + self.assertEqual( + analysis_before, + (analysis_dir / "recall_failure_analysis.csv").read_bytes(), + ) + + def test_rejects_stale_analysis_identity_and_missing_choice(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, analysis_dir, payload = _prepare_review(root) + payload["rows"][0]["manual_mask_choice"] = "accept_manual_expanded" + + stale = json.loads(json.dumps(payload)) + stale["identity"]["analysis_sha256"] = "0" * 64 + stale_path = root / "stale.json" + stale_path.write_text(json.dumps(stale)) + with self.assertRaisesRegex(ValueError, "SHA-256"): + finalize_manual_seed_review( + fixture.sample_dir, + stale_path, + root / "stale-out", + analysis_dir=analysis_dir, + tile_shape_yx=(16, 16), + ) + + missing = json.loads(json.dumps(payload)) + missing["rows"][0]["manual_mask_choice"] = "" + missing_path = root / "missing.json" + missing_path.write_text(json.dumps(missing)) + with self.assertRaisesRegex(ValueError, "invalid or missing"): + finalize_manual_seed_review( + fixture.sample_dir, + missing_path, + root / "missing-out", + analysis_dir=analysis_dir, + tile_shape_yx=(16, 16), + ) + + def test_rejects_reviewed_mask_that_duplicates_production(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, analysis_dir, payload = _prepare_review( + root, + click_xy=(380, 380), + ) + payload["rows"][0]["manual_mask_choice"] = "accept_manual_expanded" + review_path = root / "duplicate-review.json" + review_path.write_text(json.dumps(payload)) + + with self.assertRaisesRegex(ValueError, "blocking overlap"): + finalize_manual_seed_review( + fixture.sample_dir, + review_path, + root / "duplicate-out", + analysis_dir=analysis_dir, + tile_shape_yx=(16, 16), + ) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_precision_boundary_finalize.py b/tests/oocyte/test_precision_boundary_finalize.py new file mode 100644 index 0000000..a4c1070 --- /dev/null +++ b/tests/oocyte/test_precision_boundary_finalize.py @@ -0,0 +1,241 @@ +import json +import re +import tempfile +import unittest +from pathlib import Path + +import pandas as pd +import tifffile + +from aegle.oocyte.io import load_candidate_mask +from aegle.oocyte.precision_boundary_finalize import ( + finalize_precision_boundary_review, +) +from aegle.oocyte.precision_boundary_review import ( + generate_precision_boundary_review, +) +from tests.oocyte.test_precision_boundary_review import ( + _precision_review, + _replace_with_fragment_case, +) +from tests.oocyte.test_recall_review import RecallReviewFixture + + +def _boundary_review_payload(page_path: Path, *, choice: str) -> dict: + match = re.search( + r'', + page_path.read_text(), + flags=re.DOTALL, + ) + if match is None: + raise AssertionError("boundary-review page is missing its JSON payload") + embedded = json.loads(match.group(1)) + rows = embedded["rows"] + for row in rows: + row["boundary_review_choice"] = choice + row["boundary_review_notes"] = "synthetic decision" + return { + "schema_version": 1, + "review_type": "oocyte_precision_boundary_review", + "identity": embedded["identity"], + "exported_at": "2026-07-12T13:00:00Z", + "rows": rows, + } + + +class TestPrecisionBoundaryFinalize(unittest.TestCase): + def test_finalizes_selected_proposal_without_modifying_source_assets(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture = RecallReviewFixture(root) + _replace_with_fragment_case(fixture) + bundle = fixture.generate() + identity = json.loads(bundle.metadata_path.read_text())["review_identity"] + precision_path = _precision_review(fixture, identity) + pack = generate_precision_boundary_review( + fixture.sample_dir, + precision_path, + root / "boundary-pack", + ) + payload = _boundary_review_payload( + pack.page_path, + choice="use_expanded", + ) + boundary_path = root / "boundary-review.json" + boundary_path.write_text(json.dumps(payload)) + current_before = (fixture.sample_dir / "masks/accepted.npz").read_bytes() + proposal_path = Path(payload["rows"][0]["expanded_mask_path"]) + proposal_before = proposal_path.read_bytes() + + result = finalize_precision_boundary_review( + fixture.sample_dir, + precision_path, + boundary_path, + root / "precision-resolved", + tile_shape_yx=(16, 16), + ) + + self.assertEqual(result.resolved_count, 1) + self.assertEqual(result.unresolved_manual_count, 0) + self.assertEqual(result.excluded_count, 0) + self.assertEqual(set(tifffile.imread(result.labels.image_path).ravel()), {0, 1}) + candidates = pd.read_csv(result.candidates_path) + self.assertEqual( + candidates.loc[0, "precision_resolution_source"], + "precision_boundary_expanded", + ) + reviewed = load_candidate_mask( + result.out_dir / candidates.loc[0, "mask_path"] + ) + self.assertTrue(reviewed.metadata["precision_resolved"]) + self.assertFalse(reviewed.metadata["provisional_only"]) + self.assertEqual( + reviewed.metadata["precision_boundary_choice"], + "use_expanded", + ) + manifest = json.loads(result.manifest_path.read_text()) + self.assertFalse(manifest["release_ready"]) + self.assertTrue(manifest["precision_complete"]) + self.assertTrue(manifest["manual_boundary_complete"]) + self.assertFalse(manifest["recall_complete"]) + self.assertEqual(manifest["resolved_label_count"], 1) + self.assertEqual(manifest["label_export"]["overlap_pixel_count"], 0) + self.assertEqual( + current_before, + (fixture.sample_dir / "masks/accepted.npz").read_bytes(), + ) + self.assertEqual(proposal_before, proposal_path.read_bytes()) + with self.assertRaisesRegex(FileExistsError, "immutable"): + finalize_precision_boundary_review( + fixture.sample_dir, + precision_path, + boundary_path, + root / "precision-resolved", + tile_shape_yx=(16, 16), + ) + + def test_preserves_needs_manual_as_unresolved_without_a_label(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture = RecallReviewFixture(root) + bundle = fixture.generate() + identity = json.loads(bundle.metadata_path.read_text())["review_identity"] + precision_path = _precision_review(fixture, identity) + pack = generate_precision_boundary_review( + fixture.sample_dir, + precision_path, + root / "boundary-pack", + ) + payload = _boundary_review_payload( + pack.page_path, + choice="needs_manual", + ) + boundary_path = root / "boundary-review.json" + boundary_path.write_text(json.dumps(payload)) + + result = finalize_precision_boundary_review( + fixture.sample_dir, + precision_path, + boundary_path, + root / "precision-resolved", + tile_shape_yx=(16, 16), + ) + + self.assertEqual(result.resolved_count, 0) + self.assertEqual(result.unresolved_manual_count, 1) + self.assertEqual(result.excluded_count, 0) + self.assertEqual(set(tifffile.imread(result.labels.image_path).ravel()), {0}) + queue = pd.read_csv(result.manual_queue_path) + self.assertEqual(queue.loc[0, "review_key"], "baseline_v6:accepted-1") + manifest = json.loads(result.manifest_path.read_text()) + self.assertFalse(manifest["manual_boundary_complete"]) + self.assertEqual( + manifest["unresolved_review_keys"], + ["baseline_v6:accepted-1"], + ) + + def test_carries_forward_a_direct_precision_accept(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture = RecallReviewFixture(root) + bundle = fixture.generate() + identity = json.loads(bundle.metadata_path.read_text())["review_identity"] + precision_path = _precision_review(fixture, identity) + precision = json.loads(precision_path.read_text()) + precision["rows"][0]["manual_status"] = "accept" + precision["rows"][0]["manual_notes"] = "" + precision_path.write_text(json.dumps(precision)) + pack = generate_precision_boundary_review( + fixture.sample_dir, + precision_path, + root / "empty-boundary-pack", + ) + self.assertEqual(pack.card_count, 0) + self.assertIsNotNone(pack.automatic_review_path) + boundary_path = pack.automatic_review_path + assert boundary_path is not None + self.assertTrue(boundary_path.is_file()) + self.assertEqual( + list(pd.read_csv(pack.candidates_path).columns), + [ + "boundary_index", + "review_key", + "display_id", + "detector_component_id", + "detection_pass", + "x", + "y", + "current_mask_path", + "conservative_available", + "expanded_available", + ], + ) + + result = finalize_precision_boundary_review( + fixture.sample_dir, + precision_path, + boundary_path, + root / "precision-resolved", + tile_shape_yx=(16, 16), + ) + + candidates = pd.read_csv(result.candidates_path) + self.assertEqual(result.resolved_count, 1) + self.assertEqual( + candidates.loc[0, "precision_resolution_source"], + "precision_accept_current", + ) + + def test_rejects_stale_boundary_candidate_table(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture = RecallReviewFixture(root) + _replace_with_fragment_case(fixture) + bundle = fixture.generate() + identity = json.loads(bundle.metadata_path.read_text())["review_identity"] + precision_path = _precision_review(fixture, identity) + pack = generate_precision_boundary_review( + fixture.sample_dir, + precision_path, + root / "boundary-pack", + ) + payload = _boundary_review_payload( + pack.page_path, + choice="use_expanded", + ) + boundary_path = root / "boundary-review.json" + boundary_path.write_text(json.dumps(payload)) + pack.candidates_path.write_text(pack.candidates_path.read_text() + "\n") + + with self.assertRaisesRegex(ValueError, "candidate table SHA-256"): + finalize_precision_boundary_review( + fixture.sample_dir, + precision_path, + boundary_path, + root / "precision-resolved", + tile_shape_yx=(16, 16), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_precision_boundary_review.py b/tests/oocyte/test_precision_boundary_review.py new file mode 100644 index 0000000..75a29bd --- /dev/null +++ b/tests/oocyte/test_precision_boundary_review.py @@ -0,0 +1,179 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import numpy as np +import pandas as pd +import tifffile + +from aegle.oocyte import DONOR13_V6 +from aegle.oocyte.io import save_candidate_mask +from aegle.oocyte.models import BoundingBox, SegmentationMetrics +from aegle.oocyte.precision_boundary_review import ( + BOUNDARY_RECOVERY_PARAMETERS, + generate_precision_boundary_review, +) +from tests.oocyte.test_recall_review import RecallReviewFixture + + +def _precision_review(fixture: RecallReviewFixture, identity: dict) -> Path: + row = pd.read_csv(fixture.sample_dir / "html_candidates.csv").iloc[0] + path = fixture.root / "precision-review.json" + path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_precision_review", + "identity": identity, + "exported_at": "2026-07-12T12:00:00Z", + "rows": [ + { + "review_key": "baseline_v6:accepted-1", + "display_id": "#001", + "detector_component_id": "accepted-1", + "detection_pass": "baseline_v6", + "center_x": float(row["center_x"]), + "center_y": float(row["center_y"]), + "manual_status": "reject", + "manual_notes": ( + "true_oocyte; mask_truncated; mask_off_target" + ), + } + ], + } + ) + ) + return path + + +def _replace_with_fragment_case(fixture: RecallReviewFixture) -> None: + image = tifffile.imread(fixture.image_path) + yy, xx = np.ogrid[:201, :201] + patch = np.full((201, 201), 100, dtype=np.uint16) + target = (yy - 100) ** 2 + (xx - 100) ** 2 <= 30**2 + fragment = (yy - 106) ** 2 + (xx - 112) ** 2 <= 14**2 + distance = np.sqrt((yy - 100) ** 2 + (xx - 100) ** 2) + angle = (np.arctan2(yy - 100, xx - 100) + 2 * np.pi) % (2 * np.pi) + sector = (distance >= 45) & (distance <= 85) & (angle < 0.75) + patch[target] = 3500 + patch[fragment] = 7000 + patch[sector] = 5000 + image[0, 280:481, 280:481] = patch + tifffile.imwrite(fixture.image_path, image, ome=True, metadata={"axes": "CYX"}) + + bbox = BoundingBox(378, 372, 407, 401) + cropped = np.asarray(fragment[92:121, 98:127], dtype=np.bool_) + metrics = SegmentationMetrics( + threshold_method="triangle", + base_threshold=500.0, + annulus_floor=5000.0, + threshold=5000.0, + selection_mode="center_component", + area_px=int(cropped.sum()), + equivalent_diameter_um=14.0, + major_axis_um=14.0, + minor_axis_um=14.0, + eccentricity=0.0, + solidity=0.99, + circularity=0.95, + centroid_y_px=14.0, + centroid_x_px=14.0, + centroid_offset_px=13.4, + mean_intensity=7000.0, + max_intensity=7000.0, + ) + save_candidate_mask( + fixture.sample_dir / "masks/accepted.npz", + mask=cropped, + bbox=bbox, + image_shape_yx=(768, 768), + sample_id=fixture.sample_id, + candidate_id="accepted-1", + profile_name=DONOR13_V6.profile_name, + profile_fingerprint=DONOR13_V6.fingerprint(), + metrics=metrics, + implementation_version="test-v1", + ) + candidates_path = fixture.sample_dir / "html_candidates.csv" + candidates = pd.read_csv(candidates_path) + candidates.loc[0, ["bbox_x0", "bbox_y0", "bbox_x1", "bbox_y1"]] = [ + 378, + 372, + 407, + 401, + ] + candidates.to_csv(candidates_path, index=False) + + +class TestPrecisionBoundaryReview(unittest.TestCase): + def test_generates_shape_gated_replacement_without_modifying_current_mask(self): + with tempfile.TemporaryDirectory() as directory: + fixture = RecallReviewFixture(Path(directory)) + _replace_with_fragment_case(fixture) + bundle = fixture.generate() + identity = json.loads(bundle.metadata_path.read_text())["review_identity"] + review_path = _precision_review(fixture, identity) + current_before = ( + fixture.sample_dir / "masks/accepted.npz" + ).read_bytes() + + result = generate_precision_boundary_review( + fixture.sample_dir, + review_path, + fixture.sample_dir / "recall_analysis_precision_boundary_v1", + ) + + self.assertEqual(result.card_count, 1) + self.assertEqual(result.proposal_count, 1) + self.assertEqual(result.manual_only_count, 0) + table = pd.read_csv(result.candidates_path) + self.assertFalse(bool(table.loc[0, "conservative_available"])) + self.assertTrue(bool(table.loc[0, "expanded_available"])) + self.assertGreater(table.loc[0, "expanded_area_ratio"], 4.0) + self.assertGreater(table.loc[0, "expanded_current_overlap"], 0.95) + self.assertIn("Use expanded", result.page_path.read_text()) + self.assertTrue( + (result.assets_dir / "boundary-001.webp").is_file() + ) + self.assertEqual( + current_before, + (fixture.sample_dir / "masks/accepted.npz").read_bytes(), + ) + + def test_marks_nonexpanding_candidate_manual_only_and_rejects_stale_identity(self): + with tempfile.TemporaryDirectory() as directory: + fixture = RecallReviewFixture(Path(directory)) + bundle = fixture.generate() + identity = json.loads(bundle.metadata_path.read_text())["review_identity"] + review_path = _precision_review(fixture, identity) + + with mock.patch.dict( + BOUNDARY_RECOVERY_PARAMETERS, + {"min_area_growth_ratio": 2.0}, + ): + result = generate_precision_boundary_review( + fixture.sample_dir, + review_path, + fixture.sample_dir / "recall_analysis_precision_boundary_v1", + ) + self.assertEqual(result.proposal_count, 0) + self.assertEqual(result.manual_only_count, 1) + table = pd.read_csv(result.candidates_path) + self.assertEqual(table.loc[0, "proposal_status"], "manual_required") + + payload = json.loads(review_path.read_text()) + payload["identity"]["candidate_table_sha256"] = "stale" + stale_path = fixture.root / "stale-precision-review.json" + stale_path.write_text(json.dumps(payload)) + with self.assertRaisesRegex(ValueError, "candidate_table_sha256"): + generate_precision_boundary_review( + fixture.sample_dir, + stale_path, + fixture.sample_dir / "stale-boundary-review", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_precision_manual_boundary.py b/tests/oocyte/test_precision_manual_boundary.py new file mode 100644 index 0000000..6e17327 --- /dev/null +++ b/tests/oocyte/test_precision_manual_boundary.py @@ -0,0 +1,327 @@ +import json +import math +import re +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import pandas as pd +import tifffile + +from aegle.oocyte import DONOR13_V6 +from aegle.oocyte.io import load_candidate_mask, save_candidate_mask +from aegle.oocyte.models import BoundingBox, SegmentationMetrics +from aegle.oocyte.precision_boundary_finalize import ( + finalize_precision_boundary_review, +) +from aegle.oocyte.precision_boundary_review import ( + generate_precision_boundary_review, +) +from aegle.oocyte.precision_manual_boundary_finalize import ( + finalize_precision_manual_boundary_review, +) +from aegle.oocyte.precision_manual_boundary_review import ( + generate_precision_manual_boundary_review, +) +from tests.oocyte.test_recall_review import RecallReviewFixture + + +def _add_manual_target(fixture: RecallReviewFixture) -> None: + yy, xx = np.ogrid[:25, :25] + fragment = np.asarray((yy - 12) ** 2 + (xx - 12) ** 2 <= 10**2, dtype=np.bool_) + metrics = SegmentationMetrics( + threshold_method="triangle", + base_threshold=500.0, + annulus_floor=5000.0, + threshold=5000.0, + selection_mode="center_component", + area_px=int(fragment.sum()), + equivalent_diameter_um=10.0, + major_axis_um=10.0, + minor_axis_um=10.0, + eccentricity=0.0, + solidity=0.99, + circularity=0.95, + centroid_y_px=12.0, + centroid_x_px=12.0, + centroid_offset_px=0.0, + mean_intensity=10000.0, + max_intensity=10000.0, + ) + save_candidate_mask( + fixture.sample_dir / "masks/manual-target.npz", + mask=fragment, + bbox=BoundingBox(148, 148, 173, 173), + image_shape_yx=(768, 768), + sample_id=fixture.sample_id, + candidate_id="manual-target", + profile_name=DONOR13_V6.profile_name, + profile_fingerprint=DONOR13_V6.fingerprint(), + metrics=metrics, + implementation_version="test-v1", + ) + candidates_path = fixture.sample_dir / "html_candidates.csv" + candidates = pd.read_csv(candidates_path) + candidates = pd.concat( + [ + candidates, + pd.DataFrame( + [ + { + "detector_component_id": "manual-target", + "display_id": "#002", + "accepted": True, + "detector_score": 0.8, + "center_x": 160, + "center_y": 160, + "component_centroid_x": 160, + "component_centroid_y": 160, + "bbox_x0": 148, + "bbox_y0": 148, + "bbox_x1": 173, + "bbox_y1": 173, + "mask_path": "masks/manual-target.npz", + "mask_source_dir": str(fixture.sample_dir), + "detection_pass": "baseline_v6", + } + ] + ), + ], + ignore_index=True, + ) + candidates.to_csv(candidates_path, index=False) + + +def _precision_review(fixture: RecallReviewFixture, identity: dict) -> Path: + candidates = pd.read_csv(fixture.sample_dir / "html_candidates.csv") + rows = [] + for row in candidates.to_dict("records"): + target = str(row["detector_component_id"]) == "manual-target" + rows.append( + { + "review_key": f"baseline_v6:{row['detector_component_id']}", + "display_id": str(row["display_id"]), + "detector_component_id": str(row["detector_component_id"]), + "detection_pass": "baseline_v6", + "center_x": float(row["center_x"]), + "center_y": float(row["center_y"]), + "manual_status": "reject" if target else "accept", + "manual_notes": ( + "true_oocyte; mask_truncated; mask_off_target" if target else "" + ), + } + ) + path = fixture.root / "precision-review.json" + path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_precision_review", + "identity": identity, + "exported_at": "2026-07-12T12:00:00Z", + "rows": rows, + } + ) + ) + return path + + +def _embedded_payload(page_path: Path, element_id: str) -> dict: + match = re.search( + rf'', + page_path.read_text(), + flags=re.DOTALL, + ) + if match is None: + raise AssertionError(f"page is missing {element_id}") + return json.loads(match.group(1)) + + +def _prepare_base(root: Path): + fixture = RecallReviewFixture(root) + _add_manual_target(fixture) + bundle = fixture.generate() + identity = json.loads(bundle.metadata_path.read_text())["review_identity"] + precision_path = _precision_review(fixture, identity) + boundary_pack = generate_precision_boundary_review( + fixture.sample_dir, + precision_path, + root / "boundary-pack", + ) + embedded = _embedded_payload(boundary_pack.page_path, "boundary-data") + boundary_rows = embedded["rows"] + boundary_rows[0]["boundary_review_choice"] = "needs_manual" + boundary_rows[0]["boundary_review_notes"] = "trace the complete target" + boundary_path = root / "boundary-review.json" + boundary_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_precision_boundary_review", + "identity": embedded["identity"], + "exported_at": "2026-07-12T13:00:00Z", + "rows": boundary_rows, + } + ) + ) + base_dir = root / "precision-resolved-v1" + finalize_precision_boundary_review( + fixture.sample_dir, + precision_path, + boundary_path, + base_dir, + tile_shape_yx=(16, 16), + ) + return fixture, base_dir + + +def _manual_review_payload(page_path: Path, *, self_intersecting: bool = False) -> dict: + embedded = _embedded_payload(page_path, "manual-boundary-data") + row = embedded["rows"][0] + center_x = float(row["center_x"]) + center_y = float(row["center_y"]) + if self_intersecting: + vertices = [ + [center_x - 30, center_y - 30], + [center_x + 30, center_y + 30], + [center_x - 30, center_y + 30], + [center_x + 30, center_y - 30], + ] + else: + vertices = [ + [ + center_x + 34 * math.cos(2 * math.pi * index / 16), + center_y + 34 * math.sin(2 * math.pi * index / 16), + ] + for index in range(16) + ] + row["manual_boundary_choice"] = "accept_manual_contour" + row["manual_boundary_notes"] = "reviewed synthetic contour" + row["vertices_xy"] = vertices + row["vertex_count"] = len(vertices) + return { + "schema_version": 1, + "review_type": "oocyte_precision_manual_boundary_review", + "identity": embedded["identity"], + "exported_at": "2026-07-12T14:00:00Z", + "rows": [row], + } + + +class TestPrecisionManualBoundary(unittest.TestCase): + def test_generates_identity_bound_polygon_editor_without_modifying_base(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, base_dir = _prepare_base(root) + base_manifest_before = (base_dir / "precision_resolved_manifest.json").read_bytes() + + result = generate_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + root / "manual-boundary-pack", + patch_radius_px=128, + ) + + self.assertEqual(result.card_count, 1) + self.assertIn("Trace only the intended", result.page_path.read_text()) + self.assertIn("Accept contour", result.page_path.read_text()) + table = pd.read_csv(result.candidates_path) + self.assertEqual(table.loc[0, "review_key"], "baseline_v6:manual-target") + self.assertTrue((result.assets_dir / table.loc[0, "raw_asset_name"]).is_file()) + self.assertTrue( + (result.assets_dir / table.loc[0, "context_asset_name"]).is_file() + ) + self.assertEqual( + base_manifest_before, + (base_dir / "precision_resolved_manifest.json").read_bytes(), + ) + + def test_finalizes_polygon_with_base_carry_forward_and_zero_overlap(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, base_dir = _prepare_base(root) + pack = generate_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + root / "manual-boundary-pack", + patch_radius_px=128, + ) + payload = _manual_review_payload(pack.page_path) + review_path = root / "manual-boundary-review.json" + review_path.write_text(json.dumps(payload)) + base_manifest_before = (base_dir / "precision_resolved_manifest.json").read_bytes() + + result = finalize_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + review_path, + root / "precision-resolved-v2", + tile_shape_yx=(16, 16), + ) + + self.assertEqual(result.resolved_count, 2) + self.assertEqual(result.manual_added_count, 1) + self.assertEqual(result.manual_excluded_count, 0) + self.assertEqual(set(tifffile.imread(result.labels.image_path).ravel()), {0, 1, 2}) + candidates = pd.read_csv(result.candidates_path) + self.assertEqual(len(candidates), 2) + manual = candidates[ + candidates["review_key"] == "baseline_v6:manual-target" + ].iloc[0] + mask = load_candidate_mask(result.out_dir / manual["mask_path"]) + self.assertTrue(mask.metadata["reviewed_manual_polygon"]) + self.assertTrue(mask.metadata["precision_resolved"]) + self.assertTrue(mask.mask.any()) + manifest = json.loads(result.manifest_path.read_text()) + self.assertFalse(manifest["release_ready"]) + self.assertTrue(manifest["manual_boundary_complete"]) + self.assertFalse(manifest["recall_complete"]) + self.assertEqual(manifest["resolved_label_count"], 2) + self.assertEqual(manifest["overlap_audit_row_count"], 0) + self.assertEqual( + base_manifest_before, + (base_dir / "precision_resolved_manifest.json").read_bytes(), + ) + + def test_rejects_self_intersection_and_stale_candidate_table(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, base_dir = _prepare_base(root) + pack = generate_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + root / "manual-boundary-pack", + patch_radius_px=128, + ) + payload = _manual_review_payload( + pack.page_path, + self_intersecting=True, + ) + review_path = root / "self-intersection.json" + review_path.write_text(json.dumps(payload)) + with self.assertRaisesRegex(ValueError, "self-intersects"): + finalize_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + review_path, + root / "invalid-v2", + tile_shape_yx=(16, 16), + ) + + valid_payload = _manual_review_payload(pack.page_path) + stale_review_path = root / "stale-review.json" + stale_review_path.write_text(json.dumps(valid_payload)) + pack.candidates_path.write_text(pack.candidates_path.read_text() + "\n") + with self.assertRaisesRegex(ValueError, "candidate table SHA-256"): + finalize_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + stale_review_path, + root / "stale-v2", + tile_shape_yx=(16, 16), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_profiling.py b/tests/oocyte/test_profiling.py new file mode 100644 index 0000000..8316c05 --- /dev/null +++ b/tests/oocyte/test_profiling.py @@ -0,0 +1,284 @@ +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import pandas as pd +import tifffile + +from aegle.oocyte.profiling import profile_oocyte_labels + + +MAPPING_COLUMNS = [ + "label", + "detector_component_id", + "detector_score", + "acceptance_mode", + "center_x", + "center_y", + "bbox_x0", + "bbox_y0", + "bbox_x1", + "bbox_y1", + "mask_path", + "assigned_pixel_count", + "overlap_pixel_count", +] + + +def _write_antibodies(path: Path, names: list[str]) -> None: + rows = [ + { + "version": 2, + "channel_id": f"Channel:0:{index}", + "antibody_name": name, + } + for index, name in enumerate(names) + ] + pd.DataFrame(rows).to_csv(path, sep="\t", index=False) + + +def _write_mapping(path: Path, labels: np.ndarray, label_ids: list[int]) -> None: + rows = [] + for label_id in label_ids: + yy, xx = np.nonzero(labels == label_id) + rows.append( + { + "label": label_id, + "detector_component_id": f"det_{label_id:04d}", + "detector_score": 0.9, + "acceptance_mode": "reviewed", + "center_x": int(np.round(xx.mean())), + "center_y": int(np.round(yy.mean())), + "bbox_x0": max(0, int(xx.min()) - 1), + "bbox_y0": max(0, int(yy.min()) - 1), + "bbox_x1": min(labels.shape[1], int(xx.max()) + 2), + "bbox_y1": min(labels.shape[0], int(yy.max()) + 2), + "mask_path": f"masks/det_{label_id:04d}.npz", + "assigned_pixel_count": int(len(xx)), + "overlap_pixel_count": 0, + } + ) + pd.DataFrame(rows, columns=MAPPING_COLUMNS).to_csv(path, index=False) + + +class TestOocyteProfiling(unittest.TestCase): + def test_profiles_sparse_labels_edges_duplicate_markers_and_provenance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + height, width = 32, 40 + yy, xx = np.mgrid[:height, :width] + labels = np.zeros((height, width), dtype=np.uint16) + labels[0:4, 0:5] = 2 + labels[20:28, 30:39] = 7 + channels = np.stack( + [ + (yy * 10 + xx).astype(np.uint16), + (100 + yy * 2 + xx * 3).astype(np.uint16), + (1000 + yy + xx * 4).astype(np.uint16), + ] + ) + image_path = root / "raw.ome.tiff" + label_path = root / "labels.ome.tiff" + mapping_path = root / "mapping.csv" + antibodies_path = root / "antibodies.tsv" + candidates_path = root / "candidates.csv" + tifffile.imwrite( + image_path, + channels, + ome=True, + metadata={"axes": "CYX"}, + ) + tifffile.imwrite( + label_path, + labels, + ome=True, + metadata={"axes": "YX"}, + tile=(16, 16), + compression="zlib", + ) + _write_mapping(mapping_path, labels, [2, 7]) + _write_antibodies(antibodies_path, ["DAPI", "UCHL1", "UCHL1"]) + pd.DataFrame( + [ + { + "detector_component_id": "det_0002", + "detection_pass": "baseline_v6", + "boundary_warning": "False", + }, + { + "detector_component_id": "det_0007", + "detection_pass": "manual_seed", + "boundary_warning": "True", + }, + ] + ).to_csv(candidates_path, index=False) + + result = profile_oocyte_labels( + sample_id="sample-a", + image_path=image_path, + antibodies_path=antibodies_path, + label_path=label_path, + mapping_path=mapping_path, + candidates_path=candidates_path, + out_dir=root / "profiling", + pixel_size_um=0.5, + max_region_height_px=8, + merge_gap_px=0, + label_scan_height_px=8, + ) + + markers = pd.read_csv(result.artifact_paths["markers"]) + metadata = pd.read_csv(result.artifact_paths["metadata"]) + overview = pd.read_csv(result.artifact_paths["overview"]) + channel_manifest = pd.read_csv(result.artifact_paths["channels"]) + manifest = json.loads(result.artifact_paths["manifest"].read_text()) + + self.assertEqual(result.oocyte_count, 2) + self.assertEqual(result.channel_count, 3) + self.assertEqual( + list(markers.columns), + [ + "sample_id", + "oocyte_id", + "label_id", + "DAPI", + "UCHL1", + "UCHL1_1", + ], + ) + for row_index, label_id in enumerate([2, 7]): + mask = labels == label_id + self.assertEqual(markers.loc[row_index, "label_id"], label_id) + self.assertAlmostEqual(markers.loc[row_index, "DAPI"], channels[0][mask].mean()) + self.assertAlmostEqual(markers.loc[row_index, "UCHL1"], channels[1][mask].mean()) + self.assertAlmostEqual(markers.loc[row_index, "UCHL1_1"], channels[2][mask].mean()) + self.assertEqual(metadata.loc[row_index, "area_px"], int(mask.sum())) + self.assertAlmostEqual( + metadata.loc[row_index, "area_um2"], + float(mask.sum()) * 0.25, + ) + self.assertEqual(metadata.loc[0, "bbox_x0"], 0) + self.assertEqual(metadata.loc[0, "bbox_y0"], 0) + self.assertEqual(metadata.loc[1, "detection_pass"], "manual_seed") + self.assertTrue(bool(metadata.loc[1, "boundary_warning"])) + self.assertEqual(len(overview), 2) + self.assertEqual( + channel_manifest["measurement_class"].tolist(), + ["nuclear_stain", "protein_marker", "protein_marker"], + ) + self.assertEqual(manifest["measurement"]["statistic"], "raw_within_mask_mean") + self.assertEqual(manifest["bounded_read_plan"]["largest_region_shape_yx"][0], 8) + for filename, artifact in manifest["artifacts"].items(): + path = result.output_dir / filename + self.assertEqual(artifact["size_bytes"], path.stat().st_size) + self.assertEqual( + artifact["sha256"], + hashlib.sha256(path.read_bytes()).hexdigest(), + ) + + def test_zero_label_control_writes_header_only_tables(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + channels = np.zeros((2, 16, 16), dtype=np.uint16) + labels = np.zeros((16, 16), dtype=np.uint16) + image_path = root / "raw.ome.tiff" + label_path = root / "labels.ome.tiff" + mapping_path = root / "mapping.csv" + antibodies_path = root / "antibodies.tsv" + tifffile.imwrite(image_path, channels, ome=True, metadata={"axes": "CYX"}) + tifffile.imwrite(label_path, labels, ome=True, metadata={"axes": "YX"}) + pd.DataFrame(columns=MAPPING_COLUMNS).to_csv(mapping_path, index=False) + _write_antibodies(antibodies_path, ["DAPI", "UCHL1"]) + + result = profile_oocyte_labels( + sample_id="negative", + image_path=image_path, + antibodies_path=antibodies_path, + label_path=label_path, + mapping_path=mapping_path, + out_dir=root / "profiling", + pixel_size_um=0.5, + label_scan_height_px=8, + ) + + markers = pd.read_csv(result.artifact_paths["markers"]) + metadata = pd.read_csv(result.artifact_paths["metadata"]) + self.assertTrue(markers.empty) + self.assertTrue(metadata.empty) + self.assertEqual( + list(markers.columns), + ["sample_id", "oocyte_id", "label_id", "DAPI", "UCHL1"], + ) + self.assertEqual(result.oocyte_count, 0) + + def test_rejects_unmapped_label_and_assigned_count_mismatch(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + channels = np.ones((2, 16, 16), dtype=np.uint16) + labels = np.zeros((16, 16), dtype=np.uint16) + labels[2:6, 2:6] = 1 + image_path = root / "raw.ome.tiff" + label_path = root / "labels.ome.tiff" + mapping_path = root / "mapping.csv" + antibodies_path = root / "antibodies.tsv" + tifffile.imwrite(image_path, channels, ome=True, metadata={"axes": "CYX"}) + tifffile.imwrite(label_path, labels, ome=True, metadata={"axes": "YX"}) + pd.DataFrame(columns=MAPPING_COLUMNS).to_csv(mapping_path, index=False) + _write_antibodies(antibodies_path, ["DAPI", "UCHL1"]) + with self.assertRaisesRegex(ValueError, "label sets differ"): + profile_oocyte_labels( + sample_id="bad", + image_path=image_path, + antibodies_path=antibodies_path, + label_path=label_path, + mapping_path=mapping_path, + out_dir=root / "unmapped", + pixel_size_um=0.5, + ) + + _write_mapping(mapping_path, labels, [1]) + mapping = pd.read_csv(mapping_path) + mapping.loc[0, "assigned_pixel_count"] += 1 + mapping.to_csv(mapping_path, index=False) + with self.assertRaisesRegex(ValueError, "assigned_pixel_count mismatch"): + profile_oocyte_labels( + sample_id="bad", + image_path=image_path, + antibodies_path=antibodies_path, + label_path=label_path, + mapping_path=mapping_path, + out_dir=root / "bad-count", + pixel_size_um=0.5, + ) + + def test_rejects_channel_table_mismatch(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + channels = np.ones((2, 16, 16), dtype=np.uint16) + labels = np.zeros((16, 16), dtype=np.uint16) + image_path = root / "raw.ome.tiff" + label_path = root / "labels.ome.tiff" + mapping_path = root / "mapping.csv" + antibodies_path = root / "antibodies.tsv" + tifffile.imwrite(image_path, channels, ome=True, metadata={"axes": "CYX"}) + tifffile.imwrite(label_path, labels, ome=True, metadata={"axes": "YX"}) + pd.DataFrame(columns=MAPPING_COLUMNS).to_csv(mapping_path, index=False) + _write_antibodies(antibodies_path, ["UCHL1"]) + + with self.assertRaisesRegex(ValueError, "row count"): + profile_oocyte_labels( + sample_id="bad", + image_path=image_path, + antibodies_path=antibodies_path, + label_path=label_path, + mapping_path=mapping_path, + out_dir=root / "profiling", + pixel_size_um=0.5, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_qc.py b/tests/oocyte/test_qc.py new file mode 100644 index 0000000..65e8f2c --- /dev/null +++ b/tests/oocyte/test_qc.py @@ -0,0 +1,45 @@ +import unittest + +import pandas as pd + +from aegle.oocyte import accepted_duplicate_suspects + + +class TestOocyteSpatialQc(unittest.TestCase): + def test_flags_only_diameter_scaled_overlaps(self): + candidates = pd.DataFrame( + [ + { + "detector_component_id": "det_0000", + "accepted": True, + "center_x": 100, + "center_y": 100, + "local_equivalent_diameter_um": 40.0, + }, + { + "detector_component_id": "det_0001", + "accepted": True, + "center_x": 150, + "center_y": 100, + "local_equivalent_diameter_um": 40.0, + }, + { + "detector_component_id": "det_0002", + "accepted": True, + "center_x": 400, + "center_y": 400, + "local_equivalent_diameter_um": 40.0, + }, + ] + ) + + result = accepted_duplicate_suspects(candidates, pixel_size_um=0.5) + + self.assertEqual(len(result), 1) + self.assertEqual(result.iloc[0]["detector_component_id_a"], "det_0000") + self.assertEqual(result.iloc[0]["detector_component_id_b"], "det_0001") + self.assertAlmostEqual(result.iloc[0]["overlap_fraction_smaller"], 0.75) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_recall_manual_boundary.py b/tests/oocyte/test_recall_manual_boundary.py new file mode 100644 index 0000000..e42ea2d --- /dev/null +++ b/tests/oocyte/test_recall_manual_boundary.py @@ -0,0 +1,229 @@ +import hashlib +import json +import math +import re +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import pandas as pd +import tifffile + +from aegle.oocyte.io import load_candidate_mask +from aegle.oocyte.manual_seed_finalize import finalize_manual_seed_review +from aegle.oocyte.recall_manual_boundary_finalize import ( + finalize_recall_manual_boundary_review, +) +from aegle.oocyte.recall_manual_boundary_review import ( + generate_recall_manual_boundary_review, +) +from aegle.oocyte.recall_review import analyze_recall_review +from tests.oocyte.test_manual_seed_finalize import _manual_review_payload +from tests.oocyte.test_recall_review import RecallReviewFixture + + +def _embedded_payload(page_path: Path) -> dict: + match = re.search( + r'', + page_path.read_text(), + flags=re.DOTALL, + ) + if match is None: + raise AssertionError("Recall manual-boundary page is missing its payload") + return json.loads(match.group(1)) + + +def _add_second_missed_oocyte(fixture: RecallReviewFixture) -> None: + image = tifffile.imread(fixture.image_path) + yy, xx = np.ogrid[:768, :768] + target = (yy - 600) ** 2 + (xx - 600) ** 2 <= 28**2 + image[0, target] = 9000 + tifffile.imwrite( + fixture.image_path, + image, + ome=True, + metadata={"axes": "CYX"}, + ) + manifest_path = fixture.sample_dir / "run_manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["source_image_size_bytes"] = fixture.image_path.stat().st_size + manifest_path.write_text(json.dumps(manifest)) + + +def _prepare_manual_boundary(root: Path): + fixture = RecallReviewFixture(root) + _add_second_missed_oocyte(fixture) + bundle = fixture.generate() + metadata = json.loads(bundle.metadata_path.read_text()) + recall_path = root / "recall-review.json" + recall_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall", + "sample": metadata["review_identity"], + "windows": [], + "missing_oocytes": [ + { + "annotation_id": "accepted-miss", + "window_id": metadata["windows"][0]["window_id"], + "x": 160, + "y": 160, + "notes": "safe automatic boundary", + }, + { + "annotation_id": "manual-boundary-miss", + "window_id": metadata["windows"][0]["window_id"], + "x": 600, + "y": 600, + "notes": "two bad automatic boundaries", + }, + ], + } + ) + ) + analysis_dir = root / "analysis" + analyze_recall_review(fixture.sample_dir, recall_path, analysis_dir) + manual_payload = _manual_review_payload(analysis_dir / "manual_seed_review.html") + manual_payload["rows"][0]["manual_mask_choice"] = "accept_manual_expanded" + manual_payload["rows"][1]["manual_mask_choice"] = "neither" + manual_payload["rows"][1]["manual_notes"] = ( + "true_oocyte; conservative_under-segmented; " + "expanded_over-segmented_into_neighbor; needs_manual_boundary" + ) + manual_review_path = root / "manual-seed-review.json" + manual_review_path.write_text(json.dumps(manual_payload)) + base_dir = root / "reviewed-manual-seed-v1" + finalize_manual_seed_review( + fixture.sample_dir, + manual_review_path, + base_dir, + tile_shape_yx=(16, 16), + ) + pack = generate_recall_manual_boundary_review( + fixture.sample_dir, + manual_review_path, + base_dir, + root / "manual-boundary-pack", + patch_radius_px=128, + ) + return fixture, manual_review_path, base_dir, pack + + +class TestRecallManualBoundary(unittest.TestCase): + def test_generates_and_finalizes_identity_bound_recall_contour(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, _, base_dir, pack = _prepare_manual_boundary(root) + base_manifest = base_dir / "manual_seed_finalize_manifest.json" + base_manifest_before = base_manifest.read_bytes() + + self.assertEqual(pack.card_count, 1) + page = pack.page_path.read_text() + self.assertIn("oocyte_recall_manual_boundary_review", page) + self.assertIn("expanded proposal was rejected", page) + candidates = pd.read_csv(pack.candidates_path) + self.assertEqual(candidates.loc[0, "review_key"], "manual-boundary-miss") + self.assertEqual(int(candidates.loc[0, "review_index"]), 2) + self.assertEqual(candidates.loc[0, "display_id"], "#R002") + self.assertTrue( + (pack.assets_dir / candidates.loc[0, "raw_asset_name"]).is_file() + ) + self.assertTrue( + (pack.assets_dir / candidates.loc[0, "context_asset_name"]).is_file() + ) + + embedded = _embedded_payload(pack.page_path) + row = embedded["rows"][0] + center_x = float(row["center_x"]) + center_y = float(row["center_y"]) + bad_row = dict(row) + bad_row["manual_boundary_choice"] = "accept_manual_contour" + bad_row["manual_boundary_notes"] = "self-intersecting test" + bad_row["vertices_xy"] = [ + [center_x - 30, center_y - 30], + [center_x + 30, center_y + 30], + [center_x - 30, center_y + 30], + [center_x + 30, center_y - 30], + ] + bad_path = root / "bad-contour.json" + bad_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall_manual_boundary_review", + "identity": embedded["identity"], + "rows": [bad_row], + } + ) + ) + with self.assertRaisesRegex(ValueError, "self-intersects"): + finalize_recall_manual_boundary_review( + fixture.sample_dir, + base_dir, + bad_path, + root / "bad-v2", + tile_shape_yx=(16, 16), + ) + + vertices = [ + [ + center_x + 30 * math.cos(2 * math.pi * index / 20), + center_y + 30 * math.sin(2 * math.pi * index / 20), + ] + for index in range(20) + ] + row["manual_boundary_choice"] = "accept_manual_contour" + row["manual_boundary_notes"] = "reviewed Recall contour" + row["vertices_xy"] = vertices + row["vertex_count"] = len(vertices) + review_path = root / "recall-manual-boundary-review.json" + review_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall_manual_boundary_review", + "identity": embedded["identity"], + "exported_at": "2026-07-12T18:00:00Z", + "rows": [row], + } + ) + ) + result = finalize_recall_manual_boundary_review( + fixture.sample_dir, + base_dir, + review_path, + root / "reviewed-manual-seed-v2", + tile_shape_yx=(16, 16), + ) + + self.assertEqual(result.manual_added_count, 1) + self.assertEqual(result.manual_excluded_count, 0) + self.assertEqual(result.combined_label_count, 3) + self.assertEqual(result.labels.overlap_pixel_count, 0) + self.assertEqual(base_manifest_before, base_manifest.read_bytes()) + contour_table = pd.read_csv(result.candidates_path) + contour_path = result.out_dir / contour_table.loc[0, "mask_path"] + contour = load_candidate_mask(contour_path) + self.assertTrue( + contour.mask[ + 600 - contour.bbox.y0, + 600 - contour.bbox.x0, + ] + ) + manifest = json.loads(result.manifest_path.read_text()) + self.assertEqual(manifest["base_label_count"], 2) + self.assertEqual(manifest["combined_label_count"], 3) + self.assertEqual(manifest["remaining_manual_boundary_count"], 0) + self.assertTrue( + all( + hashlib.sha256(Path(record["path"]).read_bytes()).hexdigest() + == record["sha256"] + for record in manifest["artifacts"].values() + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_recall_overlay.py b/tests/oocyte/test_recall_overlay.py new file mode 100644 index 0000000..bc38f98 --- /dev/null +++ b/tests/oocyte/test_recall_overlay.py @@ -0,0 +1,295 @@ +import io +import json +import tempfile +import unittest +from pathlib import Path + +import numpy as np +from PIL import Image + +from aegle.oocyte.precision_manual_boundary_finalize import ( + finalize_precision_manual_boundary_review, +) +from aegle.oocyte.precision_manual_boundary_review import ( + generate_precision_manual_boundary_review, +) +from aegle.oocyte.manual_seed_finalize import finalize_manual_seed_review +from aegle.oocyte.recall_review import ( + RecallReviewRuntime, + _load_sample, + analyze_recall_review, + generate_recall_review_bundle, +) +from aegle.oocyte.recall_review_batch import ( + generate_batch_recall_review_bundle, +) +from tests.oocyte.test_precision_manual_boundary import ( + _manual_review_payload, + _prepare_base, +) +from tests.oocyte.test_manual_seed_finalize import ( + _manual_review_payload as _manual_seed_review_payload, +) + + +def _prepare_v2(root: Path): + fixture, base_dir = _prepare_base(root) + pack = generate_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + root / "manual-boundary-pack", + patch_radius_px=128, + ) + review_path = root / "manual-boundary-review.json" + review_path.write_text(json.dumps(_manual_review_payload(pack.page_path))) + v2_dir = root / "precision-resolved-v2" + finalize_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + review_path, + v2_dir, + tile_shape_yx=(16, 16), + ) + return fixture, base_dir, v2_dir + + +def _prepare_v2_excluding_target(root: Path): + fixture, base_dir = _prepare_base(root) + pack = generate_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + root / "manual-boundary-pack", + patch_radius_px=128, + ) + payload = _manual_review_payload(pack.page_path) + payload["rows"][0]["manual_boundary_choice"] = "exclude" + payload["rows"][0]["vertices_xy"] = [] + payload["rows"][0]["vertex_count"] = 0 + review_path = root / "manual-boundary-review.json" + review_path.write_text(json.dumps(payload)) + v2_dir = root / "precision-resolved-v2" + finalize_precision_manual_boundary_review( + fixture.sample_dir, + base_dir, + review_path, + v2_dir, + tile_shape_yx=(16, 16), + ) + return fixture, v2_dir + + +class TestRecallReviewedOverlay(unittest.TestCase): + def test_reviewed_overlay_drives_masks_coverage_and_probe(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, _, v2_dir = _prepare_v2(root) + bundle = generate_recall_review_bundle( + fixture.sample_dir, + overlay_dir=v2_dir, + window_radius_px=128, + window_stride_px=192, + overview_downsample=8, + ) + metadata = json.loads(bundle.metadata_path.read_text()) + + self.assertEqual( + metadata["review_identity"]["overlay_delivery_name"], + "precision_resolved_v2", + ) + self.assertEqual(metadata["mask_overlay"]["candidate_count"], 2) + self.assertEqual(len(metadata["candidates"]), 2) + manual_row = next( + row + for row in metadata["candidates"] + if row["detector_component_id"] == "manual-target" + ) + self.assertEqual( + manual_row["resolution_source"], + "precision_manual_contour", + ) + + with RecallReviewRuntime( + fixture.sample_dir, + overlay_dir=v2_dir, + ) as runtime: + self.assertEqual(len(runtime.sample.candidates), 2) + self.assertEqual(len(runtime.sample.detector_candidates), 2) + # This point lies outside the rejected 10 px fragment but inside + # the reviewed 34 px polygon. + self.assertTrue(runtime._point_covered(188.0, 160.0)) + probe = runtime.probe(188.0, 160.0) + self.assertTrue(probe["already_covered"]) + self.assertEqual(probe["failure_class"], "already_covered") + payload = runtime.window_payload((160, 160), 128) + target = next( + row + for row in payload["candidates"] + if row["detector_component_id"] == "manual-target" + ) + self.assertEqual( + target["resolution_source"], + "precision_manual_contour", + ) + overlay = np.asarray( + Image.open( + io.BytesIO(runtime.render_overlay((160, 160), 128)) + ) + ) + self.assertGreater(int(overlay[128, 156, 3]), 0) + + def test_rejects_unresolved_delivery_and_pre_overlay_review_json(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, base_dir, v2_dir = _prepare_v2(root) + with self.assertRaisesRegex(ValueError, "unresolved manual"): + generate_recall_review_bundle( + fixture.sample_dir, + overlay_dir=base_dir, + window_radius_px=128, + window_stride_px=192, + overview_downsample=8, + ) + + bundle = generate_recall_review_bundle( + fixture.sample_dir, + overlay_dir=v2_dir, + window_radius_px=128, + window_stride_px=192, + overview_downsample=8, + ) + old_identity = _load_sample(fixture.sample_dir).review_identity + old_review_path = root / "old-recall-review.json" + old_review_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall", + "sample": old_identity, + "windows": [], + "missing_oocytes": [], + } + ) + ) + with self.assertRaisesRegex(ValueError, "overlay_"): + analyze_recall_review( + fixture.sample_dir, + old_review_path, + root / "old-analysis", + ) + + current_review_path = root / "current-recall-review.json" + current_review_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall", + "sample": json.loads(bundle.metadata_path.read_text())[ + "review_identity" + ], + "windows": [], + "missing_oocytes": [], + } + ) + ) + table_path = analyze_recall_review( + fixture.sample_dir, + current_review_path, + root / "current-analysis", + ) + self.assertTrue(table_path.is_file()) + analysis_summary = json.loads( + (table_path.parent / "summary.json").read_text() + ) + self.assertEqual( + analysis_summary["sample"]["overlay_delivery_name"], + "precision_resolved_v2", + ) + + def test_batch_no_generate_recovers_bound_overlay(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, _, v2_dir = _prepare_v2(root) + generate_recall_review_bundle( + fixture.sample_dir, + overlay_dir=v2_dir, + window_radius_px=128, + window_stride_px=192, + overview_downsample=8, + ) + + batch = generate_batch_recall_review_bundle( + root, + sample_ids=[fixture.sample_id], + generate_samples=False, + ) + + self.assertEqual(batch.total_candidate_count, 2) + manifest = json.loads(batch.manifest_path.read_text()) + self.assertEqual( + manifest["samples"][0]["overlay_name"], + "precision_resolved_v2", + ) + self.assertIn("precision_resolved_v2", batch.index_path.read_text()) + + def test_manual_seed_finalizer_carries_forward_reviewed_overlay_only(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, v2_dir = _prepare_v2_excluding_target(root) + bundle = generate_recall_review_bundle( + fixture.sample_dir, + overlay_dir=v2_dir, + window_radius_px=128, + window_stride_px=192, + overview_downsample=8, + ) + metadata = json.loads(bundle.metadata_path.read_text()) + recall_path = root / "recall-review.json" + recall_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall", + "sample": metadata["review_identity"], + "windows": [], + "missing_oocytes": [ + { + "annotation_id": "restored-target", + "window_id": metadata["windows"][0]["window_id"], + "x": 160, + "y": 160, + "notes": "reviewed Recall miss", + } + ], + } + ) + ) + analysis_dir = root / "analysis" + analyze_recall_review(fixture.sample_dir, recall_path, analysis_dir) + manual_payload = _manual_seed_review_payload( + analysis_dir / "manual_seed_review.html" + ) + manual_payload["rows"][0][ + "manual_mask_choice" + ] = "accept_manual_expanded" + manual_path = root / "manual-review.json" + manual_path.write_text(json.dumps(manual_payload)) + + result = finalize_manual_seed_review( + fixture.sample_dir, + manual_path, + root / "finalized", + tile_shape_yx=(16, 16), + ) + + self.assertEqual(result.accepted_count, 1) + self.assertEqual(result.combined_labels.label_count, 2) + manifest = json.loads(result.manifest_path.read_text()) + self.assertEqual(manifest["production_candidate_count"], 1) + self.assertEqual( + manifest["sample"]["overlay_delivery_name"], + "precision_resolved_v2", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_recall_review.py b/tests/oocyte/test_recall_review.py new file mode 100644 index 0000000..c90c68f --- /dev/null +++ b/tests/oocyte/test_recall_review.py @@ -0,0 +1,572 @@ +import io +import json +import tempfile +import threading +import unittest +from http.server import ThreadingHTTPServer +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import urlopen + +import numpy as np +import pandas as pd +import tifffile +from PIL import Image + +from aegle.oocyte import DONOR13_V6 +from aegle.oocyte.io import save_candidate_mask +from aegle.oocyte.models import BoundingBox, ExtractedPatch, SegmentationMetrics +from aegle.oocyte.recall_review import ( + DEFAULT_OVERVIEW_DOWNSAMPLE, + DEFAULT_WINDOW_RADIUS_PX, + DEFAULT_WINDOW_STRIDE_PX, + RecallReviewRuntime, + _axis_centers, + _handler_for, + _segment_manual_seed_patch, + analyze_recall_review, + classify_recall_failure, + generate_recall_review_bundle, +) + + +class RecallReviewFixture: + def __init__(self, root: Path, sample_id: str = "synthetic-recall"): + self.root = root + self.sample_id = sample_id + self.sample_dir = root / self.sample_id + self.sample_dir.mkdir() + image = np.full((2, 768, 768), 100, dtype=np.uint16) + yy, xx = np.ogrid[:768, :768] + accepted = (yy - 380) ** 2 + (xx - 380) ** 2 <= 36**2 + missed = (yy - 160) ** 2 + (xx - 160) ** 2 <= 32**2 + image[0, accepted] = 12000 + image[0, missed] = 10000 + self.image_path = root / f"{sample_id}-source.ome.tiff" + tifffile.imwrite( + self.image_path, + image, + ome=True, + metadata={"axes": "CYX"}, + ) + bbox = BoundingBox(344, 344, 417, 417) + cropped = np.asarray(accepted[344:417, 344:417], dtype=np.bool_) + metrics = SegmentationMetrics( + threshold_method="triangle", + base_threshold=500.0, + annulus_floor=100.0, + threshold=500.0, + selection_mode="center_component", + area_px=int(cropped.sum()), + equivalent_diameter_um=36.0, + major_axis_um=36.0, + minor_axis_um=36.0, + eccentricity=0.0, + solidity=0.99, + circularity=0.95, + centroid_y_px=36.0, + centroid_x_px=36.0, + centroid_offset_px=0.0, + mean_intensity=12000.0, + max_intensity=12000.0, + ) + mask_path = self.sample_dir / "masks/accepted.npz" + save_candidate_mask( + mask_path, + mask=cropped, + bbox=bbox, + image_shape_yx=(768, 768), + sample_id=self.sample_id, + candidate_id="accepted-1", + profile_name=DONOR13_V6.profile_name, + profile_fingerprint=DONOR13_V6.fingerprint(), + metrics=metrics, + implementation_version="test-v1", + ) + pd.DataFrame( + [ + { + "detector_component_id": "accepted-1", + "display_id": "#001", + "accepted": True, + "detector_score": 0.9, + "center_x": 380, + "center_y": 380, + "component_centroid_x": 380, + "component_centroid_y": 380, + "bbox_x0": 344, + "bbox_y0": 344, + "bbox_x1": 417, + "bbox_y1": 417, + "mask_path": "masks/accepted.npz", + "mask_source_dir": str(self.sample_dir), + "detection_pass": "baseline_v6", + } + ] + ).to_csv(self.sample_dir / "html_candidates.csv", index=False) + pd.DataFrame( + [ + { + "detector_component_id": "accepted-1", + "accepted": True, + "detector_score": 0.9, + "center_x": 380, + "center_y": 380, + "component_centroid_x": 380, + "component_centroid_y": 380, + }, + { + "detector_component_id": "rejected-1", + "accepted": False, + "detector_score": 0.42, + "center_x": 160, + "center_y": 160, + "component_centroid_x": 160, + "component_centroid_y": 160, + }, + ] + ).to_csv(self.sample_dir / "candidates.csv", index=False) + pd.DataFrame( + [ + { + "detector_component_id": "coarse-1", + "coarse_center_x": 160, + "coarse_center_y": 160, + } + ] + ).to_csv(self.sample_dir / "coarse_candidates.csv", index=False) + pd.DataFrame( + columns=["detector_component_id", "duplicate_of"] + ).to_csv(self.sample_dir / "combined_duplicate_suppressed.csv", index=False) + (self.sample_dir / "run_manifest.json").write_text( + json.dumps( + { + "sample_id": self.sample_id, + "source_image": str(self.image_path), + "source_image_size_bytes": self.image_path.stat().st_size, + "resolved_channel_index": 0, + "resolved_config": DONOR13_V6.to_dict(), + "profile_fingerprint": DONOR13_V6.fingerprint(), + "implementation_version": "test-v1", + } + ) + ) + (self.sample_dir / "summary.json").write_text( + json.dumps( + { + "sample_id": self.sample_id, + "image_shape_yx": [768, 768], + "profile_name": DONOR13_V6.profile_name, + } + ) + ) + + def generate(self): + return generate_recall_review_bundle( + self.sample_dir, + window_radius_px=128, + window_stride_px=192, + overview_downsample=8, + ) + + +class TestRecallFailureClassification(unittest.TestCase): + def test_classifies_each_detector_stage(self): + common = { + "nearest_suppressed_distance_px": None, + "nearest_coarse_distance_px": None, + "nearest_refined_distance_px": None, + } + self.assertEqual( + classify_recall_failure(already_covered=True, **common), + "already_covered", + ) + self.assertEqual( + classify_recall_failure( + already_covered=False, + nearest_suppressed_distance_px=10, + nearest_coarse_distance_px=10, + nearest_refined_distance_px=10, + ), + "dedup_error", + ) + self.assertEqual( + classify_recall_failure(already_covered=False, **common), + "proposal_miss", + ) + self.assertEqual( + classify_recall_failure( + already_covered=False, + nearest_suppressed_distance_px=None, + nearest_coarse_distance_px=20, + nearest_refined_distance_px=None, + ), + "segmentation_miss", + ) + self.assertEqual( + classify_recall_failure( + already_covered=False, + nearest_suppressed_distance_px=None, + nearest_coarse_distance_px=300, + nearest_refined_distance_px=15, + ), + "acceptance_miss", + ) + + def test_manual_seed_prefers_the_component_nearest_the_click(self): + yy, xx = np.ogrid[:361, :361] + patch = np.full((361, 361), 100.0, dtype=np.float32) + target = ((yy - 180) ** 2 + (xx - 180) ** 2 <= 34**2) & ( + (yy - 180) ** 2 + (xx - 180) ** 2 >= 19**2 + ) + target &= ~((xx >= 176) & (xx <= 184) & (yy < 180)) + larger_neighbor = (yy - 70) ** 2 + (xx - 70) ** 2 <= 45**2 + patch[target] = 10000.0 + patch[larger_neighbor] = 10000.0 + + result = _segment_manual_seed_patch( + patch, + annulus_floor_percentile=95.0, + ) + + self.assertEqual( + result.metrics.selection_mode, + "manual_seed_nearest_component", + ) + self.assertLess(result.metrics.centroid_offset_px, 35.0) + self.assertFalse(bool(result.mask[70, 70])) + + def test_manual_seed_watershed_splits_touching_oocytes(self): + yy, xx = np.ogrid[:201, :201] + patch = np.full((201, 201), 100.0, dtype=np.float32) + upper = (yy - 100) ** 2 + (xx - 100) ** 2 <= 29**2 + lower = (yy - 145) ** 2 + (xx - 108) ** 2 <= 28**2 + patch[upper | lower] = 10000.0 + + result = _segment_manual_seed_patch( + patch, + annulus_floor_percentile=80.0, + annulus_inner_px=40, + annulus_outer_px=90, + ) + + self.assertEqual( + result.metrics.selection_mode, + "manual_seed_watershed_component", + ) + self.assertTrue(bool(result.mask[100, 100])) + self.assertFalse(bool(result.mask[145, 108])) + + def test_shape_recovery_is_opt_in_for_large_round_expansion(self): + yy, xx = np.ogrid[:201, :201] + patch = np.full((201, 201), 100.0, dtype=np.float32) + target = (yy - 100) ** 2 + (xx - 100) ** 2 <= 30**2 + bright_fragment = (yy - 106) ** 2 + (xx - 112) ** 2 <= 14**2 + distance = np.sqrt((yy - 100) ** 2 + (xx - 100) ** 2) + angle = (np.arctan2(yy - 100, xx - 100) + 2 * np.pi) % (2 * np.pi) + bright_annulus_sector = ( + (distance >= 45) & (distance <= 85) & (angle < 0.75) + ) + patch[target] = 3500.0 + patch[bright_fragment] = 7000.0 + patch[bright_annulus_sector] = 5000.0 + + class Source: + def read_patch(self, _center, _radius): + return ExtractedPatch( + image=patch, + bbox=BoundingBox(0, 0, 201, 201), + image_shape_yx=(201, 201), + padding_tblr=(0, 0, 0, 0), + ) + + runtime = object.__new__(RecallReviewRuntime) + runtime.source = Source() + standard = runtime.segment_manual_provisionals(100, 100) + recovered = runtime.segment_manual_provisionals( + 100, + 100, + allow_shape_recovery=True, + ) + + self.assertEqual(standard.expanded_percentile, 95.0) + self.assertEqual(recovered.expanded_percentile, 85.0) + self.assertGreater( + recovered.expanded.metrics.area_px, + 4 * standard.expanded.metrics.area_px, + ) + self.assertGreater(recovered.expanded.metrics.circularity, 0.8) + self.assertGreater(recovered.expanded.metrics.solidity, 0.9) + + +class TestRecallReviewBundle(unittest.TestCase): + def test_survey_defaults_cover_real_13_21_shape_in_117_windows(self): + self.assertEqual(DEFAULT_WINDOW_RADIUS_PX, 1280) + self.assertEqual(DEFAULT_WINDOW_STRIDE_PX, 2304) + self.assertEqual(DEFAULT_OVERVIEW_DOWNSAMPLE, 16) + + def assert_axis_coverage(length): + centers = _axis_centers( + length, + DEFAULT_WINDOW_RADIUS_PX, + DEFAULT_WINDOW_STRIDE_PX, + ) + intervals = [ + ( + max(0, center - DEFAULT_WINDOW_RADIUS_PX), + min(length, center + DEFAULT_WINDOW_RADIUS_PX + 1), + ) + for center in centers + ] + self.assertEqual(intervals[0][0], 0) + self.assertEqual(intervals[-1][1], length) + self.assertTrue( + all(right[0] <= left[1] for left, right in zip(intervals, intervals[1:])) + ) + return centers + + x_centers = assert_axis_coverage(29665) + y_centers = assert_axis_coverage(20705) + self.assertEqual(len(x_centers), 13) + self.assertEqual(len(y_centers), 9) + self.assertEqual(len(x_centers) * len(y_centers), 117) + + def test_generates_coordinate_faithful_bundle_and_exact_overlay(self): + with tempfile.TemporaryDirectory() as directory: + fixture = RecallReviewFixture(Path(directory)) + bundle = fixture.generate() + + self.assertTrue(bundle.page_path.is_file()) + self.assertTrue(bundle.overview_path.is_file()) + metadata = json.loads(bundle.metadata_path.read_text()) + self.assertEqual(metadata["image_width"], 768) + self.assertEqual(metadata["image_height"], 768) + self.assertIsInstance( + metadata["review_identity"]["source_image_mtime_ns"], + str, + ) + self.assertGreater(bundle.window_count, 4) + identity = metadata["review_identity"] + self.assertEqual(identity["recall_coverage_profile"], "custom") + self.assertEqual(identity["recall_window_count"], bundle.window_count) + self.assertEqual(len(identity["recall_window_geometry_sha256"]), 64) + self.assertIn("Add missed oocyte", bundle.page_path.read_text()) + self.assertIn("Next unreviewed", bundle.page_path.read_text()) + self.assertIn( + "recall_window_geometry_sha256", + bundle.page_path.read_text(), + ) + + with RecallReviewRuntime(fixture.sample_dir) as runtime: + patch_bytes = runtime.render_patch((380, 380), 128, "local") + patch = Image.open(io.BytesIO(patch_bytes)) + self.assertEqual(patch.size, (257, 257)) + overlay_bytes = runtime.render_overlay((380, 380), 128) + overlay = np.asarray(Image.open(io.BytesIO(overlay_bytes))) + self.assertEqual(overlay.shape, (257, 257, 4)) + self.assertGreater(int((overlay[..., 3] > 0).sum()), 100) + edge = Image.open( + io.BytesIO(runtime.render_patch((0, 0), 128, "local")) + ) + self.assertEqual(edge.size, (257, 257)) + payload = runtime.window_payload((380, 380), 128) + self.assertEqual(len(payload["candidates"]), 1) + self.assertEqual(payload["candidates"][0]["display_id"], "#001") + + def test_probe_and_offline_ingestion_preserve_production_outputs(self): + with tempfile.TemporaryDirectory() as directory: + fixture = RecallReviewFixture(Path(directory)) + bundle = fixture.generate() + candidates_before = (fixture.sample_dir / "html_candidates.csv").read_bytes() + metadata = json.loads(bundle.metadata_path.read_text()) + with RecallReviewRuntime(fixture.sample_dir) as runtime: + covered = runtime.probe(380, 380) + missed = runtime.probe(160, 160) + self.assertEqual(covered["failure_class"], "already_covered") + self.assertEqual(missed["failure_class"], "acceptance_miss") + self.assertIsNotNone(missed["manual_conservative_metrics"]) + self.assertIsNotNone(missed["manual_expanded_metrics"]) + self.assertIn( + missed["manual_conservative_percentile"], + (95.0, 90.0, 85.0, 80.0, 75.0, 70.0, 65.0, 60.0), + ) + + review_path = Path(directory) / "review.json" + review_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall", + "sample": metadata["review_identity"], + "windows": [ + { + "window_id": metadata["windows"][0]["window_id"], + "status": "has_misses", + } + ], + "missing_oocytes": [ + { + "annotation_id": "manual-1", + "window_id": metadata["windows"][0]["window_id"], + "x": 160, + "y": 160, + "notes": "synthetic missed circle", + } + ], + } + ) + ) + table_path = analyze_recall_review( + fixture.sample_dir, + review_path, + Path(directory) / "analysis", + ) + table = pd.read_csv(table_path) + self.assertEqual(table.loc[0, "failure_class"], "acceptance_miss") + self.assertTrue( + Path(table.loc[0, "manual_conservative_mask_path"]).is_file() + ) + self.assertTrue( + Path(table.loc[0, "manual_expanded_mask_path"]).is_file() + ) + self.assertTrue( + (Path(directory) / "analysis/manual_seed_review.html").is_file() + ) + self.assertIn( + 'href="../recall_review.html"', + (Path(directory) / "analysis/manual_seed_review.html").read_text(), + ) + self.assertTrue( + (Path(directory) / "analysis/review_assets/seed-001.webp").is_file() + ) + analysis_summary = json.loads( + (Path(directory) / "analysis/summary.json").read_text() + ) + self.assertEqual( + analysis_summary["sample"]["recall_window_geometry_sha256"], + metadata["review_identity"]["recall_window_geometry_sha256"], + ) + self.assertEqual( + candidates_before, + (fixture.sample_dir / "html_candidates.csv").read_bytes(), + ) + + def test_http_routes_return_health_images_and_validation_errors(self): + with tempfile.TemporaryDirectory() as directory: + fixture = RecallReviewFixture(Path(directory)) + fixture.generate() + with RecallReviewRuntime(fixture.sample_dir) as runtime: + server = ThreadingHTTPServer(("127.0.0.1", 0), _handler_for(runtime)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + base = f"http://127.0.0.1:{server.server_port}" + try: + health = json.loads(urlopen(base + "/health", timeout=3).read()) + self.assertTrue(health["read_only"]) + patch = urlopen( + base + "/api/patch.webp?x=380&y=380&radius=128", + timeout=3, + ).read() + self.assertGreater(len(patch), 1000) + with self.assertRaises(HTTPError) as context: + urlopen(base + "/api/patch.webp?x=-1&y=0&radius=128") + self.assertEqual(context.exception.code, 400) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=3) + + def test_ingestion_rejects_wrong_sample_identity(self): + with tempfile.TemporaryDirectory() as directory: + fixture = RecallReviewFixture(Path(directory)) + bundle = fixture.generate() + metadata = json.loads(bundle.metadata_path.read_text()) + metadata["review_identity"]["sample_id"] = "wrong-sample" + review_path = Path(directory) / "wrong.json" + review_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall", + "sample": metadata["review_identity"], + "windows": [], + "missing_oocytes": [], + } + ) + ) + with self.assertRaisesRegex(ValueError, "sample_id"): + analyze_recall_review( + fixture.sample_dir, + review_path, + Path(directory) / "analysis", + ) + + def test_ingestion_rejects_review_from_a_different_coverage_grid(self): + with tempfile.TemporaryDirectory() as directory: + fixture = RecallReviewFixture(Path(directory)) + bundle = fixture.generate() + metadata = json.loads(bundle.metadata_path.read_text()) + stale_identity = dict(metadata["review_identity"]) + stale_identity["recall_window_geometry_sha256"] = "0" * 64 + review_path = Path(directory) / "stale-grid.json" + review_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall", + "sample": stale_identity, + "windows": [], + "missing_oocytes": [], + } + ) + ) + + with self.assertRaisesRegex( + ValueError, + "recall_window_geometry_sha256", + ): + analyze_recall_review( + fixture.sample_dir, + review_path, + Path(directory) / "analysis", + ) + + def test_http_window_route_accepts_survey_radius(self): + with tempfile.TemporaryDirectory() as directory: + fixture = RecallReviewFixture(Path(directory)) + fixture.generate() + with RecallReviewRuntime(fixture.sample_dir) as runtime: + server = ThreadingHTTPServer(("127.0.0.1", 0), _handler_for(runtime)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + base = f"http://127.0.0.1:{server.server_port}" + try: + payload = json.loads( + urlopen( + base + "/api/window?x=380&y=380&radius=1280", + timeout=3, + ).read() + ) + self.assertEqual( + payload["bbox"], + {"x0": 0, "y0": 0, "x1": 768, "y1": 768}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=3) + + def test_runtime_rejects_metadata_when_grid_changes_without_new_identity(self): + with tempfile.TemporaryDirectory() as directory: + fixture = RecallReviewFixture(Path(directory)) + bundle = fixture.generate() + metadata = json.loads(bundle.metadata_path.read_text()) + metadata["windows"][0]["center_x"] += 1 + bundle.metadata_path.write_text(json.dumps(metadata)) + + with self.assertRaisesRegex(ValueError, "coverage identity"): + RecallReviewRuntime(fixture.sample_dir) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_recall_review_batch.py b/tests/oocyte/test_recall_review_batch.py new file mode 100644 index 0000000..f5ced4d --- /dev/null +++ b/tests/oocyte/test_recall_review_batch.py @@ -0,0 +1,130 @@ +import json +import tempfile +import threading +import unittest +from contextlib import ExitStack +from http.server import ThreadingHTTPServer +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import urlopen + +from aegle.oocyte.recall_review import RecallReviewRuntime +from aegle.oocyte.recall_review_batch import ( + _batch_handler_for, + generate_batch_recall_review_bundle, +) +from tests.oocyte.test_recall_review import RecallReviewFixture + + +class TestBatchRecallReview(unittest.TestCase): + def test_generates_sample_consoles_and_identity_isolated_batch_routes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = RecallReviewFixture(root, "sample-a") + second = RecallReviewFixture(root, "sample-b") + for fixture in (first, second): + (fixture.sample_dir / "oocytes.html").write_text( + f"{fixture.sample_id} precision" + ) + (root / "oocyte_review_index.html").write_text("precision index") + (root / "oocyte_detection_algorithm.html").write_text("algorithm") + + bundle = generate_batch_recall_review_bundle( + root, + sample_ids=["sample-a", "sample-b"], + ) + + self.assertEqual(bundle.sample_ids, ("sample-a", "sample-b")) + self.assertEqual(bundle.total_candidate_count, 2) + self.assertTrue(bundle.index_path.is_file()) + self.assertTrue(bundle.manifest_path.is_file()) + index = bundle.index_path.read_text() + self.assertIn("sample-a/review_console.html", index) + self.assertIn("sample-b/recall_review.html", index) + self.assertIn("Review workflow", index) + self.assertIn("Open console", index) + self.assertIn("notes/oocytes_detection/reviews/", index) + manifest = json.loads(bundle.manifest_path.read_text()) + self.assertEqual(manifest["sample_count"], 2) + self.assertEqual( + [row["sample_id"] for row in manifest["samples"]], + ["sample-a", "sample-b"], + ) + for fixture in (first, second): + console = fixture.sample_dir / "review_console.html" + self.assertTrue(console.is_file()) + self.assertIn(fixture.sample_id, console.read_text()) + + with ExitStack() as stack: + runtimes = { + sample_id: stack.enter_context( + RecallReviewRuntime(root / sample_id) + ) + for sample_id in bundle.sample_ids + } + server = ThreadingHTTPServer( + ("127.0.0.1", 0), + _batch_handler_for(runtimes, bundle), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + base = f"http://127.0.0.1:{server.server_port}" + try: + batch_health = json.loads( + urlopen(base + "/health", timeout=3).read() + ) + self.assertEqual( + batch_health["sample_ids"], ["sample-a", "sample-b"] + ) + root_page = urlopen(base + "/", timeout=3).read().decode() + self.assertIn("Oocyte review consoles", root_page) + with urlopen(base + "/sample-a", timeout=3) as response: + self.assertTrue(response.geturl().endswith("/sample-a/")) + self.assertIn( + "sample-a oocyte review console", + response.read().decode(), + ) + sample_health = json.loads( + urlopen(base + "/sample-b/health", timeout=3).read() + ) + self.assertEqual(sample_health["sample_id"], "sample-b") + metadata = json.loads( + urlopen(base + "/sample-a/api/metadata", timeout=3).read() + ) + self.assertEqual(metadata["sample_id"], "sample-a") + patch = urlopen( + base + + "/sample-b/api/patch.webp?x=380&y=380&radius=128", + timeout=3, + ).read() + self.assertGreater(len(patch), 1000) + precision = urlopen( + base + "/sample-a/oocytes.html", timeout=3 + ).read() + self.assertIn(b"sample-a precision", precision) + with self.assertRaises(HTTPError) as context: + urlopen(base + "/not-a-sample/api/metadata", timeout=3) + self.assertEqual(context.exception.code, 404) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=3) + + def test_rejects_duplicate_and_unsafe_sample_ids(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + RecallReviewFixture(root, "sample-a") + with self.assertRaisesRegex(ValueError, "must be unique"): + generate_batch_recall_review_bundle( + root, + sample_ids=["sample-a", "sample-a"], + ) + with self.assertRaisesRegex(ValueError, "invalid batch sample ID"): + generate_batch_recall_review_bundle( + root, + sample_ids=["../sample-a"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_release.py b/tests/oocyte/test_release.py new file mode 100644 index 0000000..a321fc8 --- /dev/null +++ b/tests/oocyte/test_release.py @@ -0,0 +1,306 @@ +import base64 +import json +import re +import tempfile +import unittest +from io import BytesIO +from pathlib import Path + +import numpy as np +import pandas as pd +import tifffile +import yaml +from PIL import Image + +from aegle.oocyte.profiling import profile_oocyte_labels +from aegle.oocyte.release import build_oocyte_release, validate_oocyte_release + + +MAPPING_COLUMNS = [ + "label", + "detector_component_id", + "detector_score", + "acceptance_mode", + "center_x", + "center_y", + "bbox_x0", + "bbox_y0", + "bbox_x1", + "bbox_y1", + "mask_path", + "assigned_pixel_count", + "overlap_pixel_count", +] + + +def _write_antibodies(path: Path) -> None: + pd.DataFrame( + [ + {"version": 2, "channel_id": "Channel:0:0", "antibody_name": "DAPI"}, + {"version": 2, "channel_id": "Channel:0:1", "antibody_name": "UCHL1"}, + ] + ).to_csv(path, sep="\t", index=False) + + +def _mapping_row(labels: np.ndarray) -> dict: + yy, xx = np.nonzero(labels == 1) + return { + "label": 1, + "detector_component_id": "det_0001", + "detector_score": 0.9, + "acceptance_mode": "reviewed", + "center_x": float(xx.mean()), + "center_y": float(yy.mean()), + "bbox_x0": int(xx.min()), + "bbox_y0": int(yy.min()), + "bbox_x1": int(xx.max()) + 1, + "bbox_y1": int(yy.max()) + 1, + "mask_path": "masks/det_0001.npz", + "assigned_pixel_count": int(len(xx)), + "overlap_pixel_count": 0, + } + + +def _write_sample(root: Path, sample_id: str, *, positive: bool) -> dict: + sample_root = root / sample_id + sample_root.mkdir() + raw = np.zeros((2, 32, 40), dtype=np.uint16) + raw[0] = 10 + raw[1] = 20 + labels = np.zeros((32, 40), dtype=np.uint16) + if positive: + labels[8:18, 12:25] = 1 + raw[1, labels == 1] = 2000 + image_path = sample_root / "raw.ome.tiff" + labels_path = sample_root / "labels.ome.tiff" + mapping_path = sample_root / "mapping.csv" + candidates_path = sample_root / "candidates.csv" + antibodies_path = sample_root / "antibodies.tsv" + tifffile.imwrite(image_path, raw, ome=True, metadata={"axes": "CYX"}) + tifffile.imwrite(labels_path, labels, ome=True, metadata={"axes": "YX"}) + _write_antibodies(antibodies_path) + rows = [_mapping_row(labels)] if positive else [] + pd.DataFrame(rows, columns=MAPPING_COLUMNS).to_csv(mapping_path, index=False) + pd.DataFrame(rows, columns=MAPPING_COLUMNS).to_csv(candidates_path, index=False) + if positive: + masks_dir = sample_root / "masks" + masks_dir.mkdir() + row = rows[0] + x0, y0, x1, y1 = ( + int(row[name]) + for name in ("bbox_x0", "bbox_y0", "bbox_x1", "bbox_y1") + ) + np.savez_compressed( + masks_dir / "det_0001.npz", + mask=labels[y0:y1, x0:x1] == 1, + bbox_xyxy=np.asarray((x0, y0, x1, y1), dtype=np.int64), + image_shape_yx=np.asarray(labels.shape, dtype=np.int64), + metadata_json=np.asarray("{}"), + ) + profiling_dir = sample_root / "profiling" + profile_oocyte_labels( + sample_id=sample_id, + image_path=image_path, + antibodies_path=antibodies_path, + label_path=labels_path, + mapping_path=mapping_path, + out_dir=profiling_dir, + pixel_size_um=0.5, + label_scan_height_px=8, + ) + review_path = sample_root / "review.json" + provenance_path = sample_root / "provenance.json" + review_path.write_text(json.dumps({"sample_id": sample_id}) + "\n") + provenance_path.write_text(json.dumps({"source": "test"}) + "\n") + entry = { + "sample_id": sample_id, + "role": "positive" if positive else "negative_control", + "image": str(image_path), + "antibodies": str(antibodies_path), + "final_labels": str(labels_path), + "final_mapping": str(mapping_path), + "final_candidates": str(candidates_path), + "profiling_dir": str(profiling_dir), + "review_exports": [str(review_path)], + "provenance_files": [str(provenance_path)], + } + if not positive: + detector_path = sample_root / "detector.csv" + rescue_path = sample_root / "rescue.csv" + pd.DataFrame([{"accepted": False}]).to_csv(detector_path, index=False) + pd.DataFrame([{"rescue_status": "failed_score"}]).to_csv( + rescue_path, + index=False, + ) + entry["detector_candidates"] = str(detector_path) + entry["rescue_diagnostics"] = str(rescue_path) + return entry + + +class TestOocyteRelease(unittest.TestCase): + def test_builds_and_validates_positive_and_negative_control(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + algorithm_path = root / "algorithm.html" + algorithm_path.write_text("algorithm\n") + spec_path = root / "release.yaml" + spec_path.write_text( + yaml.safe_dump( + { + "release_name": "synthetic_v1", + "algorithm_document": str(algorithm_path), + "samples": [ + _write_sample(root, "positive", positive=True), + _write_sample(root, "negative", positive=False), + ], + }, + sort_keys=False, + ) + ) + release_dir = root / "release" + + result = build_oocyte_release(spec_path, release_dir) + + self.assertEqual(result.sample_count, 2) + self.assertEqual(result.oocyte_count, 1) + self.assertEqual(result.validation["negative_control_count"], 1) + self.assertTrue((release_dir / "release_spec.yaml").is_file()) + self.assertTrue( + ( + release_dir + / "samples/positive/final/masks/mask-0001__det_0001.npz" + ).is_file() + ) + negative = pd.read_csv( + release_dir / "samples/negative/final/oocyte_labels.csv" + ) + self.assertTrue(negative.empty) + batch = pd.read_csv(release_dir / "batch_oocyte_by_marker.csv") + self.assertEqual(batch["oocyte_id"].tolist(), ["positive__det_0001"]) + positive_console = ( + release_dir / "samples/positive/review_console.html" + ).read_text() + negative_console = ( + release_dir / "samples/negative/review_console.html" + ).read_text() + self.assertEqual( + len( + re.findall( + r"]*\bdata-embedded-review-card(?:\s|>)", + positive_console, + ) + ), + 1, + ) + encoded_images = re.findall( + r"data:image/webp;base64,([A-Za-z0-9+/=]+)", + positive_console, + ) + self.assertEqual(len(encoded_images), 4) + image_sizes = [] + for encoded in encoded_images: + with Image.open(BytesIO(base64.b64decode(encoded))) as image: + self.assertEqual(image.format, "WEBP") + image_sizes.append(image.size) + self.assertEqual(image_sizes.count((360, 360)), 2) + self.assertEqual(image_sizes.count((3, 2)), 2) + self.assertEqual(positive_console.count("data-global-overview"), 1) + self.assertEqual( + len( + re.findall( + r"]*\bdata-global-hotspot(?:\s|>)", + positive_console, + ) + ), + 1, + ) + self.assertIn('href="#oocyte-001"', positive_console) + self.assertIn('id="oocyte-001"', positive_console) + self.assertIn('data-card-target="oocyte-001"', positive_console) + self.assertIn("best>Number(nearest.getAttribute('r'))**2", positive_console) + self.assertIn( + "hotspots.toggleAttribute('hidden',showRaw)", + positive_console, + ) + self.assertIn("Hide mask", positive_console) + self.assertIn("Hide masks", positive_console) + self.assertIn(".image-frame img[hidden]{display:none}", positive_console) + self.assertIn("profiling/oocyte_by_marker.csv", positive_console) + self.assertNotIn("/api/", positive_console) + self.assertIn("No final oocytes", negative_console) + self.assertIn("validated no-oocyte negative control", negative_console) + negative_images = re.findall( + r"data:image/webp;base64,([A-Za-z0-9+/=]+)", + negative_console, + ) + self.assertEqual(len(negative_images), 1) + with Image.open(BytesIO(base64.b64decode(negative_images[0]))) as image: + self.assertEqual(image.format, "WEBP") + self.assertEqual(image.size, (3, 2)) + self.assertEqual(negative_console.count("data-global-overview"), 1) + self.assertEqual( + len( + re.findall( + r"]*\bdata-global-hotspot(?:\s|>)", + negative_console, + ) + ), + 0, + ) + self.assertIn("No final masks in this negative control", negative_console) + positive_manifest = json.loads( + ( + release_dir + / "samples/positive/sample_release_manifest.json" + ).read_text() + ) + negative_manifest = json.loads( + ( + release_dir + / "samples/negative/sample_release_manifest.json" + ).read_text() + ) + self.assertEqual( + positive_manifest["embedded_console"]["overview_webp_count"], + 2, + ) + self.assertEqual( + positive_manifest["embedded_console"]["global_hotspot_count"], + 1, + ) + self.assertEqual( + negative_manifest["embedded_console"]["overview_webp_count"], + 1, + ) + self.assertEqual(validate_oocyte_release(release_dir)["status"], "valid") + with self.assertRaises(FileExistsError): + build_oocyte_release(spec_path, release_dir) + + readme = release_dir / "README.md" + readme.write_text(readme.read_text() + "tampered\n") + with self.assertRaisesRegex(ValueError, "artifact mismatch"): + validate_oocyte_release(release_dir) + + def test_rejects_nonzero_negative_control_detector(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + entry = _write_sample(root, "negative", positive=False) + pd.DataFrame([{"accepted": True}]).to_csv( + entry["detector_candidates"], + index=False, + ) + spec_path = root / "release.yaml" + spec_path.write_text( + yaml.safe_dump( + {"release_name": "invalid_v1", "samples": [entry]}, + sort_keys=False, + ) + ) + + with self.assertRaisesRegex(ValueError, "accepted detector objects"): + build_oocyte_release(spec_path, root / "release") + self.assertFalse((root / "release").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/oocyte/test_report.py b/tests/oocyte/test_report.py new file mode 100644 index 0000000..8487dcb --- /dev/null +++ b/tests/oocyte/test_report.py @@ -0,0 +1,180 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import numpy as np +import pandas as pd +import tifffile + +from aegle.oocyte import DONOR13_V6 +from aegle.oocyte.io import save_candidate_mask +from aegle.oocyte.models import BoundingBox, SegmentationMetrics +from aegle.oocyte.report import ( + _raw_mask_thumbnail, + algorithm_document_html, + generate_html_reports, +) + + +class TestHtmlReport(unittest.TestCase): + def test_algorithm_document_contains_inline_diagrams_and_contract(self): + document = algorithm_document_html() + self.assertGreaterEqual(document.count("= 45) & (distance <= 85) & (angle < 0.75) + patch[target] = 3500 + patch[fragment] = 7000 + patch[sector] = 5000 + image[0, 60:261, 60:261] = patch + tifffile.imwrite( + fixture.image_path, + image, + ome=True, + metadata={"axes": "CYX"}, + ) + bundle = fixture.generate() + metadata = json.loads(bundle.metadata_path.read_text()) + recall_path = root / "recall.json" + recall_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "oocyte_recall", + "sample": metadata["review_identity"], + "windows": [], + "missing_oocytes": [ + { + "annotation_id": "shape-1", + "window_id": metadata["windows"][0]["window_id"], + "x": 160, + "y": 160, + "notes": "", + } + ], + } + ) + ) + analysis_dir = root / "analysis" + analyze_recall_review(fixture.sample_dir, recall_path, analysis_dir) + page = (analysis_dir / "manual_seed_review.html").read_text() + match = re.search( + r'', + page, + flags=re.DOTALL, + ) + if match is None: + raise AssertionError("manual-seed review page is missing embedded data") + embedded = json.loads(match.group(1)) + embedded["rows"][0]["manual_mask_choice"] = "accept_manual_expanded" + embedded["rows"][0]["manual_notes"] = "boundary is incomplete" + manual_path = root / "manual.json" + manual_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "manual_seed_mask_review", + "identity": embedded["identity"], + "exported_at": "2026-07-11T12:00:00.000Z", + "rows": embedded["rows"], + } + ) + ) + result = generate_shape_recovery_review( + fixture.sample_dir, + recall_path, + manual_path, + root / "shape-review", + ) + return fixture, analysis_dir, manual_path, result + + +class TestShapeRecoveryReview(unittest.TestCase): + def test_generates_only_masks_changed_by_shape_gate(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _, _, _, result = _prepare_shape_case(root) + + self.assertEqual(result.card_count, 1) + table = pd.read_csv(result.candidates_path) + self.assertEqual(table.loc[0, "current_percentile"], 95.0) + self.assertEqual(table.loc[0, "shape_recovery_percentile"], 85.0) + self.assertGreater(table.loc[0, "shape_to_current_area_ratio"], 4.0) + page = result.page_path.read_text() + self.assertIn("Use recovery", page) + self.assertIn("Keep v4", page) + self.assertTrue((result.assets_dir / "shape-001.webp").is_file()) + + def test_finalizes_shape_replacement_without_modifying_v1(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixture, analysis_dir, manual_path, shape_pack = _prepare_shape_case(root) + base = finalize_manual_seed_review( + fixture.sample_dir, + manual_path, + root / "finalized-v1", + analysis_dir=analysis_dir, + tile_shape_yx=(16, 16), + ) + base_manifest_before = base.manifest_path.read_bytes() + base_mask_before = (base.out_dir / "reviewed_masks/manual_seed_001.npz").read_bytes() + page = shape_pack.page_path.read_text() + match = re.search( + r'', + page, + flags=re.DOTALL, + ) + self.assertIsNotNone(match) + embedded = json.loads(match.group(1)) + embedded["rows"][0]["shape_review_choice"] = "accept_shape_recovery" + embedded["rows"][0]["shape_review_notes"] = "approved" + shape_review_path = root / "shape-review.json" + shape_review_path.write_text( + json.dumps( + { + "schema_version": 1, + "review_type": "manual_seed_shape_recovery_review", + "identity": embedded["identity"], + "exported_at": "2026-07-12T12:00:00.000Z", + "rows": embedded["rows"], + } + ) + ) + + result = finalize_shape_recovery_review( + fixture.sample_dir, + shape_review_path, + base.out_dir, + root / "finalized-v2", + tile_shape_yx=(16, 16), + ) + + self.assertEqual(result.accepted_count, 1) + self.assertEqual(result.boundary_warning_count, 0) + self.assertEqual(result.delta_labels.label_count, 1) + self.assertEqual(result.combined_labels.label_count, 2) + manifest = json.loads(result.manifest_path.read_text()) + self.assertEqual(manifest["shape_replacement_count"], 1) + self.assertEqual(manifest["shape_addition_count"], 0) + self.assertEqual(manifest["combined_label_count"], 2) + self.assertFalse(manifest["base_v1_outputs_modified"]) + decisions = pd.read_csv(result.decisions_path) + self.assertEqual(decisions.loc[0, "final_source"], "shape_recovery") + self.assertEqual(base_manifest_before, base.manifest_path.read_bytes()) + self.assertEqual( + base_mask_before, + (base.out_dir / "reviewed_masks/manual_seed_001.npz").read_bytes(), + ) + + +if __name__ == "__main__": + unittest.main()