From e5cf47ec38e173cce1ccef85d964a2741bd44727 Mon Sep 17 00:00:00 2001 From: Caio Mescouto Date: Mon, 19 Oct 2026 11:37:43 +0200 Subject: [PATCH 1/2] Add study-metadata service to Session Implements a new metadata capability following the port/adapter architecture: - StudyMetadataConfig (core/models/metadata_dto.py): config DTO with section gates; from_metadata_layout() stub reserves the future peh_model MetadataLayout wire. - MetadataInterface (core/interfaces/metadata.py): generic tree-building orchestration (link keys, variable catalogue, datasets, provenance) over the session cache; abstract profile_column / map_data_type primitives implemented by the adapter. - DataFrameMetadataAdapter (adapters/metadata/): new "metadata" adapter label; non-identifying column profile (counts only) and wire-format dtype mapping. - Session.create_study_metadata(...): public facade method. - Population resolved from Study.study_entity_id_list via StudyEntity (StudyPopulation degrades to StudyEntity in the peh model). - LOD/LOQ control variables excluded by default; derivation lineage (was_derived_from) walked to include enriched/aggregated observations. - Tests: interface unit, adapter unit, Session-level, and an integration test comparing against the notebooks/example golden tree (peh_demo workflow). - Docs: session-reference.md (new method + StudyMetadataConfig), adapters.md (metadata area). --- Makefile | 1 + docs/adapters.md | 1 + docs/session-reference.md | 81 +- src/pypeh/adapters/metadata/__init__.py | 0 .../adapters/metadata/dataframe_adapter.py | 29 + src/pypeh/core/interfaces/metadata.py | 787 +++++++++++++++++ src/pypeh/core/models/metadata_dto.py | 65 ++ src/pypeh/core/session/session.py | 43 + tests/adapters/metadata/__init__.py | 0 .../metadata/test_dataframe_adapter.py | 39 + tests/core/interfaces/metadata/__init__.py | 0 .../core/interfaces/metadata/test_metadata.py | 713 +++++++++++++++ tests/core/session/test_session_metadata.py | 172 ++++ tests/integration/metadata/__init__.py | 0 .../integration/metadata/input/peh_demo.yaml | 821 ++++++++++++++++++ .../metadata/input/peh_demo_data.xlsx | Bin 0 -> 6152 bytes .../metadata/test_study_metadata.py | 595 +++++++++++++ 17 files changed, 3346 insertions(+), 1 deletion(-) create mode 100644 src/pypeh/adapters/metadata/__init__.py create mode 100644 src/pypeh/adapters/metadata/dataframe_adapter.py create mode 100644 src/pypeh/core/interfaces/metadata.py create mode 100644 src/pypeh/core/models/metadata_dto.py create mode 100644 tests/adapters/metadata/__init__.py create mode 100644 tests/adapters/metadata/test_dataframe_adapter.py create mode 100644 tests/core/interfaces/metadata/__init__.py create mode 100644 tests/core/interfaces/metadata/test_metadata.py create mode 100644 tests/core/session/test_session_metadata.py create mode 100644 tests/integration/metadata/__init__.py create mode 100644 tests/integration/metadata/input/peh_demo.yaml create mode 100644 tests/integration/metadata/input/peh_demo_data.xlsx create mode 100644 tests/integration/metadata/test_study_metadata.py diff --git a/Makefile b/Makefile index 73e1e220..2df3a9c1 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ test-dataframe: uv pip install -e ".[dataframe-adapter, test-core, test-dataframe]" uv run pytest -s -vv tests/adapters tests/core/interfaces -m dataframe --disable-warnings uv run pytest -s -vv tests/core -m dataframe --disable-warnings + uv run pytest -s -vv tests/integration/metadata -m dataframe --disable-warnings test-end_to_end: uv pip install -e ".[dataframe-adapter, export-adapter, test-core]" diff --git a/docs/adapters.md b/docs/adapters.md index d3b6b76c..c90a2b3c 100644 --- a/docs/adapters.md +++ b/docs/adapters.md @@ -9,6 +9,7 @@ Current adapter areas include: - extract - aggregation - dataops +- metadata The `Session` can use default adapters where available, or you can register a custom adapter with `session.register_adapter(...)`. Registered adapters only diff --git a/docs/session-reference.md b/docs/session-reference.md index b14d39a7..f2291b4b 100644 --- a/docs/session-reference.md +++ b/docs/session-reference.md @@ -259,6 +259,85 @@ build_validation_config( Build validation configuration objects for sections in a PEH `DataLayout`. +## Study Metadata + +```python +create_study_metadata( + dataset_series: DatasetSeries, + data_import_config: DataImportConfig, + metadata_config: StudyMetadataConfig | None = None, + validation_report: ValidationErrorReportCollection | None = None, + adapter_label: str = "metadata", +) -> dict +``` + +Build the study-metadata tree for an imported `DatasetSeries`. Starting from +the `DataImportConfig`, the service walks the PEH resources in the session +cache and combines them with the imported series into a nested dictionary: + +- `study` — resolved studies with their `population` entries + (`Study.study_entity_id_list` → `StudyEntity`; id / label / definition). + Note that a `StudyPopulation` currently loads as a base `StudyEntity` + (the peh model has no `study_populations` slot on `EntityList`), so its + discriminating fields (`research_population_type`, member lists) are not + representable yet. +- `projects` — projects linked to those studies. +- `observation_groups` — groups containing the configured observations or + anything derived from them (the enrichment / aggregation lineage). +- `observations` — the variable catalogue: every observable property of the + reachable observations (the configured ones first, then the derived ones + in dependency order: each derived observation follows the observation it + was derived from) with its + `kind` (measurement / calculation), value type, unit, categorical / + multivalued flags, value options and group labels. Indicator measurements + carry their analyte chain under `semantics` (indicator → matrix / + biochemical entity, with CHEBI `exact_matches`, quantity kind and + constraints); calculation variables carry their `calculation` (function, + input links to source variables, results). LOD / LOQ `control` properties + are excluded by default. +- `datasets` — the series' datasets with their variables, wire-format data + types (`STRING` / `INTEGER` / `FLOAT` / ...) and primary keys. Data + profiling (rows / nulls / distinct counts) is deliberately out of scope + here: per-column statistics can de-anonymize individuals in small + cohorts. Profiling will later be produced by the aggregation service and + passed in by the caller. +- `provenance` — series label, config / layout ids, dataset sections, the + derived-observation chain, the pypeh and peh_model versions (`None` when + the distribution is not installed), and an optionally attached + `ValidationErrorReportCollection` in its JSON form. Validation is never + run here; the report is accepted from the caller. + +Sections are gated through `StudyMetadataConfig` (`include_link_keys`, +`include_population`, `include_variable_catalogue`, `include_datasets`, +`include_provenance`, `exclude_control_variables`); +disabled sections are omitted from the tree entirely. Caller-provided +`extra_provenance` fields (e.g. `source_file`) are appended to the +provenance section; a key that collides with a built-in provenance field +raises `ValueError`. When no study can be +resolved for the config, the link-keys sections are empty and a warning +is logged. Observations or observation designs missing from the cache are +left out with a warning. A `ValueError` is raised when the series holds +observations that are neither configured by nor derived from the +`DataImportConfig`. + +The returned tree holds builtin types only (`dict` / `list` / `str` / +numbers / `bool` / `None`) and shares no objects between branches, so it +serializes directly with `json.dumps` or `yaml.safe_dump`. + +```python +class StudyMetadataConfig: + include_link_keys: bool = True + include_population: bool = True + include_variable_catalogue: bool = True + include_datasets: bool = True + include_provenance: bool = True + exclude_control_variables: bool = True + extra_provenance: dict = {} +``` + +Configuration DTO for `create_study_metadata`, defined in +`pypeh.core.models.metadata_dto`. + ## Adapter Methods ```python @@ -266,7 +345,7 @@ register_default_adapter(interface_functionality: str) ``` Register and return the default adapter class for `validation`, `dataops`, -`enrichment`, `extract`, or `aggregation`. +`enrichment`, `extract`, `aggregation`, or `metadata`. ```python register_adapter(interface_functionality: str, adapter) -> None diff --git a/src/pypeh/adapters/metadata/__init__.py b/src/pypeh/adapters/metadata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pypeh/adapters/metadata/dataframe_adapter.py b/src/pypeh/adapters/metadata/dataframe_adapter.py new file mode 100644 index 00000000..4e67fa6d --- /dev/null +++ b/src/pypeh/adapters/metadata/dataframe_adapter.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import logging + +from polars import DataFrame +from typing import TYPE_CHECKING + +from pypeh.adapters.dataops.dataframe_adapter import DataFrameAdapter +from pypeh.core.interfaces.metadata import MetadataInterface +from pypeh.core.models.constants import ObservablePropertyValueType + +if TYPE_CHECKING: + from pypeh.core.models.internal_data_layout import DatasetSchemaElement + +logger = logging.getLogger(__name__) + + +class DataFrameMetadataAdapter(DataFrameAdapter, MetadataInterface[DataFrame]): + data_format = DataFrame + + def map_data_type(self, schema_element: DatasetSchemaElement) -> str: + """ + Map the schema element's declared value type to the + wire-format vocabulary (e.g. "STRING", "INTEGER", "FLOAT"). + """ + data_type = schema_element.data_type + if data_type is None: + return "UNKNOWN" + return ObservablePropertyValueType(data_type).name diff --git a/src/pypeh/core/interfaces/metadata.py b/src/pypeh/core/interfaces/metadata.py new file mode 100644 index 00000000..d0388a38 --- /dev/null +++ b/src/pypeh/core/interfaces/metadata.py @@ -0,0 +1,787 @@ +""" +The metadata interface provides the study-metadata service: given an +imported `DatasetSeries` and its `DataImportConfig`, it builds a +nested metadata tree (link keys, variable catalogue, datasets, +provenance) by walking the PEH resources in the session cache. + +Usage: + Session.create_study_metadata(...) resolves this interface through + the registered "metadata" adapter. + +""" + +from __future__ import annotations + +import importlib +import importlib.metadata +import logging + +from abc import abstractmethod +from peh_model import peh +from typing import TYPE_CHECKING, Any, Generic + +from pypeh.core.cache.containers import CacheContainerView +from pypeh.core.interfaces.dataops import DataOpsInterface +from pypeh.core.models import metadata_dto +from pypeh.core.models.internal_data_layout import ( + DatasetSchemaElement, + DatasetSeries, +) +from pypeh.core.models.typing import T_DataType + +if TYPE_CHECKING: + from pypeh.core.models.validation_errors import ( + ValidationErrorReportCollection, + ) + +logger = logging.getLogger(__name__) + + +def _as_list(value: Any) -> list: + if value is None: + return [] + if isinstance(value, (list, tuple, set)): + return list(value) + return [value] + + +def _as_text(value: Any) -> str | None: + if value is None: + return None + text = getattr(value, "text", None) + return text if isinstance(text, str) else str(value) + + +def _clean_text(value: Any) -> str | None: + text = _as_text(value) + return text.strip() if isinstance(text, str) else None + + +def _as_number(value: Any, digits: int = 3) -> Any: + if value is None: + return None + try: + return round(float(value), digits) + except (TypeError, ValueError): + return _as_text(value) + + +def _package_version(distribution: str) -> str | None: + try: + return importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + return None + + +def _to_plain(value: Any) -> Any: + """ + Rebuild the tree from builtin types only: `str` subclasses (peh + ids, str enums) become `str`, and every container is copied, so + no two branches share an object. The result serializes cleanly to + JSON / YAML (no python tags, no anchors / aliases). + """ + if isinstance(value, dict): + return {_to_plain(key): _to_plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_to_plain(item) for item in value] + if isinstance(value, str): + return str.__str__(value) + return value + + +class MetadataInterface(DataOpsInterface, Generic[T_DataType]): + """ + Builds the study-metadata tree for an imported DatasetSeries. + + The orchestration (cache traversal, tree assembly, section gating + via `StudyMetadataConfig`) lives on this interface, written + generically over `T_DataType`. Adapters implement only the one + dataframe-dependent primitive: `map_data_type`. + """ + + @abstractmethod + def map_data_type(self, schema_element: DatasetSchemaElement) -> str: + """ + Map a dataset schema element to its wire-format data type + label (e.g. "STRING", "INTEGER", "FLOAT"). + """ + raise NotImplementedError( + "Abstract method on class MetadataInterface was called " + "without supporting implementation." + ) + + @classmethod + def get_default_adapter_class(cls): + try: + adapter_module = importlib.import_module( + "pypeh.adapters.metadata.dataframe_adapter" + ) + adapter_class = getattr(adapter_module, "DataFrameMetadataAdapter") + except Exception as e: + logger.error( + "Exception encountered while attempting to import a " + "Polars-based DataFrameMetadataAdapter" + ) + raise e + return adapter_class + + #### ORCHESTRATION #### + + def build_study_metadata( + self, + dataset_series: DatasetSeries[T_DataType], + data_import_config: peh.DataImportConfig, + metadata_config: metadata_dto.StudyMetadataConfig, + cache_view: CacheContainerView, + validation_report: "ValidationErrorReportCollection | None" = None, + ) -> dict[str, Any]: + """ + Build the study-metadata tree. + + Sections gated off in `metadata_config` are omitted from the + returned tree entirely, so a present key means "requested". + The tree holds builtin types only (dict / list / str / numbers + / bool / None). + """ + reachable = self._reachable_observations( + cache_view, data_import_config + ) + lineage = set(reachable) + self._check_series_matches_config( + dataset_series, data_import_config, lineage + ) + designs = self._observation_designs(cache_view, reachable) + control_ids = ( + self._collect_control_ids(cache_view, designs) + if metadata_config.exclude_control_variables + else set() + ) + tree: dict[str, Any] = {} + + if metadata_config.include_link_keys: + groups, studies, projects = self._resolve_link_keys( + cache_view, lineage + ) + if not studies: + logger.warning( + "No Study could be resolved for data import config " + f"{data_import_config.id!r}; the link-keys sections " + "will be empty." + ) + tree["study"] = [ + self._study_entry(cache_view, study, metadata_config) + for study in studies + ] + tree["projects"] = [ + {"id": project.id, "label": self._label_of(project)} + for project in projects + ] + tree["observation_groups"] = [ + {"id": group.id, "label": self._label_of(group)} + for group in groups + ] + + structure_by_op: dict[str, dict] = {} + if metadata_config.include_variable_catalogue: + tree["observations"] = self._catalogue_section( + cache_view, + designs, + control_ids, + structure_by_op, + ) + + if metadata_config.include_datasets: + tree["datasets"] = self._datasets_section( + dataset_series, + structure_by_op, + control_ids, + ) + + if metadata_config.include_provenance: + tree["provenance"] = self._provenance_section( + dataset_series=dataset_series, + data_import_config=data_import_config, + cache_view=cache_view, + metadata_config=metadata_config, + lineage=lineage, + validation_report=validation_report, + ) + + return _to_plain(tree) + + #### LINK KEYS (study / population / projects / groups) #### + + @staticmethod + def _config_observations( + data_import_config: peh.DataImportConfig, + ) -> list[str]: + observation_ids: list[str] = [] + section_mapping = data_import_config.section_mapping + if section_mapping is None: + return observation_ids + for link in section_mapping.section_mapping_links or []: + for observation_id in _as_list(link.observation_id_list): + if observation_id not in observation_ids: + observation_ids.append(observation_id) + return observation_ids + + def _reachable_observations( + self, + cache_view: CacheContainerView, + data_import_config: peh.DataImportConfig, + ) -> list[str]: + """ + The configured observations plus everything derived from them + (transitively): enrichment / aggregation DerivedObservations. + + Ordered deterministically in dependency order: the configured + observations first, then a breadth-first walk of the + derivation graph, so every derived observation follows the + observation it was derived from. Siblings keep cache order. + """ + children: dict[str, list[str]] = {} + for derived in cache_view.get_all("DerivedObservation"): + derived_id = getattr(derived, "id", None) + parent_id = getattr(derived, "was_derived_from", None) + if derived_id is None or parent_id is None: + continue + children.setdefault(parent_id, []).append(derived_id) + + ordered = self._config_observations(data_import_config) + known = set(ordered) + index = 0 + while index < len(ordered): + for child_id in children.get(ordered[index], []): + if child_id not in known: + known.add(child_id) + ordered.append(child_id) + index += 1 + return ordered + + @staticmethod + def _check_series_matches_config( + dataset_series: DatasetSeries[T_DataType], + data_import_config: peh.DataImportConfig, + lineage: set[str], + ) -> None: + """ + Every observation indexed by the series must be configured by + the import config or derived from it; otherwise the tree would + silently mix two unrelated imports. + """ + unknown = sorted( + { + observation_id + for dataset in dataset_series.parts.values() + for observation_id in dataset.observation_ids + if observation_id not in lineage + } + ) + if unknown: + raise ValueError( + f"DatasetSeries {dataset_series.label!r} holds " + "observations that are neither configured by nor " + "derived from data import config " + f"{data_import_config.id!r}: {', '.join(unknown)}." + ) + + def _observation_designs( + self, + cache_view: CacheContainerView, + observation_ids: list[str], + ) -> list[tuple[str, Any, peh.ObservationDesign]]: + """ + Resolve (observation id, observation, design) for each + reachable observation. Observations or designs missing from + the cache are skipped with a warning. + """ + resolved: list[tuple[str, Any, peh.ObservationDesign]] = [] + for observation_id in observation_ids: + observation = self._get_observation(cache_view, observation_id) + if observation is None: + logger.warning( + f"Observation {observation_id!r} is not in the cache; " + "it is left out of the study metadata." + ) + continue + design_id = observation.observation_design + design = ( + cache_view.get(design_id, "ObservationDesign") + if design_id is not None + else None + ) + if design is None: + logger.warning( + f"ObservationDesign {design_id!r} of observation " + f"{observation_id!r} is not in the cache; the " + "observation is left out of the study metadata." + ) + continue + resolved.append((observation_id, observation, design)) + return resolved + + def _collect_control_ids( + self, + cache_view: CacheContainerView, + designs: list[tuple[str, Any, peh.ObservationDesign]], + ) -> set[str]: + """ + LOD / LOQ `control` observable properties are assay quality + controls, not catalogue variables. + """ + control_ids: set[str] = set() + for _, _, design in designs: + for spec in _as_list(design.observable_property_specifications): + op = self._resolve_observable_property(cache_view, spec) + if op is None: + continue + if ( + _as_text(op.observation_result_type) == "control" + and op.id not in control_ids + ): + control_ids.add(op.id) + return control_ids + + def _resolve_link_keys( + self, + cache_view: CacheContainerView, + lineage: set[str], + ) -> tuple[list, list, list]: + """ + Walk ObservationGroups -> Study -> Project from the + observation lineage (grouping path of the linking backbone). + """ + groups = [ + group + for group in cache_view.get_all("ObservationGroup") + if set(_as_list(group.observation_id_list)) & lineage + ] + group_ids = {group.id for group in groups} + studies = [ + study + for study in cache_view.get_all("Study") + if set(_as_list(study.observation_group_id_list)) & group_ids + ] + study_ids = {study.id for study in studies} + linked_project_ids = { + project_id + for study in studies + for project_id in _as_list(study.project_id_list) + } + projects = [ + project + for project in cache_view.get_all("Project") + if set(_as_list(project.study_id_list)) & study_ids + or project.id in linked_project_ids + ] + return groups, studies, projects + + def _study_entry( + self, + cache_view: CacheContainerView, + study: peh.Study, + metadata_config: metadata_dto.StudyMetadataConfig, + ) -> dict[str, Any]: + entry: dict[str, Any] = { + "id": study.id, + "label": self._label_of(study), + } + if metadata_config.include_population: + entry["population"] = self._population_entries(cache_view, study) + return entry + + def _population_entries( + self, cache_view: CacheContainerView, study: peh.Study + ) -> list[dict[str, Any]]: + """ + Population entries come from `Study.study_entity_id_list`. + + Note: peh_model's EntityList has no `study_populations` slot, + so a `StudyPopulation` loads as a base `StudyEntity` and its + discriminating fields (`research_population_type`, member + lists) are not representable today. Entries therefore carry + id / label / definition only. + """ + entries: list[dict[str, Any]] = [] + for entity_id in _as_list(study.study_entity_id_list): + entity = cache_view.get(entity_id, "StudyEntity") + entries.append( + { + "id": entity_id, + "label": self._label_of(entity, entity_id), + "definition": _clean_text( + getattr(entity, "description", None) + ), + } + ) + return entries + + #### VARIABLE CATALOGUE #### + + def _catalogue_section( + self, + cache_view: CacheContainerView, + designs: list[tuple[str, Any, peh.ObservationDesign]], + control_ids: set[str], + structure_by_op: dict[str, dict], + ) -> list[dict[str, Any]]: + """ + Per reachable observation, its observable properties (model + grain). Each observable property is emitted under the first + observation that specifies it. + """ + seen: set[str] = set() + observations: list[dict[str, Any]] = [] + for observation_id, observation, design in designs: + properties: list[dict[str, Any]] = [] + for spec in _as_list(design.observable_property_specifications): + op = self._resolve_observable_property(cache_view, spec) + if op is None or op.id in seen: + continue + if op.id in control_ids: + continue + seen.add(op.id) + variable = self._variable_tree(cache_view, op, spec) + structure_by_op[op.id] = variable + properties.append(variable) + observations.append( + { + "id": observation_id, + "label": self._label_of(observation), + "observable_properties": properties, + } + ) + return observations + + def _variable_tree( + self, + cache_view: CacheContainerView, + op: peh.ObservableProperty, + spec: peh.ObservablePropertySpecification | None = None, + ) -> dict[str, Any]: + variable: dict[str, Any] = { + "id": op.id, + "label": getattr(op, "short_name", None) + or getattr(op, "ui_label", None), + "name": getattr(op, "name", None), + "kind": _as_text(op.observation_result_type), + "value_type": _as_text(op.value_type), + "unit": _as_text(op.unit), + "categorical": op.categorical, + "multivalued": op.multivalued, + "required": op.required, + "value_options": [ + self._option_key(option) + for option in _as_list(op.value_options) + ], + "groups": self._grouping_labels(cache_view, op.grouping_id_list), + } + semantics = self._semantics_tree(cache_view, op) + if semantics is not None: + variable["semantics"] = semantics + calculation_design = None + if spec is not None: + calculation_design = getattr(spec, "calculation_design", None) + if calculation_design is None: + calculation_design = getattr(op, "calculation_design", None) + if calculation_design is not None: + calculation = self._calculation_tree( + cache_view, calculation_design + ) + if calculation is not None: + variable["calculation"] = calculation + return variable + + @staticmethod + def _option_key(option: Any) -> str | None: + for attr in ("key", "value", "label"): + text = _as_text(getattr(option, attr, None)) + if text is not None: + return text + return None + + def _grouping_labels( + self, cache_view: CacheContainerView, grouping_id_list: Any + ) -> list[str]: + labels = [] + for group_id in _as_list(grouping_id_list): + group = cache_view.get(group_id, "Grouping") + labels.append(self._label_of(group, group_id)) + return labels + + def _semantics_tree( + self, cache_view: CacheContainerView, op: peh.ObservableProperty + ) -> dict[str, Any] | None: + """ + The analyte chain of an indicator measurement: indicator -> + matrix / biochemical entity, plus quantity kind and + constraints. Returns None when the observable property has no + resolvable indicator (nothing loaded). Only the first + resolvable indicator is emitted; further ones are logged. + """ + indicators = [ + (indicator_id, indicator) + for indicator_id in _as_list(op.has_observable_property_type) + if (indicator := cache_view.get(indicator_id, "IndicatorSubClass")) + is not None + ] + if not indicators: + return None + if len(indicators) > 1: + logger.warning( + f"ObservableProperty {op.id!r} resolves to " + f"{len(indicators)} indicators; only the first " + f"({indicators[0][0]!r}) is emitted under semantics." + ) + indicator_id, indicator = indicators[0] + matrix_id = getattr(indicator, "matrix", None) + matrix = ( + cache_view.get(matrix_id, "MatrixSubClass") if matrix_id else None + ) + biochem_id = getattr(indicator, "biochementity", None) + biochem = ( + cache_view.get(biochem_id, "BioChemEntitySubClass") + if biochem_id + else None + ) + return { + "indicator": { + "id": indicator_id, + "label": self._label_of(indicator, indicator_id), + "definition": _clean_text(indicator.description), + "indicator_type": _as_text(indicator.indicator_type), + "quantity_kind": _as_text(indicator.quantity_kind), + "exact_matches": [ + _as_text(match) + for match in _as_list(indicator.exact_matches) + ], + "relevant_observable_entity_types": [ + _as_text(entity_type) + for entity_type in _as_list( + indicator.relevant_observable_entity_types + ) + ], + "suggester": _as_text(indicator.suggester), + }, + "matrix": { + "id": matrix_id, + "label": self._label_of(matrix, matrix_id), + "definition": _clean_text( + getattr(matrix, "description", None) + ), + "exact_matches": [ + _as_text(match) + for match in _as_list( + getattr(matrix, "exact_matches", None) + ) + ], + }, + "biochem_entity": { + "id": biochem_id, + "label": self._label_of(biochem, biochem_id), + "definition": _clean_text( + getattr(biochem, "description", None) + ), + # BioChemEntitySubClass.exact_matches holds CHEBI ids + "exact_matches": [ + _as_text(match) + for match in _as_list( + getattr(biochem, "exact_matches", None) + ) + ], + "aliases": _as_list(getattr(biochem, "aliases", None)), + "molecular_weight_gram_per_mol": _as_number( + getattr(biochem, "molweight_grampermol", None) + ), + "groups": _as_list(getattr(biochem, "group_labels", None)), + "is_metabolite_of": [ + _as_text(metabolite) + for metabolite in _as_list( + getattr(biochem, "is_metabolite_of", None) + ) + ], + }, + "quantity_kind": _as_text(indicator.quantity_kind), + "constraints": [ + getattr(constraint, "constraint_id", constraint) + for constraint in _as_list(indicator.constraints) + ], + } + + def _calculation_tree( + self, + cache_view: CacheContainerView, + calculation_design: peh.CalculationDesign, + ) -> dict[str, Any] | None: + """ + Function and input/output links of a derived (calculation) + variable, making the derivation graph explicit. + """ + implementation = calculation_design.calculation_implementation + if implementation is None: + return None + inputs: list[dict[str, Any]] = [] + for kwarg in _as_list(implementation.function_kwargs): + reference = getattr(kwarg, "contextual_field_reference", None) + source_id = ( + getattr(reference, "field_label", None) + if reference is not None + else None + ) + source = ( + cache_view.get(source_id, "ObservableProperty") + if source_id + else None + ) + inputs.append( + { + "argument": getattr(kwarg, "mapping_name", None), + "source_variable": source_id, + "source_label": ( + getattr(source, "short_name", None) + or getattr(source, "ui_label", None) + ) + if source + else None, + "value": getattr(kwarg, "value", None), + } + ) + return { + "name": getattr(calculation_design, "calculation_name", None), + "function": getattr(implementation, "function_name", None), + "inputs": inputs, + "results": [ + getattr(result, "mapping_name", None) + for result in _as_list(implementation.function_results) + ], + } + + #### DATASETS #### + + def _datasets_section( + self, + dataset_series: DatasetSeries[T_DataType], + structure_by_op: dict[str, dict], + control_ids: set[str], + ) -> list[dict[str, Any]]: + datasets: list[dict[str, Any]] = [] + for dataset_label, dataset in dataset_series.parts.items(): + assert dataset.schema is not None + variables: list[dict[str, Any]] = [] + for ( + element_label, + element, + ) in dataset.schema.elements.items(): + if element.observable_property_id in control_ids: + continue + op_tree = structure_by_op.get(element.observable_property_id) + variables.append( + { + "label": element_label, + "observable_property": op_tree + if op_tree is not None + else {"id": element.observable_property_id}, + "data_type": self.map_data_type(element), + "primary_key": element_label + in dataset.schema.primary_keys, + } + ) + datasets.append( + { + "label": dataset_label, + "described_by": dataset.described_by, + "observations": sorted(dataset.observation_ids), + "variables": variables, + } + ) + return datasets + + #### PROVENANCE #### + + def _provenance_section( + self, + dataset_series: DatasetSeries[T_DataType], + data_import_config: peh.DataImportConfig, + cache_view: CacheContainerView, + metadata_config: metadata_dto.StudyMetadataConfig, + lineage: set[str], + validation_report: "ValidationErrorReportCollection | None", + ) -> dict[str, Any]: + """ + What produced this metadata: source config / layout ids, the + dataset sections, the derivation chain and the pypeh / + peh_model versions. The validation report is accepted from + the caller, never run here; it is embedded in its JSON form. + Caller-provided `extra_provenance` fields are appended but may + not override any of these built-in fields. + """ + provenance: dict[str, Any] = { + "series_label": dataset_series.label, + "data_import_config": data_import_config.id, + "data_layout": data_import_config.layout, + "sections": [ + { + "dataset": dataset.label, + "described_by": dataset.described_by, + } + for dataset in dataset_series.parts.values() + ], + "derived_observations": [ + { + "derived_observation": derived.id, + "was_derived_from": derived.was_derived_from, + } + for derived in cache_view.get_all("DerivedObservation") + if getattr(derived, "id", None) in lineage + ], + "pypeh_version": _package_version("pypeh"), + "peh_model_version": _package_version("peh_model"), + "validation_report": ( + { + observation_id: report.model_dump(mode="json") + for observation_id, report in validation_report.items() + } + if validation_report is not None + else None + ), + } + collisions = sorted( + set(metadata_config.extra_provenance) & set(provenance) + ) + if collisions: + raise ValueError( + "StudyMetadataConfig.extra_provenance may not override " + f"built-in provenance fields: {', '.join(collisions)}." + ) + provenance.update(metadata_config.extra_provenance) + return provenance + + #### HELPERS #### + + @staticmethod + def _get_observation( + cache_view: CacheContainerView, observation_id: str + ) -> Any | None: + return cache_view.get(observation_id, "Observation") or cache_view.get( + observation_id, "DerivedObservation" + ) + + @staticmethod + def _resolve_observable_property( + cache_view: CacheContainerView, spec: Any + ) -> peh.ObservableProperty | None: + reference = getattr(spec, "observable_property", None) + if reference is None: + return None + op_id = reference if isinstance(reference, str) else reference.id + return cache_view.get(op_id, "ObservableProperty") + + @staticmethod + def _label_of(entity: Any, reference: str | None = None) -> str | None: + if entity is None: + return reference + for attr in ("ui_label", "name", "short_name"): + value = getattr(entity, attr, None) + if value: + return value + return getattr(entity, "id", None) or reference diff --git a/src/pypeh/core/models/metadata_dto.py b/src/pypeh/core/models/metadata_dto.py new file mode 100644 index 00000000..d0d4c561 --- /dev/null +++ b/src/pypeh/core/models/metadata_dto.py @@ -0,0 +1,65 @@ +""" +DTOs for the study-metadata service. + +`StudyMetadataConfig` selects which sections of the study-metadata tree +are emitted and how, analogously to what `DataLayout` / `DataLayoutSection` +do for the extract service. It is a plain pypeh-internal config for now; +a future peh_model `MetadataLayout` resource can be mapped onto it via +`StudyMetadataConfig.from_metadata_layout`. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(kw_only=True) +class StudyMetadataConfig: + """ + Configuration for `Session.create_study_metadata`. + + Each `include_*` flag gates one top-level section of the emitted + metadata tree. Disabled sections are omitted from the tree entirely + (rather than emitted empty), so consumers can rely on key presence + meaning "this section was requested". + """ + + include_link_keys: bool = True + """study / projects / observation_groups sections.""" + + include_population: bool = True + """population entries under each study (requires include_link_keys).""" + + include_variable_catalogue: bool = True + """observations[].observable_properties catalogue section.""" + + include_datasets: bool = True + """datasets section (variables, dtypes, primary keys).""" + + include_provenance: bool = True + """provenance section (config/layout ids, derivation chain, versions).""" + + exclude_control_variables: bool = True + """Exclude LOD/LOQ `control` observable properties (assay QC, not + catalogue variables) from the catalogue and datasets sections.""" + + extra_provenance: dict = field(default_factory=dict) + """Caller-provided provenance fields appended to the provenance + section (e.g. `source_file`). Keys may not collide with built-in + provenance fields; a collision raises ValueError.""" + + @classmethod + def from_metadata_layout(cls, layout) -> "StudyMetadataConfig": + """ + Build a config from a peh_model `MetadataLayout` resource. + + Not implemented yet: `MetadataLayout` does not exist in the + peh_model schema. Reserved so the DTO can be populated from a + cached PEH resource once the model supports it (mirrors + `DatasetSeriesConcatenationPlan.from_alignment`). + """ + raise NotImplementedError( + "StudyMetadataConfig.from_metadata_layout is not implemented " + "yet: the peh_model schema has no `MetadataLayout` resource. " + f"layout={layout!r}." + ) diff --git a/src/pypeh/core/session/session.py b/src/pypeh/core/session/session.py index df352262..acd0e51a 100644 --- a/src/pypeh/core/session/session.py +++ b/src/pypeh/core/session/session.py @@ -42,6 +42,7 @@ ObservationAlignment, ) from pypeh.core.models.internal_data_layout import DatasetSeries, Dataset +from pypeh.core.models.metadata_dto import StudyMetadataConfig from pypeh.core.interfaces.dataops import ( AggregationInterface, DataOpsInterface, @@ -50,6 +51,7 @@ LabelCollisionStrategy, ValidationInterface, ) +from pypeh.core.interfaces.metadata import MetadataInterface from pypeh.adapters.persistence.dataset_parquet import ( dump_dataset_series_to_parquet_filesystem, load_dataset_series_from_parquet_filesystem, @@ -229,6 +231,9 @@ def register_default_adapter(self, interface_functionality: str): case "aggregation": adapter = AggregationInterface.get_default_adapter_class() self._adapter_mapping[interface_functionality] = adapter + case "metadata": + adapter = MetadataInterface.get_default_adapter_class() + self._adapter_mapping[interface_functionality] = adapter case _: raise NotImplementedError( "Session.register_default_adapter does not support " @@ -781,6 +786,44 @@ def read_tabular_dataset_series( validate_foreign_keys=validate_foreign_keys, ) + def create_study_metadata( + self, + dataset_series: DatasetSeries, + data_import_config: peh.DataImportConfig, + metadata_config: StudyMetadataConfig | None = None, + validation_report: ValidationErrorReportCollection | None = None, + adapter_label: str = "metadata", + ) -> dict: + """ + Build the study-metadata tree for an imported DatasetSeries. + + Starting from the `DataImportConfig` the service walks the PEH + resources in the session cache (observations, observation + groups, study, projects, population) and combines them with + the imported `DatasetSeries` (datasets, variables, dtypes) + into a nested dictionary: + + study / projects / observation_groups / + observations (variable catalogue) / datasets / provenance + + Sections are gated through `metadata_config` + (`StudyMetadataConfig`); disabled sections are omitted from the + tree. A `ValidationErrorReportCollection` can be attached to + the provenance section but is never produced here — validation + remains a separate step. + """ + if metadata_config is None: + metadata_config = StudyMetadataConfig() + adapter = self.get_adapter(adapter_label) + assert isinstance(adapter, MetadataInterface) + return adapter.build_study_metadata( + dataset_series=dataset_series, + data_import_config=data_import_config, + metadata_config=metadata_config, + cache_view=CacheContainerView(self.cache), + validation_report=validation_report, + ) + def get_resource( self, resource_identifier: str, resource_type: str ) -> T_NamedThingLike | None: diff --git a/tests/adapters/metadata/__init__.py b/tests/adapters/metadata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/adapters/metadata/test_dataframe_adapter.py b/tests/adapters/metadata/test_dataframe_adapter.py new file mode 100644 index 00000000..bd7fca07 --- /dev/null +++ b/tests/adapters/metadata/test_dataframe_adapter.py @@ -0,0 +1,39 @@ +import pytest + +from pypeh.core.models.constants import ObservablePropertyValueType +from pypeh.core.models.internal_data_layout import DatasetSchemaElement + +pytestmark = [pytest.mark.dataframe, pytest.mark.other] + + +@pytest.fixture +def adapter(): + from pypeh.adapters.metadata.dataframe_adapter import ( + DataFrameMetadataAdapter, + ) + + return DataFrameMetadataAdapter() + + +def _element(data_type: ObservablePropertyValueType) -> DatasetSchemaElement: + return DatasetSchemaElement( + label="chol", + observable_property_id="peh:prop_chol", + data_type=data_type, + ) + + +class TestMapDataType: + @pytest.mark.parametrize( + ("value_type", "expected"), + [ + (ObservablePropertyValueType.STRING, "STRING"), + (ObservablePropertyValueType.INTEGER, "INTEGER"), + (ObservablePropertyValueType.FLOAT, "FLOAT"), + (ObservablePropertyValueType.BOOLEAN, "BOOLEAN"), + (ObservablePropertyValueType.DATE, "DATE"), + (ObservablePropertyValueType.DATETIME, "DATETIME"), + ], + ) + def test_declared_value_types(self, adapter, value_type, expected): + assert adapter.map_data_type(_element(value_type)) == expected diff --git a/tests/core/interfaces/metadata/__init__.py b/tests/core/interfaces/metadata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/core/interfaces/metadata/test_metadata.py b/tests/core/interfaces/metadata/test_metadata.py new file mode 100644 index 00000000..ea456d77 --- /dev/null +++ b/tests/core/interfaces/metadata/test_metadata.py @@ -0,0 +1,713 @@ +import pytest +import peh_model.peh as peh + +from pypeh import Session +from pypeh.core.cache.containers import CacheContainerView +from pypeh.core.interfaces.metadata import MetadataInterface +from pypeh.core.models.constants import ObservablePropertyValueType +from pypeh.core.models.internal_data_layout import DatasetSeries +from pypeh.core.models.metadata_dto import StudyMetadataConfig + +pytestmark = [pytest.mark.dataframe, pytest.mark.interface] + + +@pytest.fixture +def metadata_adapter() -> MetadataInterface: + return MetadataInterface.get_default_adapter_class()() + + +def _populate_metadata_cache(session: Session) -> peh.DataImportConfig: + session.cache.add( + peh.ObservableProperty( + id="peh:prop_id_sample", + short_name="id_sample", + value_type="string", + required=True, + ) + ) + session.cache.add( + peh.ObservableProperty( + id="peh:prop_chol", + short_name="chol", + value_type="decimal", + unit="MicroGM-PER-L", + observation_result_type="measurement", + has_observable_property_type=["peh:indicator_chol"], + grouping_id_list=["peh:group_lipids"], + ) + ) + session.cache.add( + peh.ObservableProperty( + id="peh:prop_chol_lod", + short_name="chol_lod", + value_type="decimal", + observation_result_type="control", + ) + ) + session.cache.add( + peh.ObservableProperty( + id="peh:prop_chol_mean", + short_name="chol_mean", + value_type="decimal", + observation_result_type="calculation", + ) + ) + session.cache.add(peh.Grouping(id="peh:group_lipids", ui_label="Lipids")) + session.cache.add( + peh.IndicatorSubClass( + id="peh:indicator_chol", + ui_label="Total cholesterol", + description="Cholesterol in all lipoproteins.", + quantity_kind="http://qudt.org/vocab/quantitykind/Mass", + matrix="peh:matrix_blood", + biochementity="peh:bce_chol", + constraints=[peh.IAdoptConstraint(constraint_id="plausible")], + ) + ) + session.cache.add( + peh.MatrixSubClass( + id="peh:matrix_blood", + ui_label="Blood", + description="Whole blood.", + ) + ) + session.cache.add( + peh.BioChemEntitySubClass( + id="peh:bce_chol", + ui_label="Cholesterol", + exact_matches=["CHEBI:16288"], + aliases=["total cholesterol"], + molweight_grampermol=386.65, + group_labels=["sterol"], + ) + ) + session.cache.add( + peh.ObservationDesign( + id="peh:od_lab", + observable_property_specifications=[ + peh.ObservablePropertySpecification( + observable_property="peh:prop_id_sample" + ), + peh.ObservablePropertySpecification( + observable_property="peh:prop_chol" + ), + peh.ObservablePropertySpecification( + observable_property="peh:prop_chol_lod" + ), + ], + ) + ) + session.cache.add( + peh.ObservationDesign( + id="peh:od_summary", + observable_property_specifications=[ + peh.ObservablePropertySpecification( + observable_property="peh:prop_chol_mean", + calculation_design=peh.CalculationDesign( + calculation_name="mean_chol", + calculation_implementation=peh.CalculationImplementation( + function_name="statistics_mean", + function_kwargs=[ + peh.CalculationKeywordArgument( + contextual_field_reference=peh.ContextualFieldReference( + dataset_label="peh:obs_lab", + field_label="peh:prop_chol", + ) + ) + ], + function_results=[ + peh.CalculationResult(mapping_name="chol_mean") + ], + ), + ), + ), + ], + ) + ) + session.cache.add( + peh.Observation(id="peh:obs_lab", observation_design="peh:od_lab") + ) + session.cache.add( + peh.DerivedObservation( + id="peh:obs_summary", + observation_design="peh:od_summary", + was_derived_from="peh:obs_lab", + ) + ) + session.cache.add( + peh.ObservationGroup( + id="peh:og_lab", + ui_label="Lab assays", + observation_id_list=["peh:obs_summary"], + ) + ) + session.cache.add( + peh.StudyEntity( + id="peh:population_adults", + ui_label="Adults", + description="Adults of both sexes.", + ) + ) + session.cache.add( + peh.Study( + id="peh:study_demo", + ui_label="Demo study", + observation_group_id_list=["peh:og_lab"], + study_entity_id_list=["peh:population_adults"], + ) + ) + session.cache.add( + peh.Project( + id="peh:project_demo", + ui_label="Demo project", + study_id_list=["peh:study_demo"], + ) + ) + layout = peh.DataLayout( + id="peh:layout_demo", + ui_label="LAYOUT_DEMO", + sections=[ + peh.DataLayoutSection( + id="peh:section_lab", + ui_label="LAB", + elements=[ + peh.DataLayoutElement( + label="id_sample", + observable_property="peh:prop_id_sample", + ), + peh.DataLayoutElement( + label="chol", + observable_property="peh:prop_chol", + ), + peh.DataLayoutElement( + label="chol_lod", + observable_property="peh:prop_chol_lod", + ), + ], + ), + ], + ) + session.cache.add(layout) + import_config = peh.DataImportConfig( + id="peh:import_demo", + layout="peh:layout_demo", + section_mapping=peh.DataSectionMapping( + section_mapping_links=[ + peh.DataSectionMappingLink( + section="peh:section_lab", + observation_id_list=["peh:obs_lab"], + ) + ] + ), + ) + session.cache.add(import_config) + return import_config + + +@pytest.fixture +def metadata_workflow(): + import polars as pl + + session = Session() + import_config = _populate_metadata_cache(session) + + series = DatasetSeries(label="DEMO_STUDY") + dataset = series.add_empty_dataset("LAB") + dataset.add_observation_to_index("peh:obs_lab") + series.add_observable_property( + observation_id="peh:obs_lab", + observable_property_id="peh:prop_id_sample", + data_type=ObservablePropertyValueType.STRING, + dataset_label="LAB", + element_label="id_sample", + is_primary_key=True, + ) + series.add_observable_property( + observation_id="peh:obs_lab", + observable_property_id="peh:prop_chol", + data_type=ObservablePropertyValueType.FLOAT, + dataset_label="LAB", + element_label="chol", + ) + series.add_observable_property( + observation_id="peh:obs_lab", + observable_property_id="peh:prop_chol_lod", + data_type=ObservablePropertyValueType.FLOAT, + dataset_label="LAB", + element_label="chol_lod", + ) + dataset.data = pl.DataFrame( + { + "id_sample": ["a", "b", "b"], + "chol": [1.2, None, 3.4], + "chol_lod": [0.1, 0.1, 0.1], + } + ) + return session, import_config, series + + +def _build( + metadata_adapter, + workflow, + metadata_config: StudyMetadataConfig | None = None, +): + session, import_config, series = workflow + return metadata_adapter.build_study_metadata( + dataset_series=series, + data_import_config=import_config, + metadata_config=metadata_config or StudyMetadataConfig(), + cache_view=CacheContainerView(session.cache), + ) + + +def test_default_adapter_is_dataframe_adapter(): + from pypeh.adapters.metadata.dataframe_adapter import ( + DataFrameMetadataAdapter, + ) + + assert MetadataInterface.get_default_adapter_class() is ( + DataFrameMetadataAdapter + ) + + +class TestLinkKeys: + def test_resolves_study_population_projects_and_groups( + self, metadata_adapter, metadata_workflow + ): + tree = _build(metadata_adapter, metadata_workflow) + assert tree["study"] == [ + { + "id": "peh:study_demo", + "label": "Demo study", + "population": [ + { + "id": "peh:population_adults", + "label": "Adults", + "definition": "Adults of both sexes.", + } + ], + } + ] + assert tree["projects"] == [ + {"id": "peh:project_demo", "label": "Demo project"} + ] + assert tree["observation_groups"] == [ + {"id": "peh:og_lab", "label": "Lab assays"} + ] + + def test_study_reached_through_derived_observation_lineage( + self, metadata_adapter, metadata_workflow + ): + # the group only lists the derived observation; the config only + # the base observation — the lineage walk must bridge them. + tree = _build(metadata_adapter, metadata_workflow) + assert [group["id"] for group in tree["observation_groups"]] == [ + "peh:og_lab" + ] + + def test_unresolvable_study_warns_and_empties_sections( + self, metadata_adapter, metadata_workflow, caplog + ): + session, import_config, series = metadata_workflow + session.cache.pop("peh:og_lab", "ObservationGroup") + with caplog.at_level( + "WARNING", logger="pypeh.core.interfaces.metadata" + ): + tree = metadata_adapter.build_study_metadata( + dataset_series=series, + data_import_config=import_config, + metadata_config=StudyMetadataConfig(), + cache_view=CacheContainerView(session.cache), + ) + assert tree["study"] == [] + assert tree["projects"] == [] + assert any( + "No Study could be resolved" in r.message for r in caplog.records + ) + + def test_population_omitted_when_gated_off( + self, metadata_adapter, metadata_workflow + ): + tree = _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig(include_population=False), + ) + assert tree["study"] == [ + {"id": "peh:study_demo", "label": "Demo study"} + ] + + +class TestCatalogue: + def test_control_variables_excluded_by_default( + self, metadata_adapter, metadata_workflow + ): + tree = _build(metadata_adapter, metadata_workflow) + property_ids = [ + prop["id"] + for observation in tree["observations"] + for prop in observation["observable_properties"] + ] + assert "peh:prop_chol_lod" not in property_ids + + def test_control_variables_included_when_enabled( + self, metadata_adapter, metadata_workflow + ): + tree = _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig(exclude_control_variables=False), + ) + property_ids = [ + prop["id"] + for observation in tree["observations"] + for prop in observation["observable_properties"] + ] + assert "peh:prop_chol_lod" in property_ids + + def test_indicator_semantics_resolved_from_cache( + self, metadata_adapter, metadata_workflow + ): + tree = _build(metadata_adapter, metadata_workflow) + chol = next( + prop + for observation in tree["observations"] + for prop in observation["observable_properties"] + if prop["id"] == "peh:prop_chol" + ) + semantics = chol["semantics"] + assert semantics["indicator"]["label"] == "Total cholesterol" + assert semantics["matrix"]["label"] == "Blood" + assert semantics["biochem_entity"]["exact_matches"] == ["CHEBI:16288"] + assert ( + semantics["biochem_entity"]["molecular_weight_gram_per_mol"] + == 386.65 + ) + assert semantics["constraints"] == ["plausible"] + assert chol["groups"] == ["Lipids"] + + def test_calculation_tree_links_source_variable( + self, metadata_adapter, metadata_workflow + ): + tree = _build(metadata_adapter, metadata_workflow) + mean = next( + prop + for observation in tree["observations"] + for prop in observation["observable_properties"] + if prop["id"] == "peh:prop_chol_mean" + ) + assert mean["kind"] == "calculation" + assert mean["calculation"] == { + "name": "mean_chol", + "function": "statistics_mean", + "inputs": [ + { + "argument": None, + "source_variable": "peh:prop_chol", + "source_label": "chol", + "value": None, + } + ], + "results": ["chol_mean"], + } + + def test_derived_observation_follows_base_in_lineage( + self, metadata_adapter, metadata_workflow + ): + tree = _build(metadata_adapter, metadata_workflow) + assert [obs["id"] for obs in tree["observations"]] == [ + "peh:obs_lab", + "peh:obs_summary", + ] + + def test_multi_level_derivation_in_dependency_order( + self, metadata_adapter, metadata_workflow + ): + # the grandchild is cached before its parent: the order must + # follow the derivation graph, not cache or set order. + session = metadata_workflow[0] + session.cache.add( + peh.DerivedObservation( + id="peh:obs_grandchild", + observation_design="peh:od_summary", + was_derived_from="peh:obs_child", + ) + ) + session.cache.add( + peh.DerivedObservation( + id="peh:obs_child", + observation_design="peh:od_summary", + was_derived_from="peh:obs_summary", + ) + ) + tree = _build(metadata_adapter, metadata_workflow) + assert [obs["id"] for obs in tree["observations"]] == [ + "peh:obs_lab", + "peh:obs_summary", + "peh:obs_child", + "peh:obs_grandchild", + ] + + def test_missing_observation_design_warns_and_skips( + self, metadata_adapter, metadata_workflow, caplog + ): + session = metadata_workflow[0] + session.cache.pop("peh:od_summary", "ObservationDesign") + with caplog.at_level( + "WARNING", logger="pypeh.core.interfaces.metadata" + ): + tree = _build(metadata_adapter, metadata_workflow) + assert [obs["id"] for obs in tree["observations"]] == ["peh:obs_lab"] + assert any( + "peh:od_summary" in record.message for record in caplog.records + ) + + def test_multiple_indicators_warn_and_emit_first( + self, metadata_adapter, metadata_workflow, caplog + ): + session = metadata_workflow[0] + session.cache.add( + peh.IndicatorSubClass(id="peh:indicator_other", ui_label="Other") + ) + chol = session.cache.get("peh:prop_chol", "ObservableProperty") + chol.has_observable_property_type = [ + "peh:indicator_chol", + "peh:indicator_other", + ] + with caplog.at_level( + "WARNING", logger="pypeh.core.interfaces.metadata" + ): + tree = _build(metadata_adapter, metadata_workflow) + chol_tree = next( + prop + for observation in tree["observations"] + for prop in observation["observable_properties"] + if prop["id"] == "peh:prop_chol" + ) + assert chol_tree["semantics"]["indicator"]["id"] == ( + "peh:indicator_chol" + ) + assert any( + "2 indicators" in record.message for record in caplog.records + ) + + +class TestDatasets: + def test_dataset_variables_and_dtypes( + self, metadata_adapter, metadata_workflow + ): + tree = _build(metadata_adapter, metadata_workflow) + (dataset,) = tree["datasets"] + assert dataset["label"] == "LAB" + assert dataset["observations"] == ["peh:obs_lab"] + labels = [variable["label"] for variable in dataset["variables"]] + assert labels == ["id_sample", "chol"] # chol_lod excluded + id_sample = dataset["variables"][0] + assert id_sample["data_type"] == "STRING" + assert id_sample["primary_key"] is True + assert dataset["variables"][1]["data_type"] == "FLOAT" + + def test_no_profile_even_when_data_is_loaded( + self, metadata_adapter, metadata_workflow + ): + # profiling is deliberately out of scope: per-column counts can + # de-anonymize individuals in small cohorts; profile data will + # later be produced by the aggregation service and passed in. + tree = _build(metadata_adapter, metadata_workflow) + for variable in tree["datasets"][0]["variables"]: + assert "profile" not in variable + + +class TestProvenance: + def test_provenance_sections_and_derivation_chain( + self, metadata_adapter, metadata_workflow + ): + tree = _build(metadata_adapter, metadata_workflow) + provenance = tree["provenance"] + assert provenance["series_label"] == "DEMO_STUDY" + assert provenance["data_import_config"] == "peh:import_demo" + assert provenance["data_layout"] == "peh:layout_demo" + assert provenance["sections"] == [ + {"dataset": "LAB", "described_by": None} + ] + assert provenance["derived_observations"] == [ + { + "derived_observation": "peh:obs_summary", + "was_derived_from": "peh:obs_lab", + } + ] + assert provenance["validation_report"] is None + + def test_extra_provenance_merged( + self, metadata_adapter, metadata_workflow + ): + tree = _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig(extra_provenance={"source_file": "demo.xlsx"}), + ) + assert tree["provenance"]["source_file"] == "demo.xlsx" + + def test_extra_provenance_cannot_override_builtin_fields( + self, metadata_adapter, metadata_workflow + ): + with pytest.raises(ValueError, match="pypeh_version"): + _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig( + extra_provenance={"pypeh_version": "spoofed"} + ), + ) + + def test_missing_distribution_version_is_none( + self, metadata_adapter, metadata_workflow, monkeypatch + ): + import importlib.metadata + + def _not_installed(distribution): + raise importlib.metadata.PackageNotFoundError(distribution) + + monkeypatch.setattr(importlib.metadata, "version", _not_installed) + provenance = _build(metadata_adapter, metadata_workflow)["provenance"] + assert provenance["pypeh_version"] is None + assert provenance["peh_model_version"] is None + + def test_validation_report_attached_not_run( + self, metadata_adapter, metadata_workflow + ): + session, import_config, series = metadata_workflow + from pypeh.core.models.validation_errors import ( + ValidationErrorReport, + ValidationErrorReportCollection, + ) + + report = ValidationErrorReportCollection( + { + "peh:obs_lab": ValidationErrorReport.from_runtime_error( + ValueError("boom") + ) + } + ) + tree = metadata_adapter.build_study_metadata( + dataset_series=series, + data_import_config=import_config, + metadata_config=StudyMetadataConfig(), + cache_view=CacheContainerView(session.cache), + validation_report=report, + ) + attached = tree["provenance"]["validation_report"] + assert attached == { + "peh:obs_lab": report["peh:obs_lab"].model_dump(mode="json") + } + assert attached["peh:obs_lab"]["error_counts"]["FATAL"] == 1 + + +class TestSeriesConfigConsistency: + def test_series_observation_outside_config_raises( + self, metadata_adapter, metadata_workflow + ): + series = metadata_workflow[2] + series.parts["LAB"].add_observation_to_index("peh:obs_foreign") + with pytest.raises(ValueError, match="peh:obs_foreign"): + _build(metadata_adapter, metadata_workflow) + + def test_derived_observation_in_series_is_accepted( + self, metadata_adapter, metadata_workflow + ): + series = metadata_workflow[2] + series.parts["LAB"].add_observation_to_index("peh:obs_summary") + tree = _build(metadata_adapter, metadata_workflow) + assert tree["datasets"][0]["observations"] == [ + "peh:obs_lab", + "peh:obs_summary", + ] + + +class TestSerialization: + @staticmethod + def _assert_builtin_only(value, path="tree"): + if isinstance(value, dict): + for key, item in value.items(): + assert type(key) is str, f"{path}: key {key!r}" + TestSerialization._assert_builtin_only(item, f"{path}.{key}") + elif isinstance(value, list): + for index, item in enumerate(value): + TestSerialization._assert_builtin_only( + item, f"{path}[{index}]" + ) + else: + assert type(value) in (str, int, float, bool, type(None)), ( + f"{path}: {type(value).__name__}" + ) + + def test_tree_holds_builtin_types_only( + self, metadata_adapter, metadata_workflow + ): + tree = _build(metadata_adapter, metadata_workflow) + self._assert_builtin_only(tree) + + def test_tree_serializes_without_aliases( + self, metadata_adapter, metadata_workflow + ): + import json + + import yaml + + tree = _build(metadata_adapter, metadata_workflow) + json.dumps(tree) + dumped = yaml.safe_dump(tree, sort_keys=False) + assert "&id" not in dumped + assert "*id" not in dumped + assert yaml.safe_load(dumped) == tree + + def test_catalogue_and_datasets_share_no_objects( + self, metadata_adapter, metadata_workflow + ): + tree = _build(metadata_adapter, metadata_workflow) + catalogue_chol = next( + prop + for observation in tree["observations"] + for prop in observation["observable_properties"] + if prop["id"] == "peh:prop_chol" + ) + dataset_chol = next( + variable["observable_property"] + for variable in tree["datasets"][0]["variables"] + if variable["label"] == "chol" + ) + assert dataset_chol == catalogue_chol + assert dataset_chol is not catalogue_chol + + +class TestSectionGating: + @pytest.mark.parametrize( + ("flag", "expected_absent"), + [ + ("include_link_keys", ["study", "projects", "observation_groups"]), + ("include_variable_catalogue", ["observations"]), + ("include_datasets", ["datasets"]), + ("include_provenance", ["provenance"]), + ], + ) + def test_disabled_sections_omitted( + self, metadata_adapter, metadata_workflow, flag, expected_absent + ): + tree = _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig(**{flag: False}), + ) + for key in expected_absent: + assert key not in tree + + def test_datasets_without_catalogue_fall_back_to_ids( + self, metadata_adapter, metadata_workflow + ): + tree = _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig(include_variable_catalogue=False), + ) + variable = tree["datasets"][0]["variables"][0] + assert variable["observable_property"] == {"id": "peh:prop_id_sample"} diff --git a/tests/core/session/test_session_metadata.py b/tests/core/session/test_session_metadata.py new file mode 100644 index 00000000..e3fe36fd --- /dev/null +++ b/tests/core/session/test_session_metadata.py @@ -0,0 +1,172 @@ +import pytest +import peh_model.peh as peh + +from pypeh import Session +from pypeh.core.models.metadata_dto import StudyMetadataConfig + + +@pytest.fixture +def session_with_demo_workflow(): + """ + Minimal in-cache workflow: one observation, one study, one + population entry, one project — enough to exercise the Session + facade of the metadata service without any dataframe I/O. + """ + session = Session() + session.cache.add( + peh.ObservableProperty(id="peh:prop_id_sample", short_name="id_sample") + ) + session.cache.add( + peh.ObservationDesign( + id="peh:od_lab", + observable_property_specifications=[ + peh.ObservablePropertySpecification( + observable_property="peh:prop_id_sample" + ) + ], + ) + ) + session.cache.add( + peh.Observation(id="peh:obs_lab", observation_design="peh:od_lab") + ) + session.cache.add( + peh.ObservationGroup( + id="peh:og_lab", + observation_id_list=["peh:obs_lab"], + ) + ) + session.cache.add(peh.StudyEntity(id="peh:population", ui_label="Adults")) + session.cache.add( + peh.Study( + id="peh:study", + ui_label="Demo study", + observation_group_id_list=["peh:og_lab"], + study_entity_id_list=["peh:population"], + ) + ) + session.cache.add( + peh.Project( + id="peh:project", + ui_label="Demo project", + study_id_list=["peh:study"], + ) + ) + session.cache.add( + peh.DataLayout( + id="peh:layout", + ui_label="LAYOUT", + sections=[ + peh.DataLayoutSection( + id="peh:section_lab", + ui_label="LAB", + elements=[ + peh.DataLayoutElement( + label="id_sample", + observable_property="peh:prop_id_sample", + ) + ], + ) + ], + ) + ) + import_config = peh.DataImportConfig( + id="peh:import", + layout="peh:layout", + section_mapping=peh.DataSectionMapping( + section_mapping_links=[ + peh.DataSectionMappingLink( + section="peh:section_lab", + observation_id_list=["peh:obs_lab"], + ) + ] + ), + ) + session.cache.add(import_config) + + from pypeh.core.models.constants import ObservablePropertyValueType + from pypeh.core.models.internal_data_layout import DatasetSeries + + series = DatasetSeries(label="DEMO_STUDY") + dataset = series.add_empty_dataset("LAB") + dataset.add_observation_to_index("peh:obs_lab") + series.add_observable_property( + observation_id="peh:obs_lab", + observable_property_id="peh:prop_id_sample", + data_type=ObservablePropertyValueType.STRING, + dataset_label="LAB", + element_label="id_sample", + ) + return session, import_config, series + + +@pytest.mark.session +class TestCreateStudyMetadata: + def test_uses_registered_metadata_adapter( + self, session_with_demo_workflow + ): + from pypeh.core.interfaces.metadata import MetadataInterface + + session, import_config, series = session_with_demo_workflow + + class RecordingAdapter(MetadataInterface): + def __init__(self): + self.called = False + + def build_study_metadata(self, **kwargs): + self.called = True + self.kwargs = kwargs + return {"study": []} + + adapter = RecordingAdapter() + session.register_adapter("metadata", adapter) + result = session.create_study_metadata( + dataset_series=series, + data_import_config=import_config, + ) + assert adapter.called + assert result == {"study": []} + + def test_non_metadata_adapter_raises(self, session_with_demo_workflow): + session, import_config, series = session_with_demo_workflow + session.register_adapter("metadata", object()) + with pytest.raises(AssertionError): + session.create_study_metadata( + dataset_series=series, + data_import_config=import_config, + ) + + +@pytest.mark.dataframe +class TestCreateStudyMetadataWithDataAdapter: + def test_default_config_produces_full_tree( + self, session_with_demo_workflow + ): + session, import_config, series = session_with_demo_workflow + tree = session.create_study_metadata( + dataset_series=series, + data_import_config=import_config, + ) + assert set(tree) == { + "study", + "projects", + "observation_groups", + "observations", + "datasets", + "provenance", + } + assert tree["study"][0]["id"] == "peh:study" + assert tree["study"][0]["population"] == [ + {"id": "peh:population", "label": "Adults", "definition": None} + ] + assert tree["provenance"]["data_import_config"] == "peh:import" + + def test_metadata_config_forwarded(self, session_with_demo_workflow): + session, import_config, series = session_with_demo_workflow + tree = session.create_study_metadata( + dataset_series=series, + data_import_config=import_config, + metadata_config=StudyMetadataConfig( + include_link_keys=False, include_provenance=False + ), + ) + assert set(tree) == {"observations", "datasets"} diff --git a/tests/integration/metadata/__init__.py b/tests/integration/metadata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/metadata/input/peh_demo.yaml b/tests/integration/metadata/input/peh_demo.yaml new file mode 100644 index 00000000..c0fde9d1 --- /dev/null +++ b/tests/integration/metadata/input/peh_demo.yaml @@ -0,0 +1,821 @@ +# ============================================================================= +# pypeh metadata demo — EVERY PEH object that is reachable today +# ============================================================================= +# +# Load with: +# session.load_persisted_cache("config/peh_demo.yaml") +# +# The file is organised by *reachability tier* (see the analysis): +# +# TIER 1 pypeh traverses these for you +# -> ObservableProperty (+ value options / metadata / validation designs) +# -> ObservationDesign (+ property specifications + calculation designs) +# -> Observation / DerivedObservation +# -> DataLayout -> DataLayoutSection -> DataLayoutElement +# -> DataImportConfig / DataExportConfig / ObservationGroup +# -> ObservationAlignment (+ assemblies) +# -> DataFilter (+ filter expressions) on the extract path +# +# TIER 3 loadable from YAML, but NO pypeh code path ever looks at them: +# you must traverse the ids yourself +# -> ObservablePropertyMetadataField (decodes value_metadata) +# -> Grouping (variable panels) +# -> BioChemEntitySubClass / MatrixSubClass / IndicatorSubClass +# -> Project / Study / StudyEntity / PhysicalEntity / Stakeholder +# -> ObservationResult / ObservedValue (+ QualityData, ProvenanceData) +# -> DataRequest +# +# NOT REACHABLE AT ALL (no EntityList slot, verified): +# BioChemEntity, Matrix, Indicator, Sample, Person, Geolocation, +# Environment, HomeEnvironment, WorkEnvironment, StudyPopulation, +# SampleCollection, StudySubject, StudySubjectGroup, DataStakeholder, +# ResearchObjective, ProcessingAction, ProcessingStep, IAdoptVariable +# +# The interesting bit: Tier 3 metabolite/matrix semantics hang off +# `ObservableProperty.has_observable_property_type` -> IndicatorSubClass -> +# MatrixSubClass / BioChemEntitySubClass. pypeh never follows that link, but +# everything needed is right here in the cache. +# ============================================================================= + + +# ============================================================================= +# TIER 3a — metadata field vocabulary +# Without these, ObservableProperty.value_metadata is just a list of opaque +# (field, value) id/value pairs. +# ============================================================================= +metadata_fields: +- id: id_line + value_type: integer + description: Line number of the variable in the source codebook +- id: cb_source + value_type: string + description: Assay family / data collection the variable originates from +- id: unit_display + value_type: string + description: Human readable unit as printed in the source codebook +- id: value_status + value_type: string + description: Which status column (raw / LOD / LOQ / imputed) a value came from + + +# ============================================================================= +# TIER 3b — variable groupings (panels) +# ============================================================================= +groupings: +- id: peh:GROUPING_LIPID_PANEL + short_name: lipid_panel + ui_label: Lipid panel + description: > + Lipid variables quantified in the same lab assay batch. Referenced from + ObservableProperty.grouping_id_list and BioChemEntitySubClass.group_labels. + translations: + - property_name: ui_label + language: en + translated_value: Lipid panel + + +# ============================================================================= +# TIER 3c — the analyte chain (the "biochemical entity" metadata) +# +# BioChemEntitySubClass <- what is measured (CHEBI-anchored template) +# MatrixSubClass <- medium / matrix (blood, serum, ...) +# IndicatorSubClass <- ties the two together, and is what +# ObservableProperty.has_observable_property_type +# points at. +# +# These are owl:Class templates, not per-study entity instances: peh_model's +# EntityList has no slot for BioChemEntity/Matrix instances. +# ============================================================================= +biochementity_subclasses: +- id: peh:BCE_CHOLESTEROL + short_name: cholesterol + ui_label: Cholesterol + description: > + A sterol lipid and a component of the cell membrane of every animal cell; + also a precursor of steroid hormones and bile acids. + aliases: + - total cholesterol + exact_matches: + - CHEBI:16288 + molweight_grampermol: 386.65 + current_validation_status: validated + group_labels: + - sterol + - lipid + grouping_id_list: + - peh:GROUPING_LIPID_PANEL + has_role: + - peh:ROLE_MEASURED_ANALYTE +- id: peh:BCE_HDL_CHOLESTEROL + short_name: hdl_cholesterol + ui_label: HDL cholesterol + description: The cholesterol content of the high density lipoprotein fraction. + exact_matches: + - CHEBI:16289 + molweight_grampermol: 386.65 + current_validation_status: validated + group_labels: + - lipoprotein +- id: peh:BCE_TRIGLYCERIDE + short_name: triglyceride + ui_label: Triglyceride + description: Any triester of glycerol with three fatty acids. + exact_matches: + - CHEBI:18059 + molweight_grampermol: 885.4 + current_validation_status: validated + is_metabolite_of: + - peh:BCE_CHOLESTEROL + group_labels: + - glycerolipid + +matrix_subclasses: +- id: peh:MATRIX_BLOOD_WHOLE_BLOOD + short_name: bwb + ui_label: Blood (whole blood) + description: Whole blood matrix, the substrate for the lab assays. + context_aliases: + - context: matrix + alias: bwb +- id: peh:MATRIX_SERUM + short_name: serum + ui_label: Serum + description: The cell-free fraction of coagulated blood. + +indicator_subclasses: +- id: peh:IND_TOTAL_CHOLESTEROL + short_name: tc + ui_label: Total cholesterol + description: Sum of cholesterol contained in all lipoproteins of the sample. + indicator_type: observation + quantity_kind: http://qudt.org/vocab/quantitykind/Mass + matrix: peh:MATRIX_BLOOD_WHOLE_BLOOD + biochementity: peh:BCE_CHOLESTEROL + grouping_id_list: + - peh:GROUPING_LIPID_PANEL + relevant_observable_entity_types: + - sample + suggester: https://orcid.org/0000-0002-1825-0097 + constraints: + - constraint_id: minimum_plausible_concentration + constrains: peh:IND_TOTAL_CHOLESTEROL +- id: peh:IND_HDL_CHOLESTEROL + short_name: hdl_c + ui_label: HDL cholesterol + indicator_type: observation + quantity_kind: http://qudt.org/vocab/quantitykind/Mass + matrix: peh:MATRIX_SERUM + biochementity: peh:BCE_HDL_CHOLESTEROL + grouping_id_list: + - peh:GROUPING_LIPID_PANEL + relevant_observable_entity_types: + - sample +- id: peh:IND_TRIGLYCERIDES + short_name: tg + ui_label: Triglycerides + indicator_type: observation + quantity_kind: http://qudt.org/vocab/quantitykind/Mass + matrix: peh:MATRIX_SERUM + biochementity: peh:BCE_TRIGLYCERIDE + relevant_observable_entity_types: + - sample + + +# ============================================================================= +# TIER 1 — ObservableProperty (the column definitions) +# ============================================================================= +observable_properties: +- id: id_sample + short_name: id_sample + name: demo|sample|id_sample + ui_label: Sample identifier + description: Unique identifier of the analysed sample. + remark: Used as the identifying ObservableProperty in every sample observation. + value_type: string + required: true + immutable: true + zeroallowed: false + observation_result_type: measurement + relevant_observation_types: + - sampling + - metadata + value_metadata: + - field: id_line + value: '1' + context_aliases: + - context: sample_id + alias: sid + property_name: ui_label + +- id: age_birth_m + short_name: age_birth_m + name: demo|person|age_birth_m + ui_label: Age at birth (months) + description: Age of the participant in months at the time of sample collection. + value_type: integer + min_value: '0' + max_value: '1440' + unit: Month + quantity_kind: http://qudt.org/vocab/quantitykind/Time + observation_result_type: measurement + value_metadata: + - field: id_line + value: '2' + - field: unit_display + value: months + translations: + - property_name: ui_label + language: en + translated_value: Age at birth (months) + +- id: sex + short_name: sex + name: demo|person|sex + ui_label: Sex + description: Self-reported sex of the participant. + value_type: string + categorical: true + multivalued: false + required: true + observation_result_type: measurement + value_options: + - key: '1' + value: male + label: Male + - key: '2' + value: female + label: Female + - key: '9' + value: not reported + label: Not reported + value_metadata: + - field: id_line + value: '3' + context_aliases: + - context: sex + alias: s + property_name: ui_label + +- id: chol + short_name: chol + name: demo|bloodwholeblood|chol + ui_label: Total cholesterol + description: > + Total cholesterol concentration quantified in whole blood, in micromoles + per litre. + remark: > + Measured values (X) are given; -10 = measurement was planned but the sample + could not be measured; -1 = X < LOD; -2 = LOD <= X < LOQ. + value_type: decimal + unit: MicroGM-PER-L + quantity_kind: http://qudt.org/vocab/quantitykind/Mass + zeroallowed: true + significantdecimals: 6 + immutable: false + observation_result_type: measurement + relevant_observation_types: + - sampling + grouping_id_list: + - peh:GROUPING_LIPID_PANEL + # ---- THE TIER 3 HOOK: analyte / matrix semantics live behind this id ---- + has_observable_property_type: + - peh:IND_TOTAL_CHOLESTEROL + value_metadata: + - field: id_line + value: '20' + - field: cb_source + value: blood markers + - field: unit_display + value: ug/L + - field: value_status + value: chol + validation_designs: + - validation_name: chol_must_be_numeric + validation_error_level: error + validation_error_message_template: "chol is not numeric" + validation_expression: + validation_command: is_not_null + - validation_name: chol_coding_flag_allowed + validation_error_level: warning + validation_error_message_template: "chol contains an unexpected status code" + validation_expression: + validation_command: disjunction + validation_arg_expressions: + - validation_command: is_less_than + validation_arg_values: + - '2.5' + - validation_command: is_equal_to + validation_arg_values: + - '-1' + - validation_command: is_equal_to + validation_arg_values: + - '-2' + - validation_command: is_equal_to + validation_arg_values: + - '-10' + +- id: chol_lod + short_name: chol_lod + ui_label: Total cholesterol LOD + description: Limit of detection of the total cholesterol assay. + value_type: float + unit: MicroGM-PER-L + observation_result_type: control + value_metadata: + - field: id_line + value: '21' + - field: value_status + value: lod + +- id: chol_loq + short_name: chol_loq + ui_label: Total cholesterol LOQ + description: Limit of quantification of the total cholesterol assay. + value_type: float + unit: MicroGM-PER-L + observation_result_type: control + value_metadata: + - field: id_line + value: '22' + - field: value_status + value: loq + +- id: hdl + short_name: hdl + ui_label: HDL cholesterol + description: HDL cholesterol concentration in serum. + value_type: decimal + unit: MicroGM-PER-L + zeroallowed: true + significantdecimals: 6 + observation_result_type: measurement + grouping_id_list: + - peh:GROUPING_LIPID_PANEL + has_observable_property_type: + - peh:IND_HDL_CHOLESTEROL + value_metadata: + - field: id_line + value: '23' + - field: cb_source + value: blood markers + +- id: trigl + short_name: trigl + ui_label: Triglycerides + description: Triglyceride concentration in serum. + value_type: decimal + unit: MicroGM-PER-L + zeroallowed: true + significantdecimals: 6 + observation_result_type: measurement + has_observable_property_type: + - peh:IND_TRIGLYCERIDES + value_metadata: + - field: id_line + value: '24' + - field: cb_source + value: blood markers + +- id: chol_imputed + short_name: chol_imputed + ui_label: chol_imputed + description: > + LOD/LOQ-corrected total cholesterol: below-quantification status codes are + replaced by the limit of quantification. + value_type: decimal + unit: MicroGM-PER-L + observation_result_type: calculation + value_metadata: + - field: value_status + value: imputed + +- id: chol_mean + short_name: chol_mean + ui_label: chol_mean + description: Arithmetic mean of the LOD/LOQ-corrected total cholesterol per stratum. + value_type: decimal + unit: MicroGM-PER-L + observation_result_type: calculation + + +# ============================================================================= +# TIER 1 — ObservationDesign (which variables belong to which observation) +# ============================================================================= +observation_designs: +- id: peh:OD_SAMPLE_METADATA + short_name: od_sample_metadata + ui_label: Sample metadata design + description: Subject level metadata recorded once per participant. + observation_result_type: measurement + observable_entity_type: person + observable_entity_id_list: + - peh:PE_PERSON_0001 + observable_property_specifications: + - observable_property: id_sample + specification_category: identifying + - observable_property: sex + specification_category: required + - observable_property: age_birth_m + specification_category: required + +- id: peh:OD_LAB_ASSAYS + short_name: od_lab_assays + ui_label: Lab assay design + description: Lipid panel measured on every sample. + observation_result_type: measurement + observable_entity_type: sample + observable_entity_id_list: + - peh:SE_SAMPLE_0001 + observable_property_specifications: + - observable_property: id_sample + specification_category: identifying + - observable_property: chol + specification_category: optional + - observable_property: chol_lod + specification_category: optional + - observable_property: chol_loq + specification_category: optional + - observable_property: hdl + specification_category: optional + - observable_property: trigl + specification_category: optional + +- id: peh:OD_LAB_ENRICHED + short_name: od_lab_enriched + ui_label: Enriched lab design + description: Adds one derived variable to the lab assay observation. + observation_result_type: calculation + observable_entity_type: sample + observable_property_specifications: + - observable_property: id_sample + specification_category: identifying + - observable_property: chol_imputed + specification_category: derived + calculation_design: + calculation_name: abs_corrected_cholesterol + calculation_implementation: + function_name: compehndly.entrypoints.medium_bound_imputation + function_kwargs: + - mapping_name: measurement + contextual_field_reference: + dataset_label: peh:OBS_LAB_ASSAYS + field_label: chol + - mapping_name: loq + contextual_field_reference: + dataset_label: peh:OBS_LAB_ASSAYS + field_label: chol_loq + - mapping_name: lod + contextual_field_reference: + dataset_label: peh:OBS_LAB_ASSAYS + field_label: chol_lod + function_results: + - mapping_name: chol_imputed + value_type: decimal + unit: MicroGM-PER-L + round_decimals: 3 + +- id: peh:OD_SUMMARY + short_name: od_summary + ui_label: Summary design + description: Study level summary of the enriched cholesterol variable. + observation_result_type: calculation + observable_entity_type: sample + observable_property_specifications: + - observable_property: id_sample + specification_category: identifying + - observable_property: chol_mean + specification_category: derived + calculation_design: + calculation_name: mean_corrected_cholesterol + calculation_implementation: + function_name: pypeh.adapters.aggregation.polars_adapter.statistics.statistics_mean + function_kwargs: + # aggregation: exactly one contextual ref WITHOUT mapping_name is the + # primary source value column + - contextual_field_reference: + dataset_label: peh:OBS_LAB_ENRICHED + field_label: chol_imputed + function_results: + - mapping_name: chol_mean + value_type: decimal + unit: MicroGM-PER-L + + +# ============================================================================= +# TIER 1 — Observations (base + derived) +# ============================================================================= +observations: +- id: peh:OBS_SAMPLE_METADATA + short_name: obs_sample_metadata + ui_label: SAMPLE_METADATA + description: Participant level metadata of the demo cohort. + observation_type: metadata + observation_design: peh:OD_SAMPLE_METADATA +- id: peh:OBS_LAB_ASSAYS + short_name: obs_lab_assays + ui_label: LAB_ASSAYS + description: Lipid panel of every collected sample. + observation_type: sampling + observation_design: peh:OD_LAB_ASSAYS + +derived_observations: +- id: peh:OBS_LAB_ENRICHED + short_name: obs_lab_enriched + ui_label: LAB_ASSAYS_ENRICHED + description: Lab assays plus the LOD/LOQ corrected cholesterol variable. + was_derived_from: peh:OBS_LAB_ASSAYS + observation_design: peh:OD_LAB_ENRICHED +- id: peh:OBS_SUMMARY + short_name: obs_summary + ui_label: SUMMARY + description: Aggregated summary of the enriched lab assays. + was_derived_from: peh:OBS_LAB_ENRICHED + observation_design: peh:OD_SUMMARY + + +# ============================================================================= +# TIER 3d — ObservationGroup (enrichment / aggregation targets in one place) +# ============================================================================= +observation_groups: +- id: peh:OG_ENRICHMENT + ui_label: Enrichment targets + description: Derived observations produced by session.enrich. + observation_id_list: + - peh:OBS_LAB_ENRICHED +- id: peh:OG_AGGREGATION + ui_label: Aggregation targets + description: Derived observations produced by session.aggregate. + observation_id_list: + - peh:OBS_SUMMARY + + +# ============================================================================= +# TIER 1 — DataLayout (the codebook / table structure) +# Sheet names in the data file must equal DataLayoutSection.ui_label. +# ============================================================================= +layouts: +- id: peh:LAYOUT_DEMO + short_name: codebook_demo + ui_label: DEMO_STUDY + description: Layout of the demo study; one data_table per observation. + sections: + - id: peh:SECTION_SAMPLE_METADATA + short_name: section_sample_metadata + ui_label: SAMPLE_METADATA + description: One row per participant. + section_type: data_table + observable_entity_type: person + elements: + - label: id_sample + element_type: data_field + observable_property: id_sample + is_observable_entity_key: true + order: 1 + - label: sex + element_type: data_field + observable_property: sex + order: 2 + - label: age_birth_m + element_type: data_field + observable_property: age_birth_m + order: 3 + validation_designs: + - validation_name: id_sample_must_be_unique + validation_error_level: fatal + validation_error_message_template: "id_sample must be unique in SAMPLE_METADATA" + validation_expression: + validation_command: is_unique + + - id: peh:SECTION_LAB_ASSAYS + short_name: section_lab_assays + ui_label: LAB_ASSAYS + description: One row per sample, joined to SAMPLE_METADATA on id_sample. + section_type: data_table + observable_entity_type: sample + elements: + - label: id_sample + element_type: data_field + observable_property: id_sample + is_observable_entity_key: true + order: 1 + foreign_key_link: + section: peh:SECTION_SAMPLE_METADATA + label: id_sample + - label: chol + element_type: data_field + observable_property: chol + order: 2 + - label: chol_lod + element_type: data_field + observable_property: chol_lod + order: 3 + - label: chol_loq + element_type: data_field + observable_property: chol_loq + order: 4 + - label: hdl + element_type: data_field + observable_property: hdl + order: 5 + - label: trigl + element_type: data_field + observable_property: trigl + order: 6 + # extract-time filter metadata (only consumed by create_tabular_extract) + data_filter: + filter_expression: + filter_command: is_not_null + filter_subject_contextual_field_references: + - dataset_label: LAB_ASSAYS + field_label: chol + + - id: peh:SECTION_SUMMARY + short_name: section_summary + ui_label: SUMMARY + description: Aggregated summary table produced from the enriched assays. + section_type: property_table + observable_entity_type: sample + elements: + - label: id_sample + element_type: data_field + observable_property: id_sample + is_observable_entity_key: true + order: 1 + - label: chol_mean + element_type: data_field + observable_property: chol_mean + order: 2 + + +# ============================================================================= +# TIER 1 — DataImportConfig / DataExportConfig +# ============================================================================= +import_configs: +- id: peh:IMPORT_DEMO + short_name: import_demo + ui_label: Import demo study + description: Maps the two input sheets onto the two base observations. + layout: peh:LAYOUT_DEMO + section_mapping: + section_mapping_links: + - section: peh:SECTION_SAMPLE_METADATA + observation_id_list: + - peh:OBS_SAMPLE_METADATA + - section: peh:SECTION_LAB_ASSAYS + observation_id_list: + - peh:OBS_LAB_ASSAYS + +export_configs: +- id: peh:EXPORT_DEMO + short_name: export_demo + ui_label: Export summary + description: Projects the summary observation into the SUMMARY section. + layout: peh:LAYOUT_DEMO + section_mapping: + section_mapping_links: + - section: peh:SECTION_SUMMARY + observation_id_list: + - peh:OBS_SUMMARY + + +# ============================================================================= +# TIER 1 — ObservationAlignment (used by concatenate_tabular_dataset_series) +# ============================================================================= +observation_alignments: +- id: peh:ALIGN_DEMO + ui_label: Enrich then aggregate alignment + description: > + Assembles the enriched observation out of the base observation group so that + per-source extracts can be concatenated into one series. + observation_assemblies: + - target_observation_id: peh:OBS_LAB_ENRICHED + source_observation_groups: + - peh:OG_ENRICHMENT + observable_property_mappings: + - source_observable_property_ids: + - chol + - chol_lod + - chol_loq + target_observable_property_id: chol_imputed + + +# ============================================================================= +# TIER 3e — Study context +# Note: only the generic StudyEntity / PhysicalEntity slots exist, so entries +# load as StudyEntity / PhysicalEntity (never as Sample / Person). +# ============================================================================= +projects: +- id: peh:PROJECT_DEMO + short_name: project_demo + ui_label: Demo exposure project + description: Demonstration project for pypeh metadata extraction. + default_language: en + start_date: '2024-01-01' + end_date: '2024-12-31' + study_id_list: + - peh:STUDY_DEMO + +studies: +- id: peh:STUDY_DEMO + short_name: study_demo + ui_label: Demo cohort study + description: One cohort, two observations, one derived summary. + default_language: en + start_date: '2024-02-01' + observation_group_id_list: + - peh:OG_ENRICHMENT + - peh:OG_AGGREGATION + study_entity_id_list: + - peh:POP_DEMO_ADULTS + +stakeholders: +- id: peh:STAKEHOLDER_DATA_STEWARD + ui_label: Data steward + description: Custodian of the demo study data. + rorid: https://ror.org/05t8y2c36 + translations: + - property_name: ui_label + language: en + translated_value: Data steward + +study_entities: +- id: peh:POP_DEMO_ADULTS + ui_label: Demo study population + description: > + Adults of both sexes from the general population, sampled in 2024. +- id: peh:SE_SAMPLE_0001 + ui_label: Sample 0001 + description: > + Generic StudyEntity entry. Only the base StudyEntity slots are accepted + here — concrete Sample slots such as `matrix` are rejected by the loader. + physical_entity: peh:PE_PERSON_0001 + +physical_entities: +- id: peh:PE_PERSON_0001 + ui_label: Participant 0001 + description: Generic PhysicalEntity entry (Person slots are rejected here). + + +# ============================================================================= +# TIER 3f — ObservationResult / ObservedValue +# Per-observation measured values with unit, quality and provenance context. +# pypeh NEVER creates these during import (the import path produces dataframes, +# not PEH results), but they load fine and are pure metadata. +# ============================================================================= +observation_results: +- id: peh:OR_LAB_ASSAYS_0001 + ui_label: Lab assay result for sample 0001 + observation_result_type: measurement + observation_start_date: '2024-03-01' + observation_end_date: '2024-03-02' + observed_values: + - observable_property: chol + observable_entity: peh:SE_SAMPLE_0001 + raw_value: 4.91 + raw_unit: http://qudt.org/vocab/unit/MicroGM-PER-L + normalised_value: 4.91 + normalised_unit: http://qudt.org/vocab/unit/MilliGM-PER-Decil + value_as_string: '4.91' + quality_data: + - quality_context_key: below_loq + quality_value: 'false' + provenance_data: + - provenance_context_key: instrument + provenance_value: LC-MS/MS batch 2024-03-01 + - observable_property: hdl + observable_entity: peh:SE_SAMPLE_0001 + raw_value: 1.42 + raw_unit: http://qudt.org/vocab/unit/MicroGM-PER-L + imputed_value: 1.42 + imputed_unit: http://qudt.org/vocab/unit/MicroGM-PER-L + quality_data: + - quality_context_key: qc_flag + quality_value: pass + + +# ============================================================================= +# TIER 3g — DataRequest (metadata about the data request itself) +# ============================================================================= +data_requests: +- id: peh:DATA_REQUEST_DEMO + short_name: dr_demo + ui_label: Demo data request + description: Illustrates request level metadata. + remark_on_content: Lipid panel with LOD/LOQ columns and participant metadata. + remark_on_methodology: > + Blood samples analysed by LC-MS/MS; below-quantification values are coded + -1 / -2 / -10 as described in ObservableProperty(chol).remark. + observation_design_id_list: + - peh:OD_SAMPLE_METADATA + - peh:OD_LAB_ASSAYS + observed_entity_properties: + - observable_entity: peh:SE_SAMPLE_0001 + observable_property: chol + - observable_entity: peh:SE_SAMPLE_0001 + observable_property: hdl + # NOTE: data_stakeholders / research_objectives / processing_actions / + # processing_steps are plain id references, NOT inlined objects: peh_model's + # EntityList has no slot for DataStakeholder / ResearchObjective / + # ProcessingAction / ProcessingStep, so these ids are permanently dangling. + data_stakeholders: + - peh:DS_DATA_STEWARD \ No newline at end of file diff --git a/tests/integration/metadata/input/peh_demo_data.xlsx b/tests/integration/metadata/input/peh_demo_data.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..728a3ecc4e5e25a5052edf455dae827386a08542 GIT binary patch literal 6152 zcmZ`-byQUE+8vsqTWaW(PHE|G5Cz1ch9QS;B}KYJI;13}Ly%5UKtiOYyHoNT_1^21 z?|08ybKW&)t-b%)&+|U9HC2$1i2wiqI{fz#px@MNuZsWx+(Q8X2;pB0W$a-t5SR;C z*V6&wY{cndXWN}H0PEn!lRLW(>zlqWSn;+vtnE!SBwY6bp07%!sR(140s0pQ7|@*|JS|R;_!>82pB_px{&Y|h%j*M0PBXnjg3eOAd?y;q zN{9AogYvXd3_6Lv+dPm$uiYH|D~nPQXOjAD_A&*cMn)Mb8{xhPF!?-jSoa3i_Tt${ zzicjTTA$)jmHPdh52>=$nhfIedxt8j;3Rpe!zW`!K{AAjfrBztFBMVUQqHOpqa8$^ zd&{ex#dakUE81XsoW3QcQ;Z}Aj3>Dlp!Fr}61vRaOkRCm1?+eB{%AToQo)@rZBFU| zwcB^IOs!lMwW3Y_$VxWb*AT9V!#`&0KBhFfLiIVBTgBUH61+gx{3{5DSxQqIaPd_T z0RT++33h_mI&*RUe3!+wt9No^N$x$DTfR(%DWd1(H__zHGVx2aF~BHlBXhd~clSbL zB(Wl`(4ri-*PsJipT#0KBehQ{vDizDh}I`o>_hd6JRlIyANs;?@>*TwK2Aa?$U%`m(5QP;%-$ z#JusKM1o7IRflsK4jfUiz!Yt2y^Ihu4hsCYr9q3o{jmbF%Bw zE%9}V)E5T~T3sopjeu)I5h~L4w~dskK>_sc`arWQ-Vx>p!_NVKD^im?z_|}sLLLbK zAcvnGwp_nOakqD}F}1h1`I(r%1o{y7*}9V(@8#YLjLI{Ey3r(Ux;1UY#<~q8bQ`;r zZ*jvg->Xw%pc)9h0BtlrnOO9w@{`w*!0{pyz3Ut+OBE&Z`>C|ST8z2V>1hSLt|_uv z?_dmhhHttLw(e~U6oRUZXkt|dIi1-0t=BJvdbx9Y5dRIzs?~Lu0ZesQT!fc*iT{^c<*S#U$jCo=QCn zX1ygVNDtKK0Crm4^OcG^Nfcz!9jR2F6VOt@fEgSB9Zn$%P4 zc9Y_7odyk4T8rV=*8&*;ApNb=PlcQi>Lj_0`$k=(Tme8FB=qFte){N~KIzgiR zhQA@Z>%gbC{u86a!__S4yn+t28`~8Vw@W#x^12R%#jg?aGSv8wXBICC2y=5F&oIZH zQoYGxT~}*p{mjC46ne$*-MpEv@@gMlT$jZ9lSU|}o95C6O#b{ zRet@$u?@D6x@ya~Oe^=9L*9u^*<*8<{BGu>hG@s3j{^1;;e4jF}5= z2r8TmDyzGkPlEGHD#rL`IE*)f%s0k9gvGf&slyl$>nuTGZ;D}0d#6yP+S5EOd`Qyb zIE!JWh>PuolE%U4)$(a{ve;3?q&%=n4}0%XFg3sBA0mmhh!epQ#S>5_HXqqpz$Uu^hfSB*PtCnd!s() z@4~Q#9gwXT+NfCEKKx8o7r275}lxoXf(W!&Xqm~n$HNt(6trfl51Xffoa6CJJ_*&)p*8lAVA|5;qN>-3>43^g6D|| zyo2EWsRRG6l#yD$T6psr8~eNpRlr*sM2w(f#bLUj0554+GrWg`Ce@LA!J;e)b6!=@ z3BD#iDvcpVh52m$c)A{`p;XN}OUR}J(BLcG=5wG{!l5aTys{J2^&* zZEK@8ws+!d)C!SSw}#cZ)vC|-5TAB(J8{jHfCA-j<0xg<#0hl;d2lWTCsl$25aiXo z=lQHL3`4FK%1$x1pCpsw=JvTS&SiL|f!*o8q^iW9SkwxY-kUbhv?aVWDQ~QiN8GaR z+Lq(6`kuFIRbuX@IKux!b&qrbL~k4HPRQ56NAXhBV58z8`YX~EfwiH!Q%=3O8hTI> zrpY1F-%+84t-d{gM?(o7m7gOQyo)(onm9qsbzGdRU{L3uH`jIKGpv&a3vK^06VcJ0 zM#->@I44-i+uv?21m%I$B7L{{=A2JaT_^HRY}R#sg}LK<5tSC2H^7!$*KSsXhsbtU z-@ZhjO?>8KOiq~ih>8okM$KdX!GK4opGzewlFv^~)@g!RSb@QJUh9POrKm*^)$YB8a;V4d4DuKan0b39{XPsxp>+_erb~*uVpjCjnzAU z#u_m{Jp(E&rB|ZHKnt?N@(TKJVaf=tSTSbng=G0Sr8Bv6ughey-!fh^7`pv%anwtv zm0hZgo4$-B@{626ZBlLlox0EGh2&|nZQ9TjU>53b@&7yB6Gif!;?RaK&# zk>K0~O~^q^z!Rwz8U-;no2Yh%%149Wcm|pnR6(B|2eWaN*qwr`LetGljmW?C!=dj& zXufAYFs;Ki8(N9wqBkpOz|jap4COuz`Qzw5ZpY#AU6*(blw?VOtB;Iffenj_QYz z#IH}2TC)qK9~{dOTg@e2C1xfV;V%l2s-WJEa`T-)c3cnRL=Fc&6m?)>myxrBNT7_5 zFPSa*8hRNw>xI2jbM-D2QpZkBHjS^TP9FZ9+@u%sx)t!PL7U?D<^WG_7fXm8V@>A(?k+N-(4P*m`z{M&ll1TEY?=-Ev16odv5P$ErC=i;9L$<5iz%E*qW0^iB@+jc@1 zhldz9iG&47MJJ=vKXMYTqgU6VI3fCo(cmToktSilYRU|5D!0o@PSZn*h-=7=9^NLn zFiLTkues1s4gxb;nj9g}!PaN{;GPq&53r$>qXC3(XYJZh;MS?3I@~hf6YdrRv$Jb zg`eB+0nF%jrb-bu@0!G>Z${bw%8o$7VmMG9AfZ*_#L;usxQ;}F%c;jNWt|#_e~K{C zeW0nC4s1(9ws8;R0CUG-J#6;6A^PBTv%I`e9UuHKOU%pV zaGx1z0XzMr$il#$#O+p^#&DKK>x&=fDtym2&*y!vjwiq^+(?Ny&kx4q=d;8;4^E)I z&u_m0=1Dx>cB1h!5zNx{uKV>Gr%3_G!j81C=v3K=gb?!(zUgS8c-Vxc04Hc4t<4e- z1613#;xCDfaOrup#sD*six>{k&R-WgEC*psDi5G>FD({K-z?#9OYrvZW@6ac$4N8k z5Dvg(?_2Nkg$Uv(NfUkCJ>a25t@2LLyLzwiqMebN)g}53PJ#SutaZ(lP}nf(BCh}| zdQSPd#UAITgN13UOjNNV3;HKjc3?@vfk-9`C51o_&uHv}$}m-BD)k_+V}mZS19umoj4eIZYEA9g5~# zsLv%U(_<_j@M#YvQ2z<1m@IDx0;}0FDYhfI&HJ(wuXk1>g5>thb{RAqi!;QF&6pwD zcC5;$PQq1r@1=XQ6CaDMH9GIB@T{L6tULKt(bTG zC)>pDTXR?=G5Nc_-PQVe@AZ6@kD-2RjD<%E5uU1R(8LesOpqVj!z872($BsZP}Qd^ z&F(#)zM&jN@}lIs7JnnRHX6TdfOMfwER8uh(XHiGiq1sVJ3e_xR!Rxm=%Tw~6gq|Z z`1GgkdQJWU!rf$KXBwFtA4YZ7eL|j?lu?F zcQY=<`Ww{03p`sc-zFoxHEY7B7kqeYHn%s^bh3AF<}$N)g8Up^GU7h}G=|`-cUa$9 z7mnu3QU*o4?jprR}@sT`@`5uoxlIdjlMg~2}bo&Pa z(UHE(I?HZ^4tv+gP9@|jZ3jUEJ;e;gA4ozffuJLbrxi`~Tyl!~u{)rhXDnM1E&7PP z_pv>o@XAuAa7HK*t1L)sOOA3zfeX!SlwY53rzTM z%kS^o;p0B$qyBo(B???DioEDEZ0&2rl_fwLcAg186n^0|S*j^O8}FGKpCAV73s~h7 zp`RY@Yr0|6JdwFkYX%H9?md(`LHk?N{6=2ypW}`YeB8nNZ%s`c9DYd}FJ-gDjTf|c z$riC2Ia859U{3s?e?+DOb-V6ZSJ#s={9{~90n;S7wMyn`p*VY~h6Z1BZ{e5ZS&%`7 z%lE)4Ci;o$1P0^0vV=+Ka^;n)t4f?-hpx0C5}B|xjr$eawC7e^Za_>IG9m8>&3lx5 zDZ$||?r=Lo(b|);HjiU(>n#JNC;C6W1W1W!j8b5rA?rxFuJlMcDL-{NZ_Pu#N1$ppoJev8y80^}E`~h!%smc$o(gktgvaPe`K~A6K_g zNgne}16PosAWo4O%NGP=SwgkKN^`P)RLt}l`^uyY%+nj(mOF@EH@4T_mH=*U5fU`v z6ZP7Q{iapI z&X=`ic1iU($i&K!S(TC>G%lZVw!g02M1a0l2`171f{y#>dC`G>52AO-)w8o32>mCj z;_5Z;TyR|BXms1VwQ7E!PFT84gwL+nEvBXl0wNL8|8`5^5&ZS|`Tsor!8^T6aM$Vk z4F>@H{Da_G@n62*UEp0q><{oY{CfUxd+aXHUBl=PPb)kn;XMCnAKhiSYfJoL$%0RW z|G@Igq`1p+cWM2@(u?*VmVd9Xcfoget3O~P;4kq1+qCX7++98XF!*8oX86w{au<4c z_Wc7D!TJTgI|<( Date: Tue, 20 Oct 2026 10:41:59 +0200 Subject: [PATCH 2/2] Address study-metadata review: provenance, controls, inputs - Reject non-JSON StudyMetadataConfig.extra_provenance values and non-str keys during tree construction with a ValueError naming the offending field, instead of breaking json.dumps at serialization time; cover caller-provided provenance with serialization tests. - Classify control observable properties from the dataset schemas in addition to the observation designs, so exclude_control_variables still holds when an observation design is missing from the cache (regression test drops the design from the cache). - Calculation inputs now carry source_observation next to source_variable, matching how execution resolves inputs (contextual_field_reference.dataset_label is the observation id); the unit test covers two calculations on derived observations that share a property, each referencing its own base observation as execution requires. --- docs/session-reference.md | 10 +- src/pypeh/core/interfaces/metadata.py | 127 +++++++++- src/pypeh/core/models/metadata_dto.py | 6 +- .../core/interfaces/metadata/test_metadata.py | 229 ++++++++++++++++++ .../metadata/test_study_metadata.py | 4 + 5 files changed, 366 insertions(+), 10 deletions(-) diff --git a/docs/session-reference.md b/docs/session-reference.md index f2291b4b..004dcb15 100644 --- a/docs/session-reference.md +++ b/docs/session-reference.md @@ -293,8 +293,9 @@ cache and combines them with the imported series into a nested dictionary: carry their analyte chain under `semantics` (indicator → matrix / biochemical entity, with CHEBI `exact_matches`, quantity kind and constraints); calculation variables carry their `calculation` (function, - input links to source variables, results). LOD / LOQ `control` properties - are excluded by default. + input links to the source observation + source variable, results). + LOD / LOQ `control` properties are excluded by default, including for + datasets whose observation design is missing from the cache. - `datasets` — the series' datasets with their variables, wire-format data types (`STRING` / `INTEGER` / `FLOAT` / ...) and primary keys. Data profiling (rows / nulls / distinct counts) is deliberately out of scope @@ -313,7 +314,10 @@ Sections are gated through `StudyMetadataConfig` (`include_link_keys`, disabled sections are omitted from the tree entirely. Caller-provided `extra_provenance` fields (e.g. `source_file`) are appended to the provenance section; a key that collides with a built-in provenance field -raises `ValueError`. When no study can be +raises `ValueError`, as does a non-JSON value (`Path`, `set`, ... — +values must be finite `str` / `int` / `float` / `bool` / `None` / +`list` / str-keyed `dict`; `NaN` / `Infinity` and self-referencing +containers are rejected). When no study can be resolved for the config, the link-keys sections are empty and a warning is logged. Observations or observation designs missing from the cache are left out with a warning. A `ValueError` is raised when the series holds diff --git a/src/pypeh/core/interfaces/metadata.py b/src/pypeh/core/interfaces/metadata.py index d0388a38..6063bf0e 100644 --- a/src/pypeh/core/interfaces/metadata.py +++ b/src/pypeh/core/interfaces/metadata.py @@ -15,6 +15,7 @@ import importlib import importlib.metadata import logging +import math from abc import abstractmethod from peh_model import peh @@ -73,6 +74,66 @@ def _package_version(distribution: str) -> str | None: return None +def _check_json_value( + value: Any, + key: str, + open_containers: set[int] | None = None, +) -> None: + """ + Enforce the JSON-serialization contract on caller-provided + `extra_provenance` values: finite str / int / float / bool / + None, lists, and str-keyed dicts, recursively. Anything else + (Path, Decimal, set, the non-finite floats NaN / Infinity, ...) + and self-referencing containers raise ValueError naming the + offending key. + """ + if open_containers is None: + open_containers = set() + if value is None or isinstance(value, (str, int, bool)): + return + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError( + "StudyMetadataConfig.extra_provenance field " + f"{key!r} holds the non-finite float {value}; NaN " + "and Infinity are not valid JSON — convert it with " + "str(...)." + ) + return + if isinstance(value, (list, dict)): + # open_containers tracks the nesting stack so that only true + # cycles (which json.dumps rejects too) are flagged, not + # plain re-referencing of the same list or dict. + if id(value) in open_containers: + raise ValueError( + "StudyMetadataConfig.extra_provenance field " + f"{key!r} holds a self-referencing " + f"{type(value).__name__}." + ) + open_containers.add(id(value)) + if isinstance(value, list): + for item in value: + _check_json_value(item, key, open_containers) + else: + for sub_key, item in value.items(): + if not isinstance(sub_key, str): + raise ValueError( + "StudyMetadataConfig.extra_provenance field " + f"{key!r} holds a dict key of type " + f"{type(sub_key).__name__}; expected str — " + "convert it with str(...)." + ) + _check_json_value(item, key, open_containers) + open_containers.remove(id(value)) + return + raise ValueError( + "StudyMetadataConfig.extra_provenance field " + f"{key!r} holds a {type(value).__name__}; expected str, int, " + "float, bool, None, list or dict — convert it with " + "str(...)." + ) + + def _to_plain(value: Any) -> Any: """ Rebuild the tree from builtin types only: `str` subclasses (peh @@ -151,11 +212,17 @@ def build_study_metadata( dataset_series, data_import_config, lineage ) designs = self._observation_designs(cache_view, reachable) - control_ids = ( - self._collect_control_ids(cache_view, designs) - if metadata_config.exclude_control_variables - else set() - ) + if metadata_config.exclude_control_variables: + # Union of both views: the design walk classifies what the + # reachable observations specify, the schema walk what the + # datasets contain. The schema pass also covers datasets + # whose observation design is missing from the cache + # (missing designs are tolerated elsewhere). + control_ids = self._collect_control_ids( + cache_view, designs + ) | self._collect_schema_control_ids(cache_view, dataset_series) + else: + control_ids = set() tree: dict[str, Any] = {} if metadata_config.include_link_keys: @@ -342,6 +409,33 @@ def _collect_control_ids( control_ids.add(op.id) return control_ids + def _collect_schema_control_ids( + self, + cache_view: CacheContainerView, + dataset_series: DatasetSeries[T_DataType], + ) -> set[str]: + """ + Classify `control` observable properties the same way as in + `_collect_control_ids`, but driven by the dataset schemas + instead of the observation designs: a schema element whose + observation design is missing from the cache would otherwise + escape the exclusion despite `exclude_control_variables`. + """ + control_ids: set[str] = set() + for dataset in dataset_series.parts.values(): + if dataset.schema is None: + continue + for element in dataset.schema.elements.values(): + op_id = element.observable_property_id + if op_id is None or op_id in control_ids: + continue + op = cache_view.get(op_id, "ObservableProperty") + if op is None: + continue + if _as_text(op.observation_result_type) == "control": + control_ids.add(op.id) + return control_ids + def _resolve_link_keys( self, cache_view: CacheContainerView, @@ -636,6 +730,15 @@ def _calculation_tree( inputs.append( { "argument": getattr(kwarg, "mapping_name", None), + # execution resolves inputs by observation + property + # (dataops: context_lookup), so both are needed to + # pin the dependency when observations share a + # property. + "source_observation": ( + getattr(reference, "dataset_label", None) + if reference is not None + else None + ), "source_variable": source_id, "source_label": ( getattr(source, "short_name", None) @@ -713,7 +816,11 @@ def _provenance_section( peh_model versions. The validation report is accepted from the caller, never run here; it is embedded in its JSON form. Caller-provided `extra_provenance` fields are appended but may - not override any of these built-in fields. + not override any of these built-in fields, and their values + must be JSON-serializable (finite str / int / float / bool / + None / list / str-keyed dict); NaN, Infinity and + self-referencing containers raise ValueError like any other + unsupported value. """ provenance: dict[str, Any] = { "series_label": dataset_series.label, @@ -753,6 +860,14 @@ def _provenance_section( "StudyMetadataConfig.extra_provenance may not override " f"built-in provenance fields: {', '.join(collisions)}." ) + for key, value in metadata_config.extra_provenance.items(): + if not isinstance(key, str): + raise ValueError( + "StudyMetadataConfig.extra_provenance key of " + f"type {type(key).__name__}; expected str — " + "convert it with str(...)." + ) + _check_json_value(value, key) provenance.update(metadata_config.extra_provenance) return provenance diff --git a/src/pypeh/core/models/metadata_dto.py b/src/pypeh/core/models/metadata_dto.py index d0d4c561..4d2655e9 100644 --- a/src/pypeh/core/models/metadata_dto.py +++ b/src/pypeh/core/models/metadata_dto.py @@ -46,7 +46,11 @@ class StudyMetadataConfig: extra_provenance: dict = field(default_factory=dict) """Caller-provided provenance fields appended to the provenance section (e.g. `source_file`). Keys may not collide with built-in - provenance fields; a collision raises ValueError.""" + provenance fields; a collision raises ValueError. Values must be + JSON-serializable (finite str / int / float / bool / None / + list / str-keyed dict); any other type, a non-finite float + (NaN / Infinity) or a self-referencing container raises + ValueError naming the offending key.""" @classmethod def from_metadata_layout(cls, layout) -> "StudyMetadataConfig": diff --git a/tests/core/interfaces/metadata/test_metadata.py b/tests/core/interfaces/metadata/test_metadata.py index ea456d77..79cabb53 100644 --- a/tests/core/interfaces/metadata/test_metadata.py +++ b/tests/core/interfaces/metadata/test_metadata.py @@ -403,6 +403,7 @@ def test_calculation_tree_links_source_variable( "inputs": [ { "argument": None, + "source_observation": "peh:obs_lab", "source_variable": "peh:prop_chol", "source_label": "chol", "value": None, @@ -411,6 +412,106 @@ def test_calculation_tree_links_source_variable( "results": ["chol_mean"], } + def test_shared_property_inputs_distinguish_observations( + self, metadata_adapter, metadata_workflow + ): + # two calculations on different derived observations reference + # the same property (peh:prop_chol) under their respective + # base observations: without source_observation the inputs + # would not show which dataset column (obs_lab or obs_mid) + # each one reads from. Each contextual kwarg references the + # base observation its calculation is derived from — the + # constraint dataops enforces when such a calculation is + # executed ("All CalculationKeywordArguments should refer to + # specified source observation"). The intermediate observation + # is metadata-only test data: its design carries a bare + # measurement specification, which dataops would reject as an + # enrich target. + session = metadata_workflow[0] + session.cache.add( + peh.ObservableProperty( + id="peh:prop_chol_mean_fu", + short_name="chol_mean_fu", + value_type="decimal", + observation_result_type="calculation", + ) + ) + session.cache.add( + peh.ObservationDesign( + id="peh:od_mid", + observable_property_specifications=[ + peh.ObservablePropertySpecification( + observable_property="peh:prop_chol" + ) + ], + ) + ) + session.cache.add( + peh.ObservationDesign( + id="peh:od_followup", + observable_property_specifications=[ + peh.ObservablePropertySpecification( + observable_property="peh:prop_chol_mean_fu", + calculation_design=peh.CalculationDesign( + calculation_name="mean_chol_fu", + calculation_implementation=( + peh.CalculationImplementation( + function_name="statistics_mean", + function_kwargs=[ + peh.CalculationKeywordArgument( + contextual_field_reference=( + peh.ContextualFieldReference( + dataset_label=( + "peh:obs_mid" + ), + field_label=( + "peh:prop_chol" + ), + ) + ) + ) + ], + ) + ), + ), + ) + ], + ) + ) + session.cache.add( + peh.DerivedObservation( + id="peh:obs_mid", + observation_design="peh:od_mid", + was_derived_from="peh:obs_lab", + ) + ) + session.cache.add( + peh.DerivedObservation( + id="peh:obs_followup", + observation_design="peh:od_followup", + was_derived_from="peh:obs_mid", + ) + ) + tree = _build(metadata_adapter, metadata_workflow) + inputs_by_calculation = { + prop["id"]: prop["calculation"]["inputs"] + for observation in tree["observations"] + for prop in observation["observable_properties"] + if prop["kind"] == "calculation" + } + shared = ( + inputs_by_calculation["peh:prop_chol_mean"] + + inputs_by_calculation["peh:prop_chol_mean_fu"] + ) + assert [item["source_variable"] for item in shared] == [ + "peh:prop_chol", + "peh:prop_chol", + ] + assert [item["source_observation"] for item in shared] == [ + "peh:obs_lab", + "peh:obs_mid", + ] + def test_derived_observation_follows_base_in_lineage( self, metadata_adapter, metadata_workflow ): @@ -517,6 +618,38 @@ def test_no_profile_even_when_data_is_loaded( for variable in tree["datasets"][0]["variables"]: assert "profile" not in variable + def test_control_excluded_even_when_design_missing( + self, metadata_adapter, metadata_workflow, caplog + ): + # the dataset keeps its chol_lod column but the design that + # specifies it is gone: the design-driven control walk never + # sees it, so the schema-driven pass must classify it. + session = metadata_workflow[0] + session.cache.pop("peh:od_lab", "ObservationDesign") + with caplog.at_level( + "WARNING", logger="pypeh.core.interfaces.metadata" + ): + tree = _build(metadata_adapter, metadata_workflow) + (dataset,) = tree["datasets"] + labels = [variable["label"] for variable in dataset["variables"]] + assert labels == ["id_sample", "chol"] # chol_lod still excluded + assert any("peh:od_lab" in record.message for record in caplog.records) + # the observation itself stays tolerated (skipped with warning) + assert [obs["id"] for obs in tree["observations"]] == [ + "peh:obs_summary" + ] + + opted_out = _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig(exclude_control_variables=False), + ) + labels = [ + variable["label"] + for variable in opted_out["datasets"][0]["variables"] + ] + assert "chol_lod" in labels + class TestProvenance: def test_provenance_sections_and_derivation_chain( @@ -560,6 +693,102 @@ def test_extra_provenance_cannot_override_builtin_fields( ), ) + def test_extra_provenance_rejects_non_json_values( + self, metadata_adapter, metadata_workflow + ): + from pathlib import Path + + with pytest.raises(ValueError, match="source_file"): + _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig( + extra_provenance={"source_file": Path("demo.xlsx")} + ), + ) + with pytest.raises(ValueError, match="source_file"): + _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig( + extra_provenance={"source_file": {"tries": {1, 2}}} + ), + ) + with pytest.raises(ValueError, match="key of type int"): + _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig(extra_provenance={1: "spoofed"}), + ) + + def test_extra_provenance_rejects_non_finite_and_circular( + self, metadata_adapter, metadata_workflow + ): + for bad_float in (float("nan"), float("inf"), float("-inf")): + with pytest.raises(ValueError, match="source_file"): + _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig( + extra_provenance={"source_file": bad_float} + ), + ) + cyclic_list = ["demo"] + cyclic_list.append(cyclic_list) + with pytest.raises(ValueError, match="source_file"): + _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig( + extra_provenance={"source_file": cyclic_list} + ), + ) + cyclic_dict = {} + cyclic_dict["self"] = cyclic_dict + with pytest.raises(ValueError, match="source_file"): + _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig( + extra_provenance={"source_file": cyclic_dict} + ), + ) + + def test_shared_references_are_not_circular( + self, metadata_adapter, metadata_workflow + ): + # the same list appearing twice is not a cycle and serializes + # fine, so it must not be flagged. + shared = ["etl", "demo"] + tree = _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig( + extra_provenance={"tags": shared, "origin": shared} + ), + ) + assert tree["provenance"]["tags"] == ["etl", "demo"] + assert tree["provenance"]["origin"] == ["etl", "demo"] + + def test_caller_provenance_serializes_to_json( + self, metadata_adapter, metadata_workflow + ): + import json + + extra = { + "source_file": "demo.xlsx", + "attempt": 2, + "tags": ["etl", "demo"], + "origin": {"tool": "pypeh", "verified": True}, + } + tree = _build( + metadata_adapter, + metadata_workflow, + StudyMetadataConfig(extra_provenance=extra), + ) + assert {key: tree["provenance"][key] for key in extra} == extra + assert json.loads(json.dumps(tree)) == tree + def test_missing_distribution_version_is_none( self, metadata_adapter, metadata_workflow, monkeypatch ): diff --git a/tests/integration/metadata/test_study_metadata.py b/tests/integration/metadata/test_study_metadata.py index c70a4a65..e4a981c1 100644 --- a/tests/integration/metadata/test_study_metadata.py +++ b/tests/integration/metadata/test_study_metadata.py @@ -232,14 +232,17 @@ function: compehndly.entrypoints.medium_bound_imputation inputs: - argument: measurement + source_observation: peh:OBS_LAB_ASSAYS source_variable: chol source_label: chol value: null - argument: loq + source_observation: peh:OBS_LAB_ASSAYS source_variable: chol_loq source_label: chol_loq value: null - argument: lod + source_observation: peh:OBS_LAB_ASSAYS source_variable: chol_lod source_label: chol_lod value: null @@ -264,6 +267,7 @@ function: pypeh.adapters.aggregation.polars_adapter.statistics.statistics_mean inputs: - argument: null + source_observation: peh:OBS_LAB_ENRICHED source_variable: chol_imputed source_label: chol_imputed value: null