From 126cda2109856861f7c1f7744ede74fe332d411c Mon Sep 17 00:00:00 2001 From: noelmcloughlin Date: Fri, 17 Jul 2026 14:03:10 +0100 Subject: [PATCH] feat: optional sssom overlay feature; examples, tests --- CHANGELOG.md | 23 + copier.yaml | 17 + .../workflows/{main.yaml => main.yaml.jinja} | 13 +- template/README.md.jinja | 28 +- template/justfile | 7 + template/pyproject.toml.jinja | 4 + .../README.md.jinja | 67 ++ ...}}-personstatus.sssom.tsv{% endif %}.jinja | 10 + ...ug }}-schemaorg.sssom.tsv{% endif %}.jinja | 14 + ...e %}test_overlay_sssom.py{% endif %}.jinja | 356 ++++++++++ .../overlay_sssom.py | 623 ++++++++++++++++++ ...se_sssom %}sssom.justfile{% endif %}.jinja | 28 + tests/conftest.py | 14 + tests/helpers.py | 1 + tests/test_integration.py | 89 +++ tests/test_options.py | 129 ++++ 16 files changed, 1419 insertions(+), 4 deletions(-) rename template/.github/workflows/{main.yaml => main.yaml.jinja} (68%) create mode 100644 template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/README.md.jinja create mode 100644 template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/{% if add_example %}{{ project_slug }}-personstatus.sssom.tsv{% endif %}.jinja create mode 100644 template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/{% if add_example %}{{ project_slug }}-schemaorg.sssom.tsv{% endif %}.jinja create mode 100644 template/tests/{% if use_sssom and add_example %}test_overlay_sssom.py{% endif %}.jinja create mode 100755 template/{% if use_sssom %}scripts{% endif %}/overlay_sssom.py create mode 100644 template/{% if use_sssom %}sssom.justfile{% endif %}.jinja diff --git a/CHANGELOG.md b/CHANGELOG.md index da48e0b..1c4d8ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,29 @@ The format is inspired by [Keep a Changelog](https://keepachangelog.com/en/1.1.0 [Full changelog](https://github.com/linkml/linkml-project-copier/compare/v0.5.0...main) +### Added + +- New opt-in `use_sssom` prompt (default: no) adding + [SSSOM](https://mapping-commons.github.io/sssom/spec/) support to generated + projects: the `sssom` (sssom-py) and `ruamel.yaml` dev dependencies, a + `src//mappings/` directory for curated `*.sssom.tsv` mapping files + (example TSVs when `add_example` is chosen), and `scripts/overlay_sssom.py` + tool that merges the TSV mappings into the schema's LinkML mapping slots + (`exact_mappings`, `close_mappings`, etc.) idempotently, preserving YAML + comments and formatting. TSV parsing is delegated to sssom-py. +- New `just` recipes for `use_sssom` projects, shipped in an `sssom.justfile` + imported optionally by main justfile: `gen-sssom` (export element mappings + to SSSOM/TSV), `validate-sssom` (CURIE validation of the export), + `overlay-sssom` (apply/preview/check the mappings overlay), and + `test-overlay-sssom` (bundled overlay unit tests; only with `add_example`). +- CI drift gate for `use_sssom` projects: the generated `main.yaml` workflow + runs `just overlay-sssom --check` and fails on schema vs SSSOM TSV drift. + `just setup` applies the overlay so freshly projects start in sync. +- Curated mapping files (`src//mappings/*`) are protected from being + overwritten on template updates via `_skip_if_exists`. +- Integration tests for the `use_sssom` template option (recipes, drift-gate + lifecycle, and setup sync) alongside `use_sssom`/`add_example` combinations. + ## Release [0.5.0] - 2026-05-27 [Full changelog](https://github.com/linkml/linkml-project-copier/compare/v0.4.2...v0.5.0) diff --git a/copier.yaml b/copier.yaml index bd0587d..b1d71b7 100644 --- a/copier.yaml +++ b/copier.yaml @@ -104,6 +104,17 @@ gh_action_docs_preview: type: bool default: true +use_sssom: + help: | + Add support for SSSOM (Simple Standard for Sharing Ontology Mappings)? + This adds the "sssom" (sssom-py) and "ruamel.yaml" dev dependencies, a + src//mappings/ directory for curated *.sssom.tsv files, the + scripts/overlay_sssom.py tool to project those mappings into the schema's + LinkML mapping slots, and `just` recipes: gen-sssom, validate-sssom, + overlay-sssom, and test-overlay-sssom. + type: bool + default: false + # === copier configuration options === # https://copier.readthedocs.io/en/stable/configuring/#available-settings @@ -129,6 +140,7 @@ _skip_if_exists: - examples/README.md - project/* - src/{{project_slug}}/datamodel/* + - src/{{project_slug}}/mappings/* - src/{{project_slug}}/schema/* - tests/test_data.py - tests/data/* @@ -139,6 +151,11 @@ _message_after_copy: | Next step (if you have not previously initialized your project) run "just setup" in the root of your project directory. + {%- if use_sssom %} + + Note: "just setup" also runs "just overlay-sssom" to sync the SSSOM + mappings into the schema; CI checks this stays in sync (drift gate). + {%- endif %} _migrations: # Migrations are only run on update (not on copy) and only if the update goes diff --git a/template/.github/workflows/main.yaml b/template/.github/workflows/main.yaml.jinja similarity index 68% rename from template/.github/workflows/main.yaml rename to template/.github/workflows/main.yaml.jinja index ded0eac..38d8ba2 100644 --- a/template/.github/workflows/main.yaml +++ b/template/.github/workflows/main.yaml.jinja @@ -34,15 +34,15 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v8.2.0 with: - python-version: ${{ matrix.python-version }} + {% raw %}python-version: ${{ matrix.python-version }}{% endraw %} enable-cache: true cache-dependency-glob: "uv.lock" # https://github.com/actions/setup-python - - name: Set up Python ${{ matrix.python-version }} + - name: {% raw %}Set up Python ${{ matrix.python-version }}{% endraw %} uses: actions/setup-python@v6.3.0 with: - python-version: ${{ matrix.python-version }} + {% raw %}python-version: ${{ matrix.python-version }}{% endraw %} - name: Install just run: | @@ -53,3 +53,10 @@ jobs: - name: Run test suite run: just test +{%- if use_sssom %} + + # Fails when src//mappings/*.sssom.tsv and the schema have + # drifted apart; run "just overlay-sssom" locally to re-sync. + - name: Check SSSOM mappings are in sync with the schema + run: just overlay-sssom --check +{%- endif %} diff --git a/template/README.md.jinja b/template/README.md.jinja index e2e5c1c..0dcc80f 100644 --- a/template/README.md.jinja +++ b/template/README.md.jinja @@ -20,6 +20,10 @@ (edit this) * [datamodel/](src/{{project_slug}}/datamodel) -- generated Python datamodel +{%- if use_sssom %} + * [mappings/](src/{{project_slug}}/mappings) -- SSSOM/TSV + mapping files (`*.sssom.tsv`) +{%- endif %} * [tests/](tests/) - Python tests * [data/](tests/data) - Example data @@ -28,7 +32,29 @@ There are several pre-defined command-recipes available. They are written for the command runner [just](https://github.com/casey/just/). To list all pre-defined commands, run `just` or `just --list`. - +{% if use_sssom %} +## SSSOM Mappings + +Cross-references between schema elements and external vocabularies are +maintained as [SSSOM/TSV](https://mapping-commons.github.io/sssom/spec/) +files under [src/{{project_slug}}/mappings/](src/{{project_slug}}/mappings). +[`scripts/overlay_sssom.py`](scripts/overlay_sssom.py) merges every +`*.sssom.tsv` into the matching LinkML mapping slot on the matching +schema element, and is idempotent. Multi-module schemas under +[src/{{project_slug}}/schema/](src/{{project_slug}}/schema) are +discovered recursively. + +```sh +just overlay-sssom # apply +just overlay-sssom --dry-run # preview without writing +just overlay-sssom --check # CI gate: exit 1 if schema is out of sync +{% if add_example %}just test-overlay-sssom # run the bundled unit test +{% endif %}``` + +Run `python scripts/overlay_sssom.py --help` for the full set of flags, +overrides, and a description of how predicates, subject prefixes, and +permissible-value `meaning` promotion are handled. +{% endif %} ## Credits This project uses the template [linkml-project-copier](https://github.com/linkml/linkml-project-copier). diff --git a/template/justfile b/template/justfile index 3bec5df..15cf8e8 100644 --- a/template/justfile +++ b/template/justfile @@ -80,6 +80,10 @@ setup: _wsl2_compat_check _check-config _git-init install _git-add && _setup_par git commit -m "Initialise git with minimal project" -a || true _setup_part2: gen-project gen-doc + @if [ -f sssom.justfile ]; then \ + echo 'Applying SSSOM mappings to the schema ...'; \ + just overlay-sssom; \ + fi @echo @echo '=== Setup completed! ===' @echo 'Various model representations have been created under directory "project". By default' @@ -285,6 +289,9 @@ _ensure_examples_output: # Ensure a clean examples/output directory exists import "project.justfile" +# SSSOM mapping recipes (present only if this project enabled `use_sssom`) +import? "sssom.justfile" + # ====== Override recipes from above with custom versions ======= # Uncomment the following line to allow duplicate recipe names diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 3d37f35..c4cd901 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -26,6 +26,10 @@ dev = [ "mkdocs-pymdownx-material-extras>=2.5.6", "jupyter>=1.0.0", "mknotebooks>= 0.8.0", +{%- if use_sssom %} + "sssom>=0.4.0", + "ruamel.yaml>=0.18", +{%- endif %} ] # See https://hatch.pypa.io/latest/config/build/#file-selection for how to diff --git a/template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/README.md.jinja b/template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/README.md.jinja new file mode 100644 index 0000000..0f1e009 --- /dev/null +++ b/template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/README.md.jinja @@ -0,0 +1,67 @@ +# Mappings Directory + +This directory stores cross-references from `{{project_slug}}` schema to external +vocabularies as [SSSOM/TSV](https://mapping-commons.github.io/sssom/spec/) files. + +The TSV files are the source of truth, and `scripts/overlay_sssom.py` is the +compiler that replays those mappings into LinkML schema mapping slots. + +## How The Overlay Script Works + +When you run `just overlay-sssom`, the script: + +1. Recursively reads every `*.sssom.tsv` under this directory. +2. For each row, maps `predicate_id` to the correct LinkML slot. +3. Finds the matching schema element by `subject_id` local name. +4. Appends new CURIEs without duplicating existing ones. +5. Adds missing object-side prefixes to schema `prefixes:` from `#curie_map:` metadata. + +It is idempotent: running it again without TSV changes does nothing. + +## What A Row Targets + +Use these subject formats: + +- `{{project_slug}}:` for class, slot, enum, type, or attribute. +- `{{project_slug}}:/` for a permissible value. + +Subject prefixes are discovered from schema `default_prefix` and `name`. +Use `--subject-prefix` only for aliases. + +## Predicate Mapping Rules + +- `skos:exactMatch` -> `exact_mappings` +- `skos:closeMatch` -> `close_mappings` +- `skos:broadMatch` -> `broad_mappings` +- `skos:narrowMatch` -> `narrow_mappings` +- `skos:relatedMatch` -> `related_mappings` +- `owl:equivalentClass` -> `exact_mappings` +- `owl:equivalentProperty` -> `exact_mappings` + +Ignored: `skos:broader`, `skos:narrower`, `rdf:type`, and unsupported predicates. + +## Permissible Value Rule + +For permissible values, the first `skos:exactMatch` is promoted to `meaning` if +`meaning` is not already set. Additional exact matches are in `exact_mappings`. + +## Required TSV Fields + +Required columns are: + +- `subject_id` +- `predicate_id` +- `object_id` + +Include `#curie_map:` in metadata for every non-builtin prefix used in the file +(project prefix and external prefixes like `schema`, `NCIT`). + +## Recommended Workflow + +```sh +just overlay-sssom --dry-run +just overlay-sssom +just overlay-sssom --check +``` + +Use `--check` in CI to fail when TSV and schema are out of sync. diff --git a/template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/{% if add_example %}{{ project_slug }}-personstatus.sssom.tsv{% endif %}.jinja b/template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/{% if add_example %}{{ project_slug }}-personstatus.sssom.tsv{% endif %}.jinja new file mode 100644 index 0000000..3fbd0c8 --- /dev/null +++ b/template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/{% if add_example %}{{ project_slug }}-personstatus.sssom.tsv{% endif %}.jinja @@ -0,0 +1,10 @@ +#mapping_set_id: https://w3id.org/{{github_org}}/{{project_name}}/mappings/{{project_slug}}-personstatus.sssom.tsv +#mapping_set_description: Example permissible-value mappings for PersonStatus to NCIT terms. +#curie_map: +# {{project_slug}}: https://w3id.org/{{github_org}}/{{project_name}}/ +# NCIT: http://purl.obolibrary.org/obo/NCIT_ +#license: https://creativecommons.org/publicdomain/zero/1.0/ +subject_id predicate_id object_id mapping_justification comment +{{project_slug}}:PersonStatus/ALIVE skos:exactMatch NCIT:C37987 semapv:ManualMappingCuration ALIVE already has meaning PATO:0001421; this lands in exact_mappings instead. +{{project_slug}}:PersonStatus/DEAD skos:closeMatch NCIT:C28554 semapv:LLMBasedMatching Close mapping is independent of the existing meaning slot. +{{project_slug}}:PersonStatus/UNKNOWN skos:exactMatch NCIT:C17998 semapv:TransformerBasedMatching UNKNOWN has no meaning yet; the first exact mapping is promoted to meaning. diff --git a/template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/{% if add_example %}{{ project_slug }}-schemaorg.sssom.tsv{% endif %}.jinja b/template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/{% if add_example %}{{ project_slug }}-schemaorg.sssom.tsv{% endif %}.jinja new file mode 100644 index 0000000..90d0785 --- /dev/null +++ b/template/src/{{project_slug}}/{% if use_sssom %}mappings{% endif %}/{% if add_example %}{{ project_slug }}-schemaorg.sssom.tsv{% endif %}.jinja @@ -0,0 +1,14 @@ +#mapping_set_id: https://w3id.org/{{github_org}}/{{project_name}}/mappings/{{project_slug}}-schemaorg.sssom.tsv +#mapping_set_description: Example mappings of demo schema entities to schema.org terms (classes and slots). +#curie_map: +# {{project_slug}}: https://w3id.org/{{github_org}}/{{project_name}}/ +# schema: http://schema.org/ +#license: https://creativecommons.org/publicdomain/zero/1.0/ +#mapping_provider: http://schema.org/ +subject_id predicate_id object_id mapping_justification +{{project_slug}}:NamedThing skos:exactMatch schema:Thing semapv:ManualMappingCuration +{{project_slug}}:Person skos:exactMatch schema:Person semapv:MachineLearningBasedMatching +{{project_slug}}:PersonCollection skos:closeMatch schema:ItemList semapv:EmbeddingBasedMatching +{{project_slug}}:primary_email skos:exactMatch schema:email semapv:ManualMappingCuration +{{project_slug}}:birth_date skos:exactMatch schema:birthDate semapv:GraphRepresentationLearningBasedMatching +{{project_slug}}:age_in_years skos:relatedMatch schema:Number semapv:ManualMappingCuration diff --git a/template/tests/{% if use_sssom and add_example %}test_overlay_sssom.py{% endif %}.jinja b/template/tests/{% if use_sssom and add_example %}test_overlay_sssom.py{% endif %}.jinja new file mode 100644 index 0000000..57aa18c --- /dev/null +++ b/template/tests/{% if use_sssom and add_example %}test_overlay_sssom.py{% endif %}.jinja @@ -0,0 +1,356 @@ +"""Unit tests for the SSSOM overlay script. + +Copies the example schema and the bundled ``*.sssom.tsv`` files into a +temporary tree, applies the overlay, and asserts that mappings landed in +the expected LinkML slots. The script under test lives at +``scripts/overlay_sssom.py`` and is loaded as a module via +``importlib`` so the tests run without packaging it. +""" +import importlib.util +import shutil +import sys +from pathlib import Path + +import pytest + +# Skip the whole file when ruamel.yaml isn't installed; the script +# unconditionally imports it at module load time. +pytest.importorskip("ruamel.yaml") +from ruamel.yaml import YAML # noqa: E402 + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +SCRIPT_PATH = PROJECT_ROOT / "scripts" / "overlay_sssom.py" +SCHEMA_PATH = ( + PROJECT_ROOT / "src" / "{{project_slug}}" / "schema" / "{{project_slug}}.yaml" +) +MAPPINGS_DIR = PROJECT_ROOT / "src" / "{{project_slug}}" / "mappings" + + +def _load_overlay_module(): + """Import ``scripts/overlay_sssom.py`` as the ``overlay_sssom`` module.""" + spec = importlib.util.spec_from_file_location( + "overlay_sssom", SCRIPT_PATH, + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules["overlay_sssom"] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def overlay(): + """The imported overlay_sssom module.""" + try: + yield _load_overlay_module() + finally: + sys.modules.pop("overlay_sssom", None) + + +@pytest.fixture +def scratch(tmp_path): + """Copy the example schema and mappings into an isolated tmp tree. + + Returns ``(schema_path, mappings_dir)``. + """ + schema_dst = tmp_path / "schema" + mappings_dst = tmp_path / "mappings" + shutil.copytree(SCHEMA_PATH.parent, schema_dst) + shutil.copytree(MAPPINGS_DIR, mappings_dst) + return schema_dst / SCHEMA_PATH.name, mappings_dst + + +def _load_yaml(path): + y = YAML(typ="rt") + with open(path, "r", encoding="utf-8") as fh: + return y.load(fh) + + +def test_example_assets_exist(): + """The example schema and at least one SSSOM TSV must be present.""" + assert SCRIPT_PATH.is_file(), f"missing {SCRIPT_PATH}" + assert SCHEMA_PATH.is_file(), f"missing {SCHEMA_PATH}" + tsvs = list(MAPPINGS_DIR.glob("*.sssom.tsv")) + assert tsvs, f"no *.sssom.tsv files in {MAPPINGS_DIR}" + + +def test_overlay_applies_class_and_slot_mappings(overlay, scratch): + schema_path, mappings_dir = scratch + mappings = overlay.load_mappings(mappings_dir) + assert mappings.rows, "no mapping rows loaded" + + elements, links = overlay.overlay_file( + schema_path, mappings, extra_subject_prefixes=set(), + ) + assert elements > 0 + assert links > 0 + + data = _load_yaml(schema_path) + person = data["classes"]["Person"] + assert "schema:Person" in list(person.get("exact_mappings", [])) + + named_thing = data["classes"]["NamedThing"] + assert "schema:Thing" in list(named_thing.get("exact_mappings", [])) + + person_collection = data["classes"]["PersonCollection"] + assert "schema:ItemList" in list(person_collection.get("close_mappings", [])) + + email = data["slots"]["primary_email"] + assert "schema:email" in list(email.get("exact_mappings", [])) + + age = data["slots"]["age_in_years"] + assert "schema:Number" in list(age.get("related_mappings", [])) + + +def test_overlay_promotes_pv_meaning_and_preserves_existing(overlay, scratch): + schema_path, mappings_dir = scratch + mappings = overlay.load_mappings(mappings_dir) + overlay.overlay_file( + schema_path, mappings, extra_subject_prefixes=set(), + ) + + data = _load_yaml(schema_path) + pvs = data["enums"]["PersonStatus"]["permissible_values"] + + # UNKNOWN had no meaning; the script promotes the first exact CURIE + # from the SSSOM TSV into the meaning slot. + assert pvs["UNKNOWN"].get("meaning") == "NCIT:C17998" + + # ALIVE already has meaning PATO:0001421 (unchanged); the exact + # mapping to NCIT:C37987 is appended to exact_mappings. + assert pvs["ALIVE"].get("meaning") == "PATO:0001421" + assert "NCIT:C37987" in list(pvs["ALIVE"].get("exact_mappings", [])) + + # DEAD already has meaning PATO:0001422 (unchanged); the close + # mapping to NCIT:C28554 lands in close_mappings. + assert pvs["DEAD"].get("meaning") == "PATO:0001422" + assert "NCIT:C28554" in list(pvs["DEAD"].get("close_mappings", [])) + + +def test_overlay_injects_referenced_prefixes(overlay, scratch): + schema_path, mappings_dir = scratch + mappings = overlay.load_mappings(mappings_dir) + overlay.overlay_file( + schema_path, mappings, extra_subject_prefixes=set(), + ) + + data = _load_yaml(schema_path) + prefixes = data["prefixes"] + # NCIT is used by the personstatus TSV but not pre-declared. + assert "NCIT" in prefixes + assert prefixes["NCIT"].startswith("http://purl.obolibrary.org/obo/NCIT_") + # `schema` was already declared and must be retained. + assert "schema" in prefixes + + +def test_overlay_is_idempotent(overlay, scratch): + schema_path, mappings_dir = scratch + mappings = overlay.load_mappings(mappings_dir) + overlay.overlay_file( + schema_path, mappings, extra_subject_prefixes=set(), + ) + elements, links = overlay.overlay_file( + schema_path, mappings, extra_subject_prefixes=set(), + ) + assert (elements, links) == (0, 0) + + +def test_overlay_dry_run_does_not_write(overlay, scratch): + schema_path, mappings_dir = scratch + original = schema_path.read_bytes() + mappings = overlay.load_mappings(mappings_dir) + elements, links = overlay.overlay_file( + schema_path, mappings, extra_subject_prefixes=set(), dry_run=True, + ) + assert elements > 0 and links > 0 + assert schema_path.read_bytes() == original + + +def test_check_reports_missing_before_overlay(overlay, scratch): + schema_path, mappings_dir = scratch + mappings = overlay.load_mappings(mappings_dir) + diagnostics = overlay.check_file( + schema_path, mappings, extra_subject_prefixes=set(), + ) + # Every example TSV row is initially unapplied; expect one + # diagnostic per row plus one for the undeclared NCIT prefix. + assert any("Person" in d and "schema:Person" in d for d in diagnostics) + assert any( + "permissible_value" in d and "UNKNOWN" in d and "NCIT:C17998" in d + for d in diagnostics + ) + assert any( + "prefix 'NCIT'" in d and "not declared" in d for d in diagnostics + ) + + +def test_check_passes_after_overlay(overlay, scratch): + schema_path, mappings_dir = scratch + mappings = overlay.load_mappings(mappings_dir) + overlay.overlay_file( + schema_path, mappings, extra_subject_prefixes=set(), + ) + diagnostics = overlay.check_file( + schema_path, mappings, extra_subject_prefixes=set(), + ) + assert diagnostics == [] + + +def test_check_cli_exits_nonzero_when_out_of_sync(overlay, scratch): + schema_path, mappings_dir = scratch + rc = overlay.main([ + "--schema", str(schema_path), + "--mappings-dir", str(mappings_dir), + "--check", + ]) + assert rc == 1 + + +def test_check_cli_exits_zero_when_in_sync(overlay, scratch): + schema_path, mappings_dir = scratch + overlay.main([ + "--schema", str(schema_path), + "--mappings-dir", str(mappings_dir), + ]) + rc = overlay.main([ + "--schema", str(schema_path), + "--mappings-dir", str(mappings_dir), + "--check", + ]) + assert rc == 0 + + +# --------------------------------------------------------------------------- +# Multi-module / nested-subdirectory schemas +# --------------------------------------------------------------------------- + +_MODULE_CORE = """\ +id: https://example.org/multi/core +name: multi_core +prefixes: + multi: https://example.org/multi/ + linkml: https://w3id.org/linkml/ +default_prefix: multi +default_range: string +imports: + - linkml:types +classes: + Widget: + description: A widget. + attributes: + label: + range: string +""" + +_MODULE_NESTED = """\ +id: https://example.org/multi/ext/gadget +name: multi_ext +prefixes: + multi_ext: https://example.org/multi/ext/ + linkml: https://w3id.org/linkml/ +default_prefix: multi_ext +default_range: string +imports: + - linkml:types +classes: + Gadget: + description: A gadget defined in a nested module. +""" + +_NON_LINKML_YAML = """\ +# Not a LinkML schema; should be silently skipped. +site_name: docs +nav: + - Home: index.md +""" + +_MULTI_SSSOM = """\ +#curie_map: +# multi: https://example.org/multi/ +# multi_ext: https://example.org/multi/ext/ +# schema: http://schema.org/ +subject_id\tpredicate_id\tobject_id +multi:Widget\tskos:exactMatch\tschema:Thing +multi_ext:Gadget\tskos:exactMatch\tschema:Product +multi:NotInAnyModule\tskos:exactMatch\tschema:Other +""" + + +@pytest.fixture +def multi_module_tree(tmp_path): + """Two-module schema (root + nested subdir) plus a non-LinkML YAML.""" + schema_dir = tmp_path / "schema" + nested_dir = schema_dir / "ext" + mappings_dir = tmp_path / "mappings" + nested_dir.mkdir(parents=True) + mappings_dir.mkdir() + (schema_dir / "core.yaml").write_text(_MODULE_CORE, encoding="utf-8") + (nested_dir / "gadget.yaml").write_text(_MODULE_NESTED, encoding="utf-8") + # Decoy YAML that must be silently skipped. + (schema_dir / "mkdocs.yaml").write_text(_NON_LINKML_YAML, encoding="utf-8") + (mappings_dir / "multi.sssom.tsv").write_text( + _MULTI_SSSOM, encoding="utf-8", + ) + return schema_dir, mappings_dir + + +def test_multi_module_overlay_routes_to_correct_module( + overlay, multi_module_tree, +): + schema_dir, mappings_dir = multi_module_tree + rc = overlay.main([ + "--schema-dir", str(schema_dir), + "--mappings-dir", str(mappings_dir), + ]) + assert rc == 0 + core = _load_yaml(schema_dir / "core.yaml") + nested = _load_yaml(schema_dir / "ext" / "gadget.yaml") + assert "schema:Thing" in list( + core["classes"]["Widget"].get("exact_mappings", []) + ) + # Nested-module subject `multi_ext:Gadget` must land in the nested + # file, NOT in the core file. + assert "schema:Product" in list( + nested["classes"]["Gadget"].get("exact_mappings", []) + ) + assert "schema:Product" not in list( + core["classes"]["Widget"].get("exact_mappings", []) + ) + + +def test_multi_module_check_dedups_unresolved_across_files( + overlay, multi_module_tree, capsys, +): + schema_dir, mappings_dir = multi_module_tree + rc = overlay.main([ + "--schema-dir", str(schema_dir), + "--mappings-dir", str(mappings_dir), + "--check", + ]) + # Verification fails (3 rows initially missing + 1 unresolved subject). + assert rc == 1 + captured = capsys.readouterr() + out = captured.out + captured.err + # `multi:NotInAnyModule` truly has no match anywhere -> exactly one + # unresolved-subject diagnostic, not one per file. + assert out.count("NotInAnyModule") == 1 + # Multi-file output uses paths relative to the schema root. + assert "ext/gadget.yaml" in out + # The decoy YAML never appears in diagnostics. + assert "mkdocs.yaml" not in out + + +def test_multi_module_check_passes_after_apply(overlay, multi_module_tree): + schema_dir, mappings_dir = multi_module_tree + overlay.main([ + "--schema-dir", str(schema_dir), + "--mappings-dir", str(mappings_dir), + ]) + rc = overlay.main([ + "--schema-dir", str(schema_dir), + "--mappings-dir", str(mappings_dir), + "--check", + ]) + # Still 1 because the truly-unresolved subject persists; no + # per-row diagnostics remain. + assert rc == 1 diff --git a/template/{% if use_sssom %}scripts{% endif %}/overlay_sssom.py b/template/{% if use_sssom %}scripts{% endif %}/overlay_sssom.py new file mode 100755 index 0000000..e1ebaec --- /dev/null +++ b/template/{% if use_sssom %}scripts{% endif %}/overlay_sssom.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 +"""Overlay SSSOM mappings onto LinkML schema YAML files. + +For every ``*.sssom.tsv`` under ``--mappings-dir``, merges each row's +object CURIE into the matching LinkML mapping slot (``exact_mappings``, +``close_mappings``, ``broad_mappings``, ``narrow_mappings``, +``related_mappings``) on every element (class / slot / enum / type / +attribute / permissible value) whose local name matches the row +subject. The file is rewritten with ruamel.yaml round-trip mode so +comments and styling are preserved; the operation is idempotent. + +Layout assumptions (good-path only): + * TSV ``#``-metadata is valid YAML with a ``curie_map:`` block. + * Subjects are written as ``:`` or + ``:/`` (slash-separated permissible values). + * Subject prefixes come from each schema's own ``default_prefix`` + and ``name``; extend with repeated ``--subject-prefix`` if needed. + * When invoked with no path flags, discovers + ``/src//{schema,mappings}`` relative to this script. + +Typical usage (justfile ``overlay-sssom`` target):: + + python scripts/overlay_sssom.py # apply + python scripts/overlay_sssom.py --dry-run # preview + python scripts/overlay_sssom.py --check # CI verification gate +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedMap, CommentedSeq +from sssom.parsers import parse_sssom_table + + +SSSOM_PREDICATE_TO_LINKML_SLOT: dict[str, str] = { + "skos:exactMatch": "exact_mappings", + "skos:closeMatch": "close_mappings", + "skos:broadMatch": "broad_mappings", + "skos:narrowMatch": "narrow_mappings", + "skos:relatedMatch": "related_mappings", + "owl:equivalentClass": "exact_mappings", + "owl:equivalentProperty": "exact_mappings", +} + +_IGNORED_PREDICATES = frozenset({"skos:broader", "skos:narrower", "rdf:type"}) + +_BUILTIN_PREFIXES = frozenset({ + "sssom", "owl", "rdf", "rdfs", "skos", "semapv", + "linkml", "xsd", "dcterms", +}) + +_MAPPING_SLOT_ORDER = ( + "exact_mappings", "close_mappings", "broad_mappings", + "narrow_mappings", "related_mappings", +) + +# Keys that conventionally follow the mapping slots; a freshly-inserted +# mapping slot (or PV ``meaning``) is placed just before the first of +# these that exists in the element body. +_POST_MAPPING_ANCHORS = ( + "aliases", "in_subset", "permissible_values", "attributes", "slots", + "slot_usage", "rules", "comments", "annotations", "pattern", +) + +_ELEMENT_SECTIONS = ("classes", "slots", "enums", "types") + +_SECTION_KIND = {"classes": "class", "slots": "slot", + "enums": "enum", "types": "type"} + +# YAML files under the schema dir that carry none of these are treated +# as non-LinkML (mkdocs.yml, GitHub Actions, etc.) and silently skipped. +_LINKML_SCHEMA_KEYS = frozenset({ + "id", "name", "prefixes", "default_prefix", "default_range", + "imports", "classes", "slots", "enums", "types", "subsets", +}) + + +def _make_yaml() -> YAML: + y = YAML(typ="rt") + y.preserve_quotes = True + y.width = 4096 + y.indent(mapping=2, sequence=4, offset=2) + return y + + +def _looks_like_linkml_schema(data: object) -> bool: + return (isinstance(data, CommentedMap) + and bool(_LINKML_SCHEMA_KEYS & set(data.keys()))) + + +# --------------------------------------------------------------------------- +# Mapping index +# --------------------------------------------------------------------------- + +class MappingIndex: + """Raw SSSOM rows plus a prefix-filtered ``by_name`` view.""" + + def __init__(self) -> None: + # subject_local_name -> mapping_slot -> ordered [object CURIE, ...] + self.by_name: dict[str, dict[str, list[str]]] = {} + # CURIE prefix -> URI, from each TSV's ``#curie_map:`` block. + self.prefix_uris: dict[str, str] = {} + # (subject_curie, slot, object_curie) triples from all TSVs. + self.rows: list[tuple[str, str, str]] = [] + + def build_for_prefixes(self, prefixes: set[str]) -> None: + self.by_name.clear() + with_colon = tuple(f"{p}:" for p in prefixes) + for subject, slot, obj in self.rows: + local: str | None = None + for px in with_colon: + if subject.startswith(px): + local = subject[len(px):] + break + if local is None: + continue + entries = self.by_name.setdefault(local, {}).setdefault(slot, []) + if obj not in entries: + entries.append(obj) + + def used_prefixes_for(self, name: str) -> set[str]: + return {c.split(":", 1)[0] + for curies in self.by_name.get(name, {}).values() + for c in curies if ":" in c} + + +def load_mappings(mappings_dir: Path) -> MappingIndex: + """Load every ``*.sssom.tsv`` under ``mappings_dir`` (recursive). + + Parsing (TSV rows, ``#``-metadata, and ``curie_map``) is delegated to + sssom-py, which also drops malformed rows with a logged warning instead + of failing the whole file. + """ + idx = MappingIndex() + for tsv in sorted(mappings_dir.rglob("*.sssom.tsv")): + msdf = parse_sssom_table(tsv) + + for px, uri in msdf.prefix_map.items(): + if px in _BUILTIN_PREFIXES: + continue + idx.prefix_uris.setdefault(str(px), str(uri)) + + columns = set(msdf.df.columns) + if not {"subject_id", "predicate_id", "object_id"} <= columns: + continue + for row in msdf.df.itertuples(index=False): + subject = str(row.subject_id).strip() + predicate = str(row.predicate_id).strip() + obj = str(row.object_id).strip() + if not subject or not obj or predicate in _IGNORED_PREDICATES: + continue + slot = SSSOM_PREDICATE_TO_LINKML_SLOT.get(predicate) + if slot is None: + continue + idx.rows.append((subject, slot, obj)) + return idx + + +# --------------------------------------------------------------------------- +# Schema helpers +# --------------------------------------------------------------------------- + +def discover_subject_prefixes(data: CommentedMap) -> set[str]: + """Return the CURIE prefixes that identify the schema itself.""" + out: set[str] = set() + for key in ("default_prefix", "name"): + v = data.get(key) + if isinstance(v, str) and v: + out.add(v) + return out + + +def _insert_before_anchors(body: CommentedMap, key: str, value) -> None: + """Insert ``key`` before the first post-mapping anchor; else append.""" + if key in body: + body[key] = value + return + keys = list(body.keys()) + for anchor in _POST_MAPPING_ANCHORS: + if anchor in keys: + body.insert(keys.index(anchor), key, value) + return + body[key] = value + + +def _merge_mappings( + body: CommentedMap, + slot_map: dict[str, list[str]], + *, + is_permissible_value: bool = False, +) -> tuple[bool, int]: + """Merge ``slot_map`` into ``body``; return ``(touched, links_added)``. + + For permissible values, the first ``exact_mappings`` CURIE is + promoted to ``meaning`` when meaning is unset; the remaining exact + CURIEs still land in ``exact_mappings``. + """ + touched = False + links_added = 0 + for slot in _MAPPING_SLOT_ORDER: + curies = slot_map.get(slot) + if not curies: + continue + + if is_permissible_value and slot == "exact_mappings": + remaining = list(curies) + meaning = body.get("meaning") + if not meaning: + _insert_before_anchors(body, "meaning", remaining.pop(0)) + links_added += 1 + touched = True + else: + remaining = [c for c in remaining if c != meaning] + if not remaining: + continue + curies = remaining + + existing = body.get(slot) + if existing is None: + _insert_before_anchors(body, slot, CommentedSeq(curies)) + links_added += len(curies) + touched = True + continue + new_list = list(existing) + added = [c for c in curies if c not in new_list] + if added: + body[slot] = CommentedSeq(new_list + added) + links_added += len(added) + touched = True + return touched, links_added + + +def _ensure_prefixes( + data: CommentedMap, + needed: set[str], + prefix_uris: dict[str, str], + own_prefixes: set[str], +) -> bool: + prefixes = data.get("prefixes") + if not isinstance(prefixes, dict): + prefixes = CommentedMap() + data["prefixes"] = prefixes + changed = False + for px in sorted(needed): + if (px in _BUILTIN_PREFIXES or px in own_prefixes + or px in prefixes): + continue + uri = prefix_uris.get(px) + if uri is None: + continue + prefixes[px] = uri + changed = True + return changed + + +def _iter_targets( + data: CommentedMap, + by_name: dict[str, dict[str, list[str]]], +): + """Yield ``(kind, display_name, body, slot_map, is_pv, subject)`` for + each element in ``data`` that has a matching mapping in ``by_name``. + ``subject`` is the SSSOM local name used to look up ``slot_map``. + """ + for section in _ELEMENT_SECTIONS: + collection = data.get(section) + if not isinstance(collection, dict): + continue + kind = _SECTION_KIND[section] + for name, body in collection.items(): + if not isinstance(body, CommentedMap): + continue + slot_map = by_name.get(name) + if slot_map: + yield kind, name, body, slot_map, False, name + if section == "classes": + attrs = body.get("attributes") + if not isinstance(attrs, dict): + continue + for aname, abody in attrs.items(): + if not isinstance(abody, CommentedMap): + continue + asm = by_name.get(aname) + if asm: + yield ("attribute", f"{name}.{aname}", + abody, asm, False, aname) + + enums = data.get("enums") + if not isinstance(enums, dict): + return + for subject, slot_map in by_name.items(): + if "/" not in subject: + continue + enum_name, _, pv_name = subject.partition("/") + enum_body = enums.get(enum_name) + if not isinstance(enum_body, CommentedMap): + continue + pvs = enum_body.get("permissible_values") + if not isinstance(pvs, dict): + continue + pv_body = pvs.get(pv_name) + if isinstance(pv_body, CommentedMap): + yield ("permissible_value", pv_name, pv_body, slot_map, + True, subject) + + +# --------------------------------------------------------------------------- +# Overlay +# --------------------------------------------------------------------------- + +def overlay_file( + schema_path: Path, + mappings: MappingIndex, + extra_subject_prefixes: set[str], + *, + dry_run: bool = False, + display_path: str | None = None, # accepted for CLI symmetry; unused here +) -> tuple[int, int]: + """Overlay ``mappings`` onto one schema YAML in place. + + Returns ``(elements_updated, links_added)``. The file is rewritten + only when at least one element was modified or a new prefix was + declared (and ``dry_run`` is False). + """ + del display_path # not needed once diagnostics are suppressed + y = _make_yaml() + with open(schema_path, encoding="utf-8") as fh: + data = y.load(fh) + if not _looks_like_linkml_schema(data): + return 0, 0 + + subject_prefixes = discover_subject_prefixes(data) | extra_subject_prefixes + if not subject_prefixes: + return 0, 0 + mappings.build_for_prefixes(subject_prefixes) + if not mappings.by_name: + return 0, 0 + + elements_updated = 0 + links_added = 0 + used_prefixes: set[str] = set() + for _kind, _disp, body, slot_map, is_pv, subject in _iter_targets( + data, mappings.by_name, + ): + touched, n_added = _merge_mappings( + body, slot_map, is_permissible_value=is_pv, + ) + if touched: + elements_updated += 1 + links_added += n_added + used_prefixes |= mappings.used_prefixes_for(subject) + + prefixes_changed = _ensure_prefixes( + data, used_prefixes, mappings.prefix_uris, subject_prefixes, + ) + + if (elements_updated or prefixes_changed) and not dry_run: + with open(schema_path, "w", encoding="utf-8") as fh: + y.dump(data, fh) + return elements_updated, links_added + + +# --------------------------------------------------------------------------- +# Verification (read-only) +# --------------------------------------------------------------------------- + +def _row_is_present( + body: CommentedMap, slot: str, curie: str, + *, is_pv: bool, pv_promoted: set[str], +) -> bool: + """True iff ``(slot, curie)`` is already reflected in ``body``. + + For PV ``exact_mappings`` rows, a CURIE counts as present if it + matches ``meaning``. When several exact rows target a PV with no + meaning yet, only the first is treated as the (future) promotion; + the caller passes ``pv_promoted`` to keep that state across rows. + """ + existing = list(body.get(slot) or []) + if curie in existing: + return True + if is_pv and slot == "exact_mappings": + meaning = body.get("meaning") + if meaning and curie == meaning: + return True + if not meaning and not pv_promoted: + pv_promoted.add(curie) + return False + + +def _relative_display(path: Path, root: Path | None) -> str: + if root is None: + return path.name + try: + return path.relative_to(root).as_posix() + except ValueError: + return path.name + + +def _diagnose( + data: CommentedMap, + display: str, + by_name: dict[str, dict[str, list[str]]], + subject_prefixes: set[str], + known_subjects: set[str], +) -> list[str]: + diagnostics: list[str] = [] + matched: set[str] = set() + for kind, name, body, slot_map, is_pv, subject in _iter_targets( + data, by_name, + ): + matched.add(subject) + promoted: set[str] = set() + for slot in _MAPPING_SLOT_ORDER: + for curie in slot_map.get(slot) or (): + if not _row_is_present(body, slot, curie, + is_pv=is_pv, pv_promoted=promoted): + diagnostics.append( + f"{display}: {kind} {name!r} missing " + f"{slot}: {curie}" + ) + + for name in sorted(set(by_name) - matched - known_subjects): + diagnostics.append( + f"{display}: SSSOM subject {name!r} does not resolve to any " + "schema element (class / slot / enum / type / attribute / " + "permissible value)" + ) + + declared = set((data.get("prefixes") or {}).keys()) + referenced = {curie.split(":", 1)[0] + for slot_map in by_name.values() + for curies in slot_map.values() + for curie in curies if ":" in curie} + for px in sorted(referenced - declared - _BUILTIN_PREFIXES + - subject_prefixes): + diagnostics.append( + f"{display}: prefix {px!r} is referenced by a mapping but not " + "declared in the schema's prefixes: block" + ) + return diagnostics + + +def check_files( + schema_paths: list[Path], + mappings: MappingIndex, + extra_subject_prefixes: set[str], + *, + schema_root: Path | None = None, +) -> list[str]: + """Multi-file verification driver. + + Snapshots each file's ``by_name`` view so a later + ``build_for_prefixes`` call doesn't clobber it, then emits per-file + diagnostics using a global match-set so an unresolved-subject + diagnostic only fires for subjects that resolve in no file. + """ + y = _make_yaml() + per_file: list[tuple[str, CommentedMap | None, set[str] | None, + dict | None]] = [] + globally_matched: set[str] = set() + + for path in schema_paths: + display = _relative_display(path, schema_root) + with open(path, encoding="utf-8") as fh: + data = y.load(fh) + if not _looks_like_linkml_schema(data): + continue + subject_prefixes = ( + discover_subject_prefixes(data) | extra_subject_prefixes + ) + if not subject_prefixes: + per_file.append((display, None, None, None)) + continue + mappings.build_for_prefixes(subject_prefixes) + snapshot = {k: {s: list(v) for s, v in slots.items()} + for k, slots in mappings.by_name.items()} + for _k, _n, _b, _sm, _ipv, subject in _iter_targets(data, snapshot): + globally_matched.add(subject) + per_file.append((display, data, subject_prefixes, snapshot)) + + diagnostics: list[str] = [] + for display, data, subject_prefixes, by_name in per_file: + if data is None: + diagnostics.append( + f"{display}: no default_prefix/name in schema and no " + "--subject-prefix supplied; cannot verify" + ) + continue + if not by_name: + continue + assert subject_prefixes is not None + diagnostics.extend(_diagnose( + data, display, by_name, subject_prefixes, globally_matched, + )) + return diagnostics + + +def check_file( + schema_path: Path, + mappings: MappingIndex, + extra_subject_prefixes: set[str], + *, + display_path: str | None = None, # accepted for backwards compat + known_subjects: set[str] | None = None, # ditto +) -> list[str]: + """Single-file wrapper around :func:`check_files`.""" + del display_path, known_subjects + return check_files([schema_path], mappings, extra_subject_prefixes) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _autodiscover(script_path: Path) -> tuple[Path, str] | None: + """Locate ``/src//`` from the shipped script location.""" + root = script_path.resolve().parent.parent + src = root / "src" + if not src.is_dir(): + return None + candidates = [d for d in sorted(src.iterdir()) + if d.is_dir() and (d / "schema").is_dir()] + if len(candidates) == 1: + return root, candidates[0].name + return None + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + src = p.add_mutually_exclusive_group() + src.add_argument("--schema-dir", type=Path, + help="dir of LinkML schema YAMLs (recursive); " + "defaults to src//schema") + src.add_argument("--schema", type=Path, + help="single LinkML schema YAML file") + p.add_argument("--mappings-dir", type=Path, + help="dir of *.sssom.tsv files; defaults to " + "src//mappings") + p.add_argument("--subject-prefix", action="append", default=[], + metavar="PREFIX", + help="extra subject-side CURIE prefix (repeatable)") + mode = p.add_mutually_exclusive_group() + mode.add_argument("--dry-run", action="store_true", + help="report changes without writing any files") + mode.add_argument("--check", action="store_true", + help="read-only verification gate (exit 1 on drift)") + args = p.parse_args(argv) + + need_discovery = (args.schema is None and args.schema_dir is None + or args.mappings_dir is None) + if need_discovery: + discovered = _autodiscover(Path(__file__)) + if discovered is None: + print("ERROR: could not auto-discover src//; pass " + "--schema-dir/--schema and --mappings-dir", + file=sys.stderr) + return 1 + root, slug = discovered + if args.schema is None and args.schema_dir is None: + args.schema_dir = root / "src" / slug / "schema" + if args.mappings_dir is None: + args.mappings_dir = root / "src" / slug / "mappings" + + if args.schema: + schemas = [args.schema] + schema_root: Path | None = args.schema.parent + else: + schemas = sorted(args.schema_dir.rglob("*.yaml")) + if not schemas: + print(f"No YAML schemas in {args.schema_dir}", file=sys.stderr) + return 1 + schema_root = args.schema_dir + + mappings = load_mappings(args.mappings_dir) + if not mappings.rows: + print(f"No mapping rows loaded from {args.mappings_dir}", + file=sys.stderr) + return 0 + + extra = {px.rstrip(":") for px in args.subject_prefix if px} + + if args.check: + diagnostics = check_files(schemas, mappings, extra, + schema_root=schema_root) + for line in diagnostics: + print(line) + if diagnostics: + print(f"\nVerification FAILED: {len(diagnostics)} issue(s) " + f"across {len(schemas)} schema file(s). Run without " + "--check to apply.", file=sys.stderr) + return 1 + print(f"\nVerification OK: {len(schemas)} schema file(s) are in " + "sync with the SSSOM TSVs.") + return 0 + + files_changed = total_elements = total_links = 0 + for path in schemas: + display = _relative_display(path, schema_root) + elements, links = overlay_file( + path, mappings, extra, dry_run=args.dry_run, + ) + if elements: + files_changed += 1 + total_elements += elements + total_links += links + prefix = "[dry-run] " if args.dry_run else "" + print(f" {prefix}{display}: +{links} links across " + f"{elements} elements") + + if total_links: + verb = "would apply" if args.dry_run else "applied" + print(f"\nOverlay complete: {total_links} new mappings {verb} to " + f"{total_elements} elements across {files_changed} files") + else: + print("\nOverlay complete: schemas already in sync with SSSOM " + "files - no changes needed") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/template/{% if use_sssom %}sssom.justfile{% endif %}.jinja b/template/{% if use_sssom %}sssom.justfile{% endif %}.jinja new file mode 100644 index 0000000..adc3121 --- /dev/null +++ b/template/{% if use_sssom %}sssom.justfile{% endif %}.jinja @@ -0,0 +1,28 @@ +{% raw %}# ============== SSSOM mapping recipes ============== +# Requires the "sssom" (sssom-py) dev dependency, added because this project +# was created with `use_sssom: true`. See https://github.com/mapping-commons/sssom-py + +# Export schema element mappings (exact_mappings, close_mappings, enum meaning, etc.) to an SSSOM TSV file +[group('model development')] +gen-sssom: + mkdir -p {{dest}}/sssom + uv run gen-sssom -o {{dest}}/sssom/{{schema_name}}.sssom.tsv {{source_schema_path}} + +# Note: sssom-py's full JsonSchema validation is skipped because linkml's +# gen-sssom writes the schema's SPDX license id (e.g. "MIT") into the +# `license` metadata field, which sssom-py's schema requires to be a URI. +# Validate CURIEs in the generated SSSOM mapping file against the schema's prefix map +[group('model development')] +validate-sssom: gen-sssom + uv run sssom validate -V StrictCurieFormat {{dest}}/sssom/{{schema_name}}.sssom.tsv + +# Apply SSSOM mappings under src//mappings to schema YAML (with FLAGS) +[group('model development')] +overlay-sssom *FLAGS: + uv run python scripts/overlay_sssom.py {{FLAGS}} +{% endraw %}{% if add_example %} +# Run the SSSOM overlay unit tests +[group('model development')] +test-overlay-sssom: gen-python + uv run python -m pytest tests/ -k overlay_sssom -v +{% endif %} \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 107b0f4..1018305 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,6 +40,20 @@ def no_docs_preview_project(tmp_path_factory): return generate_project(dest, {"gh_action_docs_preview": False}) +@pytest.fixture(scope="session") +def sssom_project(tmp_path_factory): + """Project generated with use_sssom=True.""" + dest = tmp_path_factory.mktemp("sssom") + return generate_project(dest, {"use_sssom": True}) + + +@pytest.fixture(scope="session") +def sssom_no_example_project(tmp_path_factory): + """Project generated with use_sssom=True and add_example=False.""" + dest = tmp_path_factory.mktemp("sssom_no_example") + return generate_project(dest, {"use_sssom": True, "add_example": False}) + + @pytest.fixture(scope="session", params=ALL_LICENSES) def license_project(request, tmp_path_factory): """Project generated for each license type. Returns (license_name, project_path).""" diff --git a/tests/helpers.py b/tests/helpers.py index feae10d..87b3cb5 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -25,6 +25,7 @@ "add_example": True, "gh_action_pypi": True, "gh_action_docs_preview": True, + "use_sssom": False, } diff --git a/tests/test_integration.py b/tests/test_integration.py index ab55aab..75c8eb2 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -80,3 +80,92 @@ def test_just_gen_doc(integration_project): assert result.returncode == 0, ( f"just gen-doc failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" ) + + +# --------------------------------------------------------------------------- +# use_sssom=True project +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def sssom_integration_project(tmp_path_factory): + """Generate a use_sssom=True project with git init and installed deps.""" + dest = tmp_path_factory.mktemp("sssom_integration") + project = generate_project(dest, {"use_sssom": True}) + git_init(project) + result = run_just(project, "install") + if result.returncode != 0: + pytest.fail( + f"just install failed during fixture setup:\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + return project + + +def _restore_schema(project): + """Discard overlay edits so each test sees the pristine generated schema.""" + subprocess.run( + ["git", "checkout", "--", "src/test_schema/schema"], + cwd=project, + check=True, + capture_output=True, + ) + + +def test_just_gen_and_validate_sssom(sssom_integration_project): + result = run_just(sssom_integration_project, "validate-sssom") + assert result.returncode == 0, ( + f"just validate-sssom failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" + ) + tsv = sssom_integration_project / "project/sssom/test_schema.sssom.tsv" + assert tsv.is_file(), "gen-sssom did not produce the expected TSV" + + +def test_just_test_overlay_sssom(sssom_integration_project): + result = run_just(sssom_integration_project, "test-overlay-sssom") + assert result.returncode == 0, ( + f"just test-overlay-sssom failed:\n" + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + + +def test_sssom_drift_lifecycle(sssom_integration_project): + """The CI drift gate: check fails on a fresh project, passes after apply.""" + project = sssom_integration_project + try: + check_fresh = run_just(project, "overlay-sssom", "--check") + assert check_fresh.returncode == 1, ( + f"expected drift on fresh project:\n" + f"stdout: {check_fresh.stdout}\nstderr: {check_fresh.stderr}" + ) + assert "Verification FAILED" in check_fresh.stdout + check_fresh.stderr + + apply = run_just(project, "overlay-sssom") + assert apply.returncode == 0, ( + f"just overlay-sssom failed:\nstdout: {apply.stdout}\nstderr: {apply.stderr}" + ) + + check_synced = run_just(project, "overlay-sssom", "--check") + assert check_synced.returncode == 0, ( + f"expected in-sync after apply:\n" + f"stdout: {check_synced.stdout}\nstderr: {check_synced.stderr}" + ) + finally: + _restore_schema(project) + + +def test_just_setup_applies_sssom_overlay(sssom_integration_project): + """_setup_part2 applies the overlay, so setup leaves the project in sync.""" + project = sssom_integration_project + try: + result = run_just(project, "_setup_part2") + assert result.returncode == 0, ( + f"just _setup_part2 failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" + ) + check = run_just(project, "overlay-sssom", "--check") + assert check.returncode == 0, ( + f"project not in sync after setup:\n" + f"stdout: {check.stdout}\nstderr: {check.stderr}" + ) + finally: + _restore_schema(project) diff --git a/tests/test_options.py b/tests/test_options.py index 65cb77b..cdbc9f0 100644 --- a/tests/test_options.py +++ b/tests/test_options.py @@ -55,3 +55,132 @@ def test_pages_build_absent(self, no_docs_preview_project): def test_deploy_docs_present(self, no_docs_preview_project): assert (no_docs_preview_project / ".github/workflows/deploy-docs.yaml").exists() + + +class TestWithoutSssom: + """With use_sssom=False (the default), all sssom/overlay assets should be absent.""" + + ABSENT_FILES = [ + "sssom.justfile", + "scripts/overlay_sssom.py", + "src/test_schema/mappings/README.md", + "src/test_schema/mappings/test_schema-schemaorg.sssom.tsv", + "src/test_schema/mappings/test_schema-personstatus.sssom.tsv", + "tests/test_overlay_sssom.py", + ] + + @pytest.mark.parametrize("relpath", ABSENT_FILES) + def test_file_absent(self, default_project, relpath): + assert not (default_project / relpath).exists(), f"Should be absent: {relpath}" + + def test_sssom_dependency_absent(self, default_project): + pyproject = (default_project / "pyproject.toml").read_text(encoding="utf-8") + assert "sssom" not in pyproject + assert "ruamel" not in pyproject + dev_array = pyproject.split("dev = [", 1)[1].split("]", 1)[0] + assert "\n\n" not in dev_array, "stray blank line in dev dependency array" + + def test_justfile_import_is_optional(self, default_project): + # The main justfile always references sssom.justfile via an optional + # import, so `just` works whether or not the file is present. + assert 'import? "sssom.justfile"' in (default_project / "justfile").read_text( + encoding="utf-8" + ) + + def test_ci_drift_gate_absent(self, default_project): + workflow = (default_project / ".github/workflows/main.yaml").read_text( + encoding="utf-8" + ) + assert "overlay-sssom" not in workflow + assert "${{ matrix.python-version }}" in workflow + + +class TestWithSssom: + """With use_sssom=True (and default add_example=True), all sssom/overlay assets present.""" + + PRESENT_FILES = [ + "sssom.justfile", + "scripts/overlay_sssom.py", + "src/test_schema/mappings/README.md", + "src/test_schema/mappings/test_schema-schemaorg.sssom.tsv", + "src/test_schema/mappings/test_schema-personstatus.sssom.tsv", + "tests/test_overlay_sssom.py", + ] + + @pytest.mark.parametrize("relpath", PRESENT_FILES) + def test_file_present(self, sssom_project, relpath): + assert (sssom_project / relpath).exists(), f"Missing: {relpath}" + + def test_overlay_script_is_executable(self, sssom_project): + path = sssom_project / "scripts/overlay_sssom.py" + assert path.stat().st_mode & 0o111, "overlay_sssom.py should be executable" + + def test_sssom_dependency_present(self, sssom_project): + pyproject = (sssom_project / "pyproject.toml").read_text(encoding="utf-8") + assert '"sssom' in pyproject + assert "ruamel.yaml" in pyproject + + def test_justfile_recipes_present(self, sssom_project): + justfile = (sssom_project / "sssom.justfile").read_text(encoding="utf-8") + assert "gen-sssom:" in justfile + assert "validate-sssom:" in justfile + assert "overlay-sssom " in justfile + assert "test-overlay-sssom:" in justfile + # Just-syntax variables must survive rendering; Jinja tags must not leak. + assert "{{dest}}" in justfile + assert "{%" not in justfile + + def test_pyproject_dev_array_has_no_blank_lines(self, sssom_project): + pyproject = (sssom_project / "pyproject.toml").read_text(encoding="utf-8") + dev_array = pyproject.split("dev = [", 1)[1].split("]", 1)[0] + assert "\n\n" not in dev_array, "stray blank line in dev dependency array" + + def test_readme_list_is_contiguous(self, sssom_project): + readme = (sssom_project / "README.md").read_text(encoding="utf-8") + assert "Python datamodel\n * [mappings/]" in readme, ( + "mappings bullet must directly follow datamodel bullet " + "(a blank line would split the Markdown list)" + ) + + def test_ci_drift_gate_present(self, sssom_project): + workflow = (sssom_project / ".github/workflows/main.yaml").read_text( + encoding="utf-8" + ) + assert "just overlay-sssom --check" in workflow + # GitHub Actions ${{ }} must survive; Jinja tags must not leak. + assert "${{ matrix.python-version }}" in workflow + assert "{% raw %}" not in workflow + + +class TestSssomWithoutExample: + """With use_sssom=True and add_example=False, only example-specific assets are absent.""" + + ABSENT_FILES = [ + "src/test_schema/mappings/test_schema-schemaorg.sssom.tsv", + "src/test_schema/mappings/test_schema-personstatus.sssom.tsv", + "tests/test_overlay_sssom.py", + ] + + PRESENT_FILES = [ + "sssom.justfile", + "scripts/overlay_sssom.py", + "src/test_schema/mappings/README.md", + ] + + @pytest.mark.parametrize("relpath", ABSENT_FILES) + def test_file_absent(self, sssom_no_example_project, relpath): + assert not (sssom_no_example_project / relpath).exists(), ( + f"Should be absent: {relpath}" + ) + + @pytest.mark.parametrize("relpath", PRESENT_FILES) + def test_file_present(self, sssom_no_example_project, relpath): + assert (sssom_no_example_project / relpath).exists(), f"Missing: {relpath}" + + def test_test_overlay_recipe_absent(self, sssom_no_example_project): + # The test recipe would fail without the bundled test file (pytest + # collects nothing) and its gen-python prerequisite needs a schema. + justfile = (sssom_no_example_project / "sssom.justfile").read_text( + encoding="utf-8" + ) + assert "test-overlay-sssom" not in justfile