add-downloader

README.md

@@ -20,6 +20,7 @@ CLI entrypoints provided by this project:
 - `pubmate-yamlconcat`
 - `pubmate-mint`
 - `pubmate-cleanrdf`
+- `pubmate-download-nanopubs`
 - `pubmate-extract-assertions`
 - `pubmate-validate-defining`
 - `pubmate-mint-publish`
@@ -131,6 +132,39 @@ pubmate-mint-publish \
   --intro-nanopub-uri https://w3id.org/np/RA...
 ```
 
+### 6) Download published nanopubs from a query endpoint
+
+Use `pubmate-download-nanopubs` when a query endpoint returns the nanopublication
+URIs you want to mirror locally. The endpoint should return SPARQL results JSON;
+by default Pubmate reads nanopub URIs from the `np` binding and downloads each one
+as `<nanopub-uri>.trig`.
+
+You can pass a complete endpoint URL:
+
+```bash
+pubmate-download-nanopubs \
+  "https://query.knowledgepixels.com/api/RA.../your-query?ontology=https%3A%2F%2Fw3id.org%2Fyourspace%2Fvocabulary" \
+  --output-dir published \
+  --manifest build/nanopub-network-published.tsv
+```
+
+Or split filters into repeatable query parameters:
+
+```bash
+pubmate-download-nanopubs \
+  "https://query.knowledgepixels.com/api/RA.../your-query" \
+  --query-param "ontology=https://w3id.org/yourspace/vocabulary" \
+  --query-param "ontologyNamespace=https://w3id.org/yourspace/vocabulary" \
+  --output-dir published \
+  --manifest build/nanopub-network-published.tsv
+```
+
+Useful options:
+- `--np-column`: read nanopub URIs from a binding other than `np`.
+- `--min-count`: fail if the endpoint returns fewer nanopubs than expected.
+- `--replace`: replace existing `.trig` files in `--output-dir` with exactly the endpoint result set.
+- `--no-validate`: skip parsing each downloaded TriG file as a nanopublication.
+
 ## Real-Life Publishing Checklist
 
 Before real publish:
@@ -139,6 +173,7 @@ Before real publish:
 3. Run `pubmate-validate-defining`, then `pubmate-mint-publish --dry-run`.
 4. Publish a small subset first (e.g., a temporary small assertion folder).
 5. Then publish the full batch.
+6. Use `pubmate-download-nanopubs --replace` to rebuild the local `published` folder from the network when needed.
 
 ## Troubleshooting
 
@@ -148,3 +183,5 @@ Before real publish:
   - verify assertion folder contains `.ttl` files.
 - URI prefix differences:
   - published nanopub URIs may use `purl.org` or `w3id.org` prefixes depending on server behavior.
+- Download endpoint returns no nanopubs:
+  - verify the endpoint returns SPARQL results JSON and that nanopub URIs are in the binding named by `--np-column` (`np` by default).

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pubmate"
-version = "0.2.2"
+version = "0.2.3"
 description = "Helper tool aiding with publishing nanopublications departing from a linkml schema"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -23,6 +23,7 @@ dev = [
 [project.scripts]
 pubmate-bootstrap-identity = "pubmate.cli.bootstrap_identity:cli"
 pubmate-cleanrdf = "pubmate.cli.cleanrdf:cli"
+pubmate-download-nanopubs = "pubmate.cli.download_nanopubs:cli"
 pubmate-extract-assertions = "pubmate.cli.extract_assertions:cli"
 pubmate-migrate = "pubmate.cli.migrate:cli"
 pubmate-mint = "pubmate.cli.mint:cli"

src/pubmate/_nanopub_build.py

@@ -35,10 +35,7 @@ def _bnode_sort_key(graph: rdflib.Graph, bnode: rdflib.BNode) -> tuple:
     the source's random bnode ids. Bnodes with identical content sort equal —
     which is fine, since swapping their labels yields an isomorphic graph.
     """
-    items = [
-        (str(p), "" if isinstance(o, rdflib.BNode) else str(o))
-        for p, o in graph.predicate_objects(bnode)
-    ]
+    items = [(str(p), "" if isinstance(o, rdflib.BNode) else str(o)) for p, o in graph.predicate_objects(bnode)]
     return tuple(sorted(items))
 
 

src/pubmate/cli/bootstrap_identity.py

@@ -38,14 +38,24 @@ def _code(np_uri: str) -> str:
 
 
 @click.command()
-@click.option("--bot-name", required=True, help="foaf:name for the agent, e.g. \"Biochementity bot\".")
+@click.option("--bot-name", required=True, help='foaf:name for the agent, e.g. "Biochementity bot".')
 @click.option("--owner-orcid", required=True, help="ORCID URI of the human owner (frbr:owner + attribution).")
-@click.option("--bot-id", default="bot", show_default=True, help="Local URI segment for the agent (…/RA<code>/<bot-id>).")
+@click.option(
+    "--bot-id", default="bot", show_default=True, help="Local URI segment for the agent (…/RA<code>/<bot-id>)."
+)
 @click.option("--owner-name", help="Optional foaf:name for the owner ORCID.")
 @click.option("--output-dir", required=True, type=click.Path(file_okay=False, path_type=pathlib.Path))
-@click.option("--generate-keys", is_flag=True, help="Generate a fresh RSA keypair into --output-dir (id_rsa / id_rsa.pub).")
-@click.option("--private-key", type=click.Path(exists=True, dir_okay=False), help="Existing private key (instead of --generate-keys).")
-@click.option("--public-key", type=click.Path(exists=True, dir_okay=False), help="Existing public key (with --private-key).")
+@click.option(
+    "--generate-keys", is_flag=True, help="Generate a fresh RSA keypair into --output-dir (id_rsa / id_rsa.pub)."
+)
+@click.option(
+    "--private-key",
+    type=click.Path(exists=True, dir_okay=False),
+    help="Existing private key (instead of --generate-keys).",
+)
+@click.option(
+    "--public-key", type=click.Path(exists=True, dir_okay=False), help="Existing public key (with --private-key)."
+)
 @click.option("--license", "license_", default=None, help="Override pubinfo license URI (default CC BY 4.0).")
 @click.option("--test-server", is_flag=True, help="Mark the introduction for the nanopub test server.")
 @click.option("--publish", is_flag=True, help="Publish the signed introduction to the network (default: sign only).")
@@ -87,9 +97,14 @@ def cli(
 
     kwargs = {} if license_ is None else {"license": license_}
     np = build_introduction(
-        private_key=priv, public_key=pub, bot_name=bot_name,
-        owner_orcid=owner_orcid, bot_local_name=bot_id, owner_name=owner_name,
-        test_server=test_server, **kwargs,
+        private_key=priv,
+        public_key=pub,
+        bot_name=bot_name,
+        owner_orcid=owner_orcid,
+        bot_local_name=bot_id,
+        owner_name=owner_name,
+        test_server=test_server,
+        **kwargs,
     )
     np.sign()
     agent_uri = f"{np.metadata.np_uri}/{bot_id}"

src/pubmate/cli/download_nanopubs.py

@@ -0,0 +1,102 @@
+import logging
+import pathlib
+
+import click
+
+from pubmate.download import (
+    download_nanopubs,
+    endpoint_url_with_params,
+    nanopub_uris_from_query,
+    sync_nanopubs,
+    write_manifest,
+)
+
+logging.basicConfig(level=logging.INFO, format="::%(levelname)s:: %(message)s")
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.argument("endpoint_url")
+@click.option(
+    "--query-param",
+    "query_params",
+    multiple=True,
+    help="Additional endpoint query parameter as key=value. Repeatable.",
+)
+@click.option(
+    "--output-dir",
+    required=True,
+    type=click.Path(file_okay=False, path_type=pathlib.Path),
+    help="Where to write downloaded .trig nanopublications.",
+)
+@click.option(
+    "--manifest",
+    type=click.Path(dir_okay=False, path_type=pathlib.Path),
+    help="Optional TSV manifest to write.",
+)
+@click.option("--np-column", default="np", show_default=True, help="SPARQL JSON result column containing nanopub URIs.")
+@click.option("--timeout", default=60, show_default=True, type=int, help="HTTP timeout in seconds.")
+@click.option("--retries", default=3, show_default=True, type=int, help="Number of retries per HTTP request.")
+@click.option(
+    "--min-count",
+    default=1,
+    show_default=True,
+    type=int,
+    help="Fail if the endpoint returns fewer nanopublications than this.",
+)
+@click.option(
+    "--replace/--no-replace",
+    default=False,
+    show_default=True,
+    help="Replace existing .trig files in --output-dir with exactly the endpoint result set.",
+)
+@click.option(
+    "--validate/--no-validate",
+    default=True,
+    show_default=True,
+    help="Parse each downloaded TriG file and require it to declare an np:Nanopublication.",
+)
+def cli(
+    endpoint_url: str,
+    query_params: tuple[str, ...],
+    output_dir: pathlib.Path,
+    manifest: pathlib.Path | None,
+    np_column: str,
+    timeout: int,
+    retries: int,
+    min_count: int,
+    replace: bool,
+    validate: bool,
+) -> None:
+    """Download nanopublication .trig files listed by a query endpoint.
+
+    ENDPOINT_URL is expected to return SPARQL results JSON. Pubmate reads
+    nanopublication URIs from the ``np`` binding by default, then downloads each
+    URI as ``<URI>.trig`` into ``--output-dir``.
+    """
+    try:
+        query_url = endpoint_url_with_params(endpoint_url, query_params)
+    except ValueError as error:
+        raise click.ClickException(str(error)) from error
+    np_uris = nanopub_uris_from_query(query_url, np_column=np_column, timeout=timeout, retries=retries)
+    if len(np_uris) < min_count:
+        raise click.ClickException(f"Expected at least {min_count} nanopub(s), got {len(np_uris)}.")
+
+    downloader = sync_nanopubs if replace else download_nanopubs
+    downloaded = downloader(
+        np_uris,
+        output_dir=output_dir,
+        timeout=timeout,
+        retries=retries,
+        validate=validate,
+    )
+
+    if manifest is not None:
+        write_manifest(manifest, downloaded)
+        logger.info("Wrote manifest (%d entries) -> %s", len(downloaded), manifest)
+
+    logger.info("Downloaded %d nanopublication(s) -> %s", len(downloaded), output_dir)
+
+
+if __name__ == "__main__":
+    cli()

src/pubmate/cli/migrate.py

@@ -41,21 +41,45 @@ def _code(np_uri: str) -> str:
 
 
 @click.command()
-@click.option("--assertion-folder", "-a", required=True, type=click.Path(exists=True, file_okay=False, path_type=pathlib.Path))
+@click.option(
+    "--assertion-folder", "-a", required=True, type=click.Path(exists=True, file_okay=False, path_type=pathlib.Path)
+)
 @click.option("--namespace", default="https://w3id.org/peh/biochementities/", show_default=True)
-@click.option("--output-dir", required=True, type=click.Path(file_okay=False, path_type=pathlib.Path), help="Where to write all minted .trig nanopubs (defining + superseding).")
-@click.option("--id-map-file", type=click.Path(dir_okay=False, path_type=pathlib.Path), help="TSV id-map to write/merge (old_id -> thing_uri, np_uri).")
+@click.option(
+    "--output-dir",
+    required=True,
+    type=click.Path(file_okay=False, path_type=pathlib.Path),
+    help="Where to write all minted .trig nanopubs (defining + superseding).",
+)
+@click.option(
+    "--id-map-file",
+    type=click.Path(dir_okay=False, path_type=pathlib.Path),
+    help="TSV id-map to write/merge (old_id -> thing_uri, np_uri).",
+)
 @click.option("--orcid-id")
 @click.option("--name")
-@click.option("--default-suggester", help="ORCID attributed as the suggester (prov:wasAttributedTo) for any term that carries none — e.g. the contributor of an existing batch being migrated.")
-@click.option("--nanopub-type", "nanopub_types", multiple=True, help="URI tagged in pubinfo as npx:hasNanopubType on every nanopub (repeatable).")
-@click.option("--template", help="Assertion-template URI tagged in pubinfo as nt:wasCreatedFromTemplate on every nanopub (for Nanodash rendering).")
+@click.option(
+    "--default-suggester",
+    help="ORCID attributed as the suggester (prov:wasAttributedTo) for any term that carries none — e.g. the contributor of an existing batch being migrated.",
+)
+@click.option(
+    "--nanopub-type",
+    "nanopub_types",
+    multiple=True,
+    help="URI tagged in pubinfo as npx:hasNanopubType on every nanopub (repeatable).",
+)
+@click.option(
+    "--template",
+    help="Assertion-template URI tagged in pubinfo as nt:wasCreatedFromTemplate on every nanopub (for Nanodash rendering).",
+)
 @click.option("--part-of", help="URI each term links to via dcterms:isPartOf in its assertion (e.g. the vocabulary).")
 @click.option("--private-key", type=click.Path(exists=True, dir_okay=False))
 @click.option("--public-key", type=click.Path(exists=True, dir_okay=False))
 @click.option("--intro-nanopub-uri")
 @click.option("--test-server", is_flag=True, help="Publish to the nanopub test server (with --private-key).")
-@click.option("--use-testsuite-keys", is_flag=True, help="Sign with nanopub-testsuite-connector key material (test server).")
+@click.option(
+    "--use-testsuite-keys", is_flag=True, help="Sign with nanopub-testsuite-connector key material (test server)."
+)
 @click.option("--testsuite-key", default="rsa-key1", show_default=True, hidden=True)
 @click.option("--testsuite-ref", default="main", show_default=True, hidden=True)
 @click.option("--dry-run", is_flag=True, help="Sign only (offline); do not publish to the network.")
@@ -89,18 +113,30 @@ def cli(
     skipped, so the migration is resumable.
     """
     signing = resolve_signing(
-        orcid_id=orcid_id, name=name, private_key=private_key, public_key=public_key,
-        intro_nanopub_uri=intro_nanopub_uri, use_testsuite_keys=use_testsuite_keys,
-        testsuite_key=testsuite_key, testsuite_ref=testsuite_ref, test_server=test_server, dry_run=dry_run,
+        orcid_id=orcid_id,
+        name=name,
+        private_key=private_key,
+        public_key=public_key,
+        intro_nanopub_uri=intro_nanopub_uri,
+        use_testsuite_keys=use_testsuite_keys,
+        testsuite_key=testsuite_key,
+        testsuite_ref=testsuite_ref,
+        test_server=test_server,
+        dry_run=dry_run,
     )
     builder = DefiningNanopubBuilder(
-        namespace, profile=signing.profile, test_server=signing.test_server,
-        nanopub_types=nanopub_types, template=template,
+        namespace,
+        profile=signing.profile,
+        test_server=signing.test_server,
+        nanopub_types=nanopub_types,
+        template=template,
     )
     minter = SequentialMinter(builder, default_suggester_orcid=default_suggester)
     supersession = SupersessionBuilder(
-        profile=signing.profile, test_server=signing.test_server,
-        nanopub_types=nanopub_types, template=template,
+        profile=signing.profile,
+        test_server=signing.test_server,
+        nanopub_types=nanopub_types,
+        template=template,
     )
 
     files = sorted(assertion_folder.glob(pattern))
@@ -114,11 +150,18 @@ def cli(
         graph.parse(path, format="turtle")
         assertions[_term_subject(graph, namespace, path)] = graph
 
-    existing = IdMap.from_tsv(id_map_file.read_text(encoding="utf-8")) if id_map_file and id_map_file.exists() else IdMap()
+    existing = (
+        IdMap.from_tsv(id_map_file.read_text(encoding="utf-8")) if id_map_file and id_map_file.exists() else IdMap()
+    )
 
     result = migrate_terms(
-        assertions, namespace=namespace, minter=minter, supersession_builder=supersession,
-        existing=existing, dry_run=dry_run, part_of=part_of,
+        assertions,
+        namespace=namespace,
+        minter=minter,
+        supersession_builder=supersession,
+        existing=existing,
+        dry_run=dry_run,
+        part_of=part_of,
     )
 
     output_dir.mkdir(parents=True, exist_ok=True)