diff --git a/README.md b/README.md index 8d2fa07..5660817 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Dedomena -Consistent research data access for agents and scientists, starting with biology. -OpenAlex, Europe PMC (including PubMed), and EPO patent data share a streaming page -and provenance contract. Queries retain their native source semantics. +Consistent research data access for agents and scientists across biology and finance. +OpenAlex, Europe PMC, EPO, SEC EDGAR, FRED/ALFRED, ECB, World Bank and Hyperliquid share a +streaming page and provenance contract. Queries retain their native source semantics. ## Install @@ -38,10 +38,60 @@ source identifiers, exact request provenance, SHA-256 hashes, a saved-response snapshot ID, retrieval times, completeness, cost and transfer size. Partial enumeration fails explicitly. Native records retain source-specific fields. -Default caches persist for 24 hours at `~/.cache/dedomena/sources.sqlite3`. +Research and traditional finance caches persist for 24 hours at `~/.cache/dedomena/sources.sqlite3`. `refresh=True` fetches new data while keeping previous snapshots for offline replay. `DEDOMENA_SOURCE_STORE` selects a shared store, including across Canary workers. +## Finance sources + +~~~python +from dedomena.sources import SEC, FRED, ECB, WorldBank + +with SEC() as source: # SEC_USER_AGENT: your organization and contact email + page = source.company_facts(320193) # All concepts, units, and filing vintages + consume(page.records, page.provenance.to_dict()) + +with FRED() as source: # FRED_API_KEY, free registration + for page in source.release_observations(53): # Up to 500,000 observations/request + consume(page.records, page.provenance.to_dict()) + for page in source.observations("GDP", as_of="2020-01-01"): + consume(page.records, page.provenance.to_dict()) + +with ECB() as source: # No key; currency units per EUR + page = source.fx(["USD", "GBP", "JPY"], frequency="M", + start_period="2025-01", end_period="2025-12") + consume(page.records, page.provenance.to_dict()) + +with WorldBank() as source: # No key; up to 60 indicators in one query + for page in source.search(["NY.GDP.MKTP.CD", "FP.CPI.TOTL.ZG"], + countries="all", date="1970:2024"): + consume(page.records, page.provenance.to_dict()) +~~~ + +Metadata, units, scaling, missing values, status and filing dates remain native. +FRED supports ALFRED knowledge dates; SEC retains amendments; ECB exposes +revisions; World Bank serves current revised indicators. +See [finance access, throughput and semantics](docs/FINANCE.md). + +~~~python +from dedomena.sources import Hyperliquid + +with Hyperliquid() as source: # Public market data, no key or wallet + markets = source.markets() # Native metadata and asset contexts + book = source.order_book("BTC") + consume(book.records, book.provenance.to_dict()) +~~~ + +Hyperliquid adds mids, spot/perpetual metadata, books, candles and funding history. +Market snapshots default to a zero cache TTL. Candle retention is explicitly +incomplete; funding supports timestamp checkpoints. +See [Hyperliquid retrieval and weight](docs/HYPERLIQUID.md). + +`IPPool` routes the same source clients through owned local IPs or proxies, with +shared weighted per-IP admission, key budgets and cooldowns. Reuse one pool for +Hyperliquid and OpenAlex; extra IPs do not multiply OpenAlex's key allowance. +See [shared IP routing examples](docs/IP_ROUTING.md). + ## Agent CLI ~~~sh @@ -49,11 +99,18 @@ python -m dedomena.sources benchmark openalex 'CRISPR' python -m dedomena.sources benchmark europepmc 'TITLE:CRISPR' python -m dedomena.sources search epo 'ta="CRISPR"' --max-pages 1 python -m dedomena.sources quota openalex +python -m dedomena.sources markets hyperliquid +python -m dedomena.sources book hyperliquid BTC +python -m dedomena.sources fetch sec 320193 +python -m dedomena.sources benchmark fred 53 --operation release +python -m dedomena.sources observations fred GDP --as-of 2020-01-01 +python -m dedomena.sources benchmark worldbank 'NY.GDP.MKTP.CD;FP.CPI.TOTL.ZG' --period 1970:2024 +python -m dedomena.sources benchmark ecb EXR/M.USD+GBP+JPY.EUR.SP00.A --start-period 2025-01 --end-period 2025-12 ~~~ Benchmarks default to one page; `--max-pages` controls acquisition explicitly. -Search streams JSON pages and a final receipt. Source failures emit structured -JSON to stderr with a nonzero exit status. Credentials stay in environment variables. +Collection operations stream JSON pages and a final receipt. Source failures emit structured +JSON to stderr with a nonzero exit status. Credentials stay in environment variables. Offline replay requires no current API key. See [source limits, usage and Canary integration](docs/SOURCES.md). Downstream services can expose these same clients to their agents. diff --git a/dedomena/sources/__init__.py b/dedomena/sources/__init__.py index 7189125..a2e5ab7 100644 --- a/dedomena/sources/__init__.py +++ b/dedomena/sources/__init__.py @@ -1,11 +1,17 @@ """Agent-ready research sources; streaming pages share one provenance contract.""" from .core import (BudgetExceeded, InvalidResponse, Page, Provenance, SearchLimitExceeded, SourceError, Store, Throttled) +from .egress import IPPool, IPRoute from .openalex import OpenAlex from .europepmc import EuropePMC from .epo import EPO +from .sec import SEC +from .fred import FRED +from .ecb import ECB +from .worldbank import WorldBank +from .hyperliquid import Hyperliquid __all__ = [ - "OpenAlex", "EuropePMC", "EPO", "Page", "Provenance", "Store", - "SourceError", "BudgetExceeded", "Throttled", "InvalidResponse", "SearchLimitExceeded", + "OpenAlex", "EuropePMC", "EPO", "SEC", "FRED", "ECB", "WorldBank", "Hyperliquid", "Page", "Provenance", "Store", + "IPPool", "IPRoute", "SourceError", "BudgetExceeded", "Throttled", "InvalidResponse", "SearchLimitExceeded", ] diff --git a/dedomena/sources/__main__.py b/dedomena/sources/__main__.py index 79d8cbd..2ed78a3 100644 --- a/dedomena/sources/__main__.py +++ b/dedomena/sources/__main__.py @@ -1,74 +1,193 @@ """Agent-friendly JSON CLI. Credentials are read exclusively from the environment.""" import argparse +from contextlib import ExitStack from datetime import date import json +import os +from pathlib import Path import sys import time -from . import EPO, EuropePMC, OpenAlex, SourceError, Store +from . import (ECB, EPO, FRED, SEC, EuropePMC, Hyperliquid, IPPool, IPRoute, + OpenAlex, SourceError, Store, WorldBank) + + +def _ip_pool(path): + try: + with Path(path).expanduser().open("rb") as handle: + data = handle.read(65537) + if len(data) > 65536: + raise ValueError + routes = json.loads(data) + except (OSError, ValueError, UnicodeDecodeError): + raise ValueError("IP routes require a readable JSON file of at most 64 KiB") from None + if (not isinstance(routes, list) or not 1 <= len(routes) <= 1000 + or any(not isinstance(route, dict) or "public_ip" not in route + or not set(route).issubset({"public_ip", "local_address", "proxy"}) + for route in routes)): + raise ValueError("IP routes require public_ip and one local_address or proxy per entry") + return IPPool(IPRoute(**route) for route in routes) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("action", choices=("search", "fetch", "quota", "usage", "replay", "benchmark")) - parser.add_argument("source", choices=("openalex", "europepmc", "epo")) + market_operations = ("markets", "mids", "book", "candles", "funding") + operations = ("search", "observations", "release", "catalogue", *market_operations) + parser.add_argument("action", choices=(*operations, "fetch", "quota", "usage", "replay", "benchmark")) + parser.add_argument("source", choices=("openalex", "europepmc", "epo", "sec", "fred", "ecb", "worldbank", "hyperliquid")) parser.add_argument("query", nargs="?") parser.add_argument("--store", help="Shared SQLite snapshot/allowance store") + parser.add_argument("--ip-routes", help="JSON file of owned public IP routes and local bindings/proxies") + parser.add_argument("--start-time", type=int, help="Hyperliquid start time in epoch milliseconds") + parser.add_argument("--end-time", type=int, help="Hyperliquid end time in epoch milliseconds") + parser.add_argument("--interval", help="Hyperliquid candle interval, e.g. 1h") + parser.add_argument("--spot", action="store_true", help="Hyperliquid spot market metadata") + parser.add_argument("--dex", help="Hyperliquid perpetual DEX name") parser.add_argument("--profile", choices=("discovery", "evidence"), default="discovery") parser.add_argument("--filter") parser.add_argument("--refresh", action="store_true") parser.add_argument("--max-pages", type=int) parser.add_argument("--start-date", type=date.fromisoformat) parser.add_argument("--end-date", type=date.fromisoformat) + parser.add_argument("--operation", choices=operations, default="search", help="Operation to benchmark") + parser.add_argument("--as-of", help="FRED/ALFRED knowledge date YYYY-MM-DD") + parser.add_argument("--countries", help="World Bank codes joined by semicolons, or all") + parser.add_argument("--period", help="World Bank year/month/quarter or range, e.g. 2020:2025") + parser.add_argument("--start-period", help="ECB ISO date or SDMX period") + parser.add_argument("--end-period", help="ECB ISO date or SDMX period") + parser.add_argument("--updated-after", help="ECB revision delta since ISO timestamp with timezone") + parser.add_argument("--last-n", type=int, help="ECB latest observations per series") args = parser.parse_args(argv) + operation = args.operation if args.action == "benchmark" else args.action if args.max_pages is not None and args.max_pages < 1: parser.error("--max-pages must be positive") if args.filter and args.source != "openalex": parser.error("--filter is supported by openalex only") - if args.action in ("fetch", "replay") and not args.query: + if args.as_of and (args.source != "fred" or operation not in ("search", "fetch", "observations")): + parser.error("--as-of requires FRED series search, fetch, or observations") + if (args.countries or args.period) and (args.source != "worldbank" or operation not in ("search", "observations")): + parser.error("--countries and --period require World Bank search or observations") + if (args.start_period or args.end_period or args.updated_after or args.last_n is not None) and (args.source != "ecb" or operation not in ("search", "observations")): + parser.error("SDMX period/delta/last-n options require ECB search or observations") + if args.action != "benchmark" and args.operation != "search": + parser.error("--operation selects the benchmark operation only") + if operation in market_operations and args.source != "hyperliquid": + parser.error("market operations require hyperliquid") + if args.source == "hyperliquid" and operation in ("search", "observations", "release"): + parser.error("hyperliquid supports markets, mids, book, candles, funding, and fetch") + if (args.start_time is not None or args.end_time is not None or args.interval) and ( + args.source != "hyperliquid" or operation not in ("candles", "funding")): + parser.error("epoch time and interval options require Hyperliquid candles or funding") + if args.interval and operation != "candles": + parser.error("--interval requires candles") + if operation in ("candles", "funding") and args.start_time is None: + parser.error("historical market operations require --start-time") + if operation == "candles" and (args.end_time is None or not args.interval): + parser.error("candles require --end-time and --interval") + if args.spot and (args.source != "hyperliquid" or operation not in ("markets", "catalogue")): + parser.error("--spot requires Hyperliquid markets or catalogue") + if args.dex is not None and (args.source != "hyperliquid" or operation not in ("markets", "catalogue", "mids")): + parser.error("--dex requires Hyperliquid markets or mids") + if operation == "release" and args.source != "fred": + parser.error("release requires fred") + if operation == "catalogue" and args.source not in ("ecb", "worldbank", "hyperliquid"): + parser.error("catalogue requires ecb, worldbank or hyperliquid") + if operation == "observations" and args.source not in ("fred", "ecb", "worldbank"): + parser.error("observations requires fred, ecb, or worldbank") + if operation in ("fetch", "replay", "release", "observations", "book", "candles", "funding") and not args.query: parser.error("this action needs an identifier") - if args.action in ("search", "benchmark") and not args.query and not (args.source == "openalex" and args.filter): + if operation == "search" and not args.query and not (args.source == "openalex" and args.filter): parser.error("search needs a query, or an OpenAlex filter") if bool(args.start_date) != bool(args.end_date) or (args.start_date and args.source != "epo"): parser.error("date partitions require both dates and the epo source") - store = Store(args.store) if args.store else None - constructors = {"openalex": OpenAlex, "europepmc": EuropePMC, "epo": EPO} - kwargs = {"store": store} if store else {} - if args.source == "openalex": - kwargs["profile"] = args.profile - elif args.source == "europepmc": - kwargs["result_type"] = "core" if args.profile == "evidence" else "lite" - source = constructors[args.source](**kwargs) + store = None + resources = ExitStack() + def emit(value): print(json.dumps(value, ensure_ascii=False, allow_nan=False)) + + def pages(source): + if args.source == "hyperliquid": + if operation in ("markets", "catalogue"): + return iter((source.markets(spot=args.spot, dex=args.dex or "", refresh=args.refresh),)) + if operation == "mids": + return iter((source.all_mids(dex=args.dex or "", refresh=args.refresh),)) + if operation == "book": + return iter((source.order_book(args.query, refresh=args.refresh),)) + if operation == "candles": + return iter((source.candles(args.query, args.interval, start_time=args.start_time, + end_time=args.end_time, refresh=args.refresh),)) + return source.funding_history(args.query, start_time=args.start_time, + end_time=args.end_time, refresh=args.refresh) + if operation == "release": + if not args.query.isascii() or not args.query.isdecimal(): + raise ValueError("release identifier must be a positive integer") + return source.release_observations(int(args.query), refresh=args.refresh) + if operation == "catalogue": + return iter((source.dataflows(refresh=args.refresh),)) if args.source == "ecb" else source.indicators(refresh=args.refresh) + if args.source == "fred": + method = source.observations if operation == "observations" else source.search + return method(args.query, as_of=args.as_of, refresh=args.refresh) + if args.source == "worldbank": + options = {"refresh": args.refresh} + if args.countries: + options["countries"] = args.countries + if args.period: + options["date"] = args.period + return source.search(args.query, **options) + if args.source == "ecb": + flow, key = source._query(args.query) + return iter((source.observations(flow, key, start_period=args.start_period, + end_period=args.end_period, updated_after=args.updated_after, + last_n=args.last_n, refresh=args.refresh),)) + if args.source == "openalex": + return source.search(args.query, filter=args.filter, refresh=args.refresh) + if args.source == "epo" and args.start_date: + return source.search_partitioned(args.query, args.start_date, args.end_date, + refresh=args.refresh, + biblio=args.profile == "evidence") + if args.source == "epo": + return source.search(args.query, refresh=args.refresh, biblio=args.profile == "evidence") + return source.search(args.query, refresh=args.refresh) + try: - with source: + if args.action == "replay": + store = Store(args.store or os.environ.get("DEDOMENA_SOURCE_STORE", + str(Path.home() / ".cache" / "dedomena" / "sources.sqlite3"))) + raw = store.replay(args.query) + if raw.provenance.source != args.source: + raise SourceError("snapshot belongs to a different source") + emit({"provenance": raw.provenance.to_dict(), "body": raw.body.decode("utf-8")}) + return 0 + store = Store(args.store) if args.store else None + constructors = {"openalex": OpenAlex, "europepmc": EuropePMC, "epo": EPO, + "sec": SEC, "fred": FRED, "ecb": ECB, "worldbank": WorldBank, + "hyperliquid": Hyperliquid} + kwargs = {"store": store} if store else {} + if args.ip_routes: + kwargs["ip_pool"] = resources.enter_context(_ip_pool(args.ip_routes)) + if args.source == "openalex": + kwargs["profile"] = args.profile + elif args.source == "europepmc": + kwargs["result_type"] = "core" if args.profile == "evidence" else "lite" + with constructors[args.source](**kwargs) as source: if args.action == "quota": emit(source.quota()) elif args.action == "usage": emit(source.usage()) - elif args.action == "replay": - raw = source.store.replay(args.query) - emit({"provenance": raw.provenance.to_dict(), "body": raw.body.decode("utf-8")}) elif args.action == "fetch": - emit(source.fetch(args.query, refresh=args.refresh).to_dict()) + options = {"refresh": args.refresh} + if args.source == "fred": + options["as_of"] = args.as_of + emit(source.fetch(args.query, **options).to_dict()) else: - if args.source == "openalex": - pages = source.search(args.query, filter=args.filter, refresh=args.refresh) - elif args.source == "epo" and args.start_date: - pages = source.search_partitioned(args.query, args.start_date, args.end_date, - refresh=args.refresh, - biblio=args.profile == "evidence") - elif args.source == "epo": - pages = source.search(args.query, refresh=args.refresh, biblio=args.profile == "evidence") - else: - pages = source.search(args.query, refresh=args.refresh) - limit = args.max_pages if args.max_pages else (1 if args.action == "benchmark" else None) started = time.perf_counter() + stream = pages(source) + limit = args.max_pages or (1 if args.action == "benchmark" else None) totals = {"pages": 0, "records": 0, "credits_used": 0, "wire_bytes": 0, "decoded_bytes": 0, "cache_hits": 0} last = None - for page in pages: + for page in stream: last = page totals["pages"] += 1 totals["records"] += len(page.records) @@ -76,11 +195,11 @@ def emit(value): totals["wire_bytes"] += page.wire_bytes totals["decoded_bytes"] += page.decoded_bytes totals["cache_hits"] += int(page.cache_hit) - if args.action == "search": + if args.action != "benchmark": emit(page.to_dict()) if limit and totals["pages"] >= limit: break - emit({"source": args.source, **totals, + emit({"source": args.source, "operation": operation, **totals, "elapsed_seconds": round(time.perf_counter() - started, 3), "complete": bool(last and last.complete), "next_cursor": last.next_cursor if last else None, @@ -88,12 +207,13 @@ def emit(value): "last_parameters": dict(last.provenance.parameters) if last else {}, "usage": source.usage()}) except (SourceError, ValueError, KeyError) as error: - # KeyError is a snapshot ID, never a raw upstream body or credential. message = "snapshot not found" if isinstance(error, KeyError) else str(error) - print(json.dumps({"source": args.source, "error": type(error).__name__, - "message": message, "complete": False}), file=sys.stderr) + emit_error = {"source": args.source, "error": type(error).__name__, + "message": message, "complete": False} + print(json.dumps(emit_error), file=sys.stderr) return 2 finally: + resources.close() if store: store.close() return 0 diff --git a/dedomena/sources/core.py b/dedomena/sources/core.py index cc972f9..5d403e8 100644 --- a/dedomena/sources/core.py +++ b/dedomena/sources/core.py @@ -7,6 +7,7 @@ from email.utils import parsedate_to_datetime import hashlib import json +import logging import math import os import re @@ -20,12 +21,13 @@ import httpx -SCHEMA_VERSION = "0.4.0" +SCHEMA_VERSION = "0.6.0" PUBLIC_HEADERS = { "content-type", "content-encoding", "x-ratelimit-limit", "x-ratelimit-remaining", "x-ratelimit-credits-used", "x-ratelimit-reset", "retry-after", "x-individualquotaperhour-used", "x-registeredquotaperweek-used", "x-registeredpayingquotaperweek-used", "x-throttling-control", "x-rejection-reason", + "etag", "last-modified", "date", } @@ -92,6 +94,8 @@ class Provenance: public_request_headers: Mapping[str, str] | None = None request_body: str | None = None completed_at_utc: str | None = None + http_status: int | None = None + egress_ip_sha256: str | None = None def to_dict(self) -> dict: return asdict(self) @@ -140,6 +144,20 @@ def json(self) -> dict: raise InvalidResponse(f"{self.provenance.source}: invalid JSON response") from None +class _PrivateQueryLogFilter(logging.Filter): + # A stable filter avoids races caused by per-request filter removal. Redact + # the query field itself, so no credential registry or lifetime is needed. + def filter(self, record): + message = record.getMessage() + redacted = re.sub(r'([?&]api_key=)[^&\s"<>]+', r'\1[REDACTED]', message) + if redacted != message: + record.msg, record.args = redacted, () + return True + + +logging.getLogger("httpx").addFilter(_PrivateQueryLogFilter()) + + class Store: """SQLite WAL storage shared by threads/processes using the same source key. @@ -262,13 +280,18 @@ def charge(self, scope: str, unit: str, amount: int, limit: int | None, period: def reserve(self, scope: str, unit: str, amount: int, limit: int | None, period: str, *, window_key: str) -> str: """Reserve atomically without mixing refundable estimates with provider usage.""" - reservation = uuid.uuid4().hex with self.transaction() as db: - baseline, pending = self._budget(db, scope, unit, window_key) - if limit is not None and baseline + pending + amount > limit: - raise BudgetExceeded(f"{scope.split(':')[0]}: {period} {unit} allowance exhausted") - db.execute("INSERT INTO budget_reservations VALUES(?,?,?,?,?)", - (reservation, scope, unit, window_key, amount)) + return self._reserve_in(db, scope, unit, amount, limit, period, window_key=window_key) + + def _reserve_in(self, db, scope: str, unit: str, amount: int, limit: int | None, + period: str, *, window_key: str) -> str: + """Reserve inside the caller's final dispatch transaction.""" + reservation = uuid.uuid4().hex + baseline, pending = self._budget(db, scope, unit, window_key) + if limit is not None and baseline + pending + amount > limit: + raise BudgetExceeded(f"{scope.split(':')[0]}: {period} {unit} allowance exhausted") + db.execute("INSERT INTO budget_reservations VALUES(?,?,?,?,?)", + (reservation, scope, unit, window_key, amount)) return reservation def settle(self, reservation: str, actual: int): @@ -325,7 +348,7 @@ def pace(self, scope: str, rate: float, sleep: Callable[[float], None], max_wait self.pace_many(((scope, rate),), sleep, max_wait) def pace_many(self, limits: tuple[tuple[str, float], ...], - sleep: Callable[[float], None], max_wait: float): + sleep: Callable[[float], None], max_wait: float, before_admit=None): """Admit all rate scopes together; waiters acquire no stale future dispatch slots.""" began = self.clock() while True: @@ -341,9 +364,16 @@ def pace_many(self, limits: tuple[tuple[str, float], ...], due = max(due, row[0] if row else now, cooldown[0] if cooldown else now) rates.append((scope, effective)) delay = due - now - if delay > max(0, max_wait - (now - began)): + if (delay > max(0, max_wait - (now - began)) + or (max_wait > 0 and now - began > max_wait)): raise Throttled(limits[0][0].split(":")[0], delay) if not delay: + if before_admit is not None: + before_admit(db, now) + # No separately locking work may intervene between admission and send. + now = self.clock() + if max_wait > 0 and now - began > max_wait: + raise Throttled(limits[0][0].split(":")[0], 0) for scope, rate in rates: db.execute("INSERT OR REPLACE INTO pacing VALUES(?,?)", (scope, now + 1 / rate)) return @@ -365,7 +395,8 @@ def __init__(self, source: str, base_url: str, *, credential: str = "", requests_per_second: float = 10, cache_ttl: float = 86400, max_response_bytes: int = 16_000_000, max_attempts: int = 3, max_wait: float = 60, sleep: Callable[[float], None] = time.sleep, - license: str = ""): + license: str = "", ip_pool=None, ip_limit: tuple[int, float] | None = None, + ip_throttle_only: bool = False): if not base_url.startswith("https://"): raise ValueError("source URL must use HTTPS") if requests_per_second <= 0 or not math.isfinite(requests_per_second): @@ -375,13 +406,29 @@ def __init__(self, source: str, base_url: str, *, credential: str = "", or type(max_attempts) is not int or max_attempts < 1 or not math.isfinite(max_wait) or max_wait < 0): raise ValueError("invalid transport limits") + from .egress import IPPool, IPRoute + if ip_pool is not None and (not isinstance(ip_pool, IPPool) or client is not None): + raise ValueError("ip_pool must be an IPPool and cannot be combined with client") + if ip_limit is not None and ( + not isinstance(ip_limit, tuple) or len(ip_limit) != 2 + or type(ip_limit[0]) is not int or ip_limit[0] < 1 + or type(ip_limit[1]) not in (int, float) + or not math.isfinite(ip_limit[1]) or ip_limit[1] <= 0): + raise ValueError("ip_limit must contain positive integer capacity and window seconds") + if type(ip_throttle_only) is not bool: + raise ValueError("ip_throttle_only must be boolean") self.source, self.base_url = source, base_url.rstrip("/") self.scope = source + ":" + digest(credential.encode()) self._secrets = tuple(value for value in {credential, *credential.split(":")} if len(value) >= 8) - self.client = client or httpx.Client(timeout=30, follow_redirects=False, - limits=httpx.Limits(max_connections=100, - max_keepalive_connections=100)) - self._owns_client = client is None + self.client = None if ip_pool is not None else client or httpx.Client( + timeout=30, follow_redirects=False, + limits=httpx.Limits(max_connections=100, max_keepalive_connections=100)) + self._owns_client = client is None and ip_pool is None + self.ip_pool = ip_pool + self._owns_pool = ip_pool is None and ip_limit is not None + if self._owns_pool: + self.ip_pool = IPPool([IPRoute("0.0.0.0", client=self.client)]) + self.ip_limit, self.ip_throttle_only = ip_limit, ip_throttle_only self.store = store or Store(os.environ.get("DEDOMENA_SOURCE_STORE", str(Path.home() / ".cache" / "dedomena" / "sources.sqlite3"))) self._owns_store = store is None @@ -390,6 +437,8 @@ def __init__(self, source: str, base_url: str, *, credential: str = "", self.max_wait, self.sleep, self.license = max_wait, sleep, license def close(self): + if self._owns_pool: + self.ip_pool.close() if self._owns_client: self.client.close() if self._owns_store: @@ -427,17 +476,29 @@ def _retry_after(self, value: str | None) -> float: return 0 def _request(self, method: str, path: str, *, params: Mapping | None = None, - headers: Mapping | None = None, content: bytes | None = None, + headers: Mapping | None = None, private_params: Mapping | None = None, + content: bytes | None = None, form: Mapping | None = None, auth=None, estimated_credits: int = 0, credit_limit: int | None = None, byte_limit: int | None = None, service: str = "requests", service_rate: float | None = None, refresh: bool = False, sensitive: bool = False, observe: Callable[..., None] | None = None, before_request: Callable[[], None] | None = None, - headers_factory: Callable[[], Mapping] | None = None) -> Response: + headers_factory: Callable[[], Mapping] | None = None, + rate_weight: int = 1, actual_weight: Callable[[bytes], int] | None = None) -> Response: if method not in ("GET", "POST") or not path.startswith("/") or path.startswith("//") or "?" in path: raise ValueError("invalid source request") routes = { + "hyperliquid": (("POST", r"/info"),), + "fred": (("GET", r"/(?:series(?:/search|/observations|/vintagedates)?|v2/release/observations)"),), + "worldbank": (("GET", r"/country/[A-Za-z0-9;]+/indicator/[A-Za-z0-9._;\-]+"), + ("GET", r"/indicator(?:/[A-Za-z0-9._;\-]+)?")), + "sec": (("GET", r"/submissions/CIK[0-9]{10}(?:-submissions-[0-9]+)?[.]json"), + ("GET", r"/api/xbrl/companyfacts/CIK[0-9]{10}[.]json"), + ("GET", r"/api/xbrl/companyconcept/CIK[0-9]{10}/[A-Za-z][A-Za-z0-9_.-]*/[A-Za-z][A-Za-z0-9_.-]*[.]json"), + ("GET", r"/api/xbrl/frames/[A-Za-z][A-Za-z0-9_.-]*/[A-Za-z][A-Za-z0-9_.-]*/[A-Za-z0-9_-]+/CY[0-9]{4}(?:Q[1-4])?I?[.]json")), + "ecb": (("GET", r"/data/[A-Za-z][A-Za-z0-9_,.-]*(?:/[A-Za-z0-9_.+\-]*)?"), + ("GET", r"/dataflow/ECB/all/latest")), "openalex": (("GET", r"/works(?:/.+)?"), ("GET", r"/rate-limit")), "europepmc": (("GET", r"/search"), ("POST", r"/searchPOST"), ("GET", r"/PMC[0-9]+/fullTextXML")), @@ -450,9 +511,22 @@ def _request(self, method: str, path: str, *, params: Mapping | None = None, method == verb and re.fullmatch(pattern, path) for verb, pattern in routes[self.source]): raise ValueError("source route is outside the read-only adapter") + if self.source == "hyperliquid": + from .hyperliquid import info_policy + if params or private_params or form is not None or auth is not None: + raise ValueError("Hyperliquid info requests use a public JSON body only") + rate_weight, actual_weight = info_policy(content) + if type(rate_weight) is not int or rate_weight < 1: + raise ValueError("rate_weight must be a positive integer") params = {str(k): str(v) for k, v in (params or {}).items()} if any(k.lower() in ("api_key", "access_token", "authorization") for k in params): raise ValueError("credentials must not appear in source parameters") + private_params = {str(k): str(v) for k, v in (private_params or {}).items()} + # FRED v1 only accepts a query key. Keep it out of public request identity + # and snapshots; the hashed credential namespace still isolates caches. + if private_params and (self.source != "fred" or set(private_params) != {"api_key"} + or self.scope != "fred:" + digest(private_params["api_key"].encode())): + raise ValueError("private query credentials are restricted to FRED authentication") form = {str(k): str(v) for k, v in (form or {}).items()} if form is not None else None if form and any(k.lower() in ("api_key", "access_token", "authorization") for k in form): raise ValueError("credentials must not appear in source forms") @@ -498,23 +572,45 @@ def _request(self, method: str, path: str, *, params: Mapping | None = None, limits = ((self.scope + ":all", self.rate),) if service_rate is not None: limits += ((pace_scope, service_rate),) - self.store.pace_many(limits, self.sleep, self.max_wait) - credit_window = window("day", self.store.clock()) - byte_window = window("week", self.store.clock()) - credit_reservation = self.store.reserve(self.scope, "credits", estimated_credits, - credit_limit, "day", window_key=credit_window) - byte_reservation = None - byte_reserved = self.max_response_bytes if byte_limit is not None else 0 - try: - if byte_reserved: - byte_reservation = self.store.reserve(self.scope, "bytes", byte_reserved, - byte_limit, "week", window_key=byte_window) - except BudgetExceeded: - self.store.settle(credit_reservation, 0) - raise + admitted = {} + + def reserve_budgets(db, now): + # IP capacity, pacing and these reservations commit together. No + # additional database lock can leave a queued dispatch lease stale. + while True: + day, week = window("day", now), window("week", now) + credit = self.store._reserve_in(db, self.scope, "credits", estimated_credits, + credit_limit, "day", window_key=day) + byte = (self.store._reserve_in(db, self.scope, "bytes", self.max_response_bytes, + byte_limit, "week", window_key=week) + if byte_limit is not None else None) + fresh = self.store.clock() + if day == window("day", fresh) and week == window("week", fresh): + admitted.update(credit=credit, byte=byte, day=day, week=week) + return + # Reanchor if even the local transaction spans a quota reset. + db.execute("DELETE FROM budget_reservations WHERE id=?", (credit,)) + if byte: + db.execute("DELETE FROM budget_reservations WHERE id=?", (byte,)) + now = fresh + + lease = None + if self.ip_pool is not None: + capacity, period = self.ip_limit or (None, 60.0) + lease = self.ip_pool.acquire(self.store, self.source, rate_weight, capacity, + period, self.sleep, self.max_wait, pacing=limits, + before_admit=reserve_budgets) + else: + self.store.pace_many(limits, self.sleep, self.max_wait, reserve_budgets) + active_client = lease.client if lease is not None else self.client + weight_stats = ({"ip_weight_reserved": rate_weight, "ip_weight_used": rate_weight} + if lease is not None else {}) + credit_window, byte_window = admitted["day"], admitted["week"] + credit_reservation, byte_reservation = admitted["credit"], admitted["byte"] started = self.store.clock() try: - with self.client.stream(method, url, params=params, headers=attempt_headers, + with active_client.stream( + method, url, params={**params, **private_params}, headers=attempt_headers, content=request_body, auth=auth, timeout=30, follow_redirects=False) as upstream: received = bytearray() @@ -531,7 +627,7 @@ def _request(self, method: str, path: str, *, params: Mapping | None = None, if k.lower() in PUBLIC_HEADERS} except httpx.HTTPError: self.store.record(self.scope, requests=1, failures=1, credits_used=estimated_credits, - **{service + "_requests": 1}) + **{service + "_requests": 1}, **weight_stats) # An uncertain request may have been charged. Keep its reservations. if attempt + 1 < self.max_attempts: self.sleep(min(2 ** attempt, self.max_wait)) @@ -539,8 +635,20 @@ def _request(self, method: str, path: str, *, params: Mapping | None = None, raise SourceError(f"{self.source}: transport failure") from None except InvalidResponse: self.store.record(self.scope, requests=1, failures=1, credits_used=estimated_credits, - **{service + "_requests": 1}) + **{service + "_requests": 1}, **weight_stats) raise + if lease is not None and 200 <= status < 300 and actual_weight is not None: + try: + used_weight = actual_weight(body) + if type(used_weight) is not int or not 1 <= used_weight <= rate_weight: + raise ValueError + except (ValueError, TypeError, UnicodeDecodeError): + self.store.record(self.scope, requests=1, failures=1, + credits_used=estimated_credits, + **{service + "_requests": 1}, **weight_stats) + raise InvalidResponse(f"{self.source}: invalid response weight") from None + self.ip_pool.settle(self.store, lease, used_weight) + weight_stats["ip_weight_used"] = used_weight try: charged = int(safe_headers.get("x-ratelimit-credits-used", estimated_credits)) if charged < 0: @@ -551,11 +659,16 @@ def _request(self, method: str, path: str, *, params: Mapping | None = None, if byte_reservation is not None: self.store.settle(byte_reservation, wire) self.store.record(self.scope, requests=1, wire_bytes=wire, decoded_bytes=len(body), - credits_used=charged, failures=int(status >= 400), **{service + "_requests": 1}) + credits_used=charged, failures=int(status >= 400), + **{service + "_requests": 1}, **weight_stats) # Persist provider cooldowns even when the caller cannot wait or retry. # Budget rejection takes precedence in the error contract, never in pacing. retry_delay = max(2 ** attempt, self._retry_after(safe_headers.get("retry-after"))) - if status == 429 or 500 <= status <= 599: + if status == 429 and lease is not None: + self.ip_pool.defer(self.store, self.source, lease.route, retry_delay) + if (500 <= status <= 599 or (status == 429 and ( + lease is None or not self.ip_throttle_only + or safe_headers.get("x-ratelimit-remaining") == "0"))): self.store.defer(self.scope + ":all", retry_delay) if observe: observe(safe_headers, request_windows={"day": credit_window, "week": byte_window}) @@ -588,7 +701,9 @@ def _request(self, method: str, path: str, *, params: Mapping | None = None, acquired, snapshot_id, license=self.license, public_request_headers=identity["headers"], request_body=request_body.decode("utf-8") if request_body is not None else None, - completed_at_utc=utc_stamp(self.store.clock())) + completed_at_utc=utc_stamp(self.store.clock()), http_status=status, + egress_ip_sha256=(digest(lease.public_ip.encode()) + if lease and lease.public_ip != "0.0.0.0" else None)) response = Response(body, safe_headers, p, False, charged, wire) if not sensitive: self.store.put(cache_key, response) diff --git a/dedomena/sources/ecb.py b/dedomena/sources/ecb.py new file mode 100644 index 0000000..6c97a68 --- /dev/null +++ b/dedomena/sources/ecb.py @@ -0,0 +1,176 @@ +"""ECB SDMX: batched dimensions, full attributes, revision deltas, and EUR FX.""" +from __future__ import annotations + +import csv +from datetime import date, datetime +import io +import re +from typing import Iterable, Iterator + +from defusedxml import ElementTree + +from .core import InvalidResponse, Page, Transport, positive_int + + +class ECB(Transport): + """Preserve source CSV strings, units, multipliers, status, and series keys.""" + + def __init__(self, **kwargs): + # No published request quota: a local policy, not provider entitlement. + kwargs.setdefault("requests_per_second", 2) + super().__init__("ecb", "https://data-api.ecb.europa.eu/service", + license="ECB reuse policy; third-party data may require permission", **kwargs) + + @staticmethod + def _flow(flow: str) -> str: + if not isinstance(flow, str) or not re.fullmatch( + r"[A-Za-z][A-Za-z0-9_-]*(?:,[A-Za-z0-9_-]+(?:,[A-Za-z0-9_.-]+)?)?", flow): + raise ValueError("flow must be an SDMX dataflow ID or agency,ID,version") + return flow + + @staticmethod + def _key(key: str) -> str: + if not isinstance(key, str) or key in (".", "..") or not re.fullmatch(r"[A-Za-z0-9_.+\-]*", key): + raise ValueError("key must contain SDMX dimension codes, dots, and plus signs") + return key + + @staticmethod + def _period(value: str | None) -> str | None: + if value is None: + return None + if not isinstance(value, str) or not re.fullmatch( + r"[0-9]{4}(?:-S[12]|-Q[1-4]|-W(?:0[1-9]|[1-4][0-9]|5[0-3])|-(?:0[1-9]|1[0-2])(?:-(?:0[1-9]|[12][0-9]|3[01]))?)?", value): + raise ValueError("period must be an ISO date or SDMX reporting period") + if len(value) == 10: + date.fromisoformat(value) + return value + + def _data(self, flow: str, key: str, *, detail: str = "full", + start_period: str | None = None, end_period: str | None = None, + updated_after: str | None = None, include_history: bool = False, + last_n: int | None = None, refresh: bool = False) -> Page: + flow, key = self._flow(flow), self._key(key) + if type(include_history) is not bool: + raise ValueError("include_history must be a boolean") + params = {"format": "csvdata", "detail": detail, + "includeHistory": str(include_history).lower()} + start, end = self._period(start_period), self._period(end_period) + # Lexical order is chronological only within the same SDMX format. + # Preserve mixed-frequency bounds for the provider to interpret. + if (start and end and re.sub(r"[0-9]", "", start) == re.sub(r"[0-9]", "", end) + and start > end): + raise ValueError("start_period must not follow end_period") + if start: + params["startPeriod"] = start + if end: + params["endPeriod"] = end + if updated_after: + try: + parsed = datetime.fromisoformat(updated_after.replace("Z", "+00:00")) + if parsed.utcoffset() is None: + raise ValueError + except (ValueError, TypeError, AttributeError): + raise ValueError("updated_after must be an ISO timestamp with timezone") from None + params["updatedAfter"] = updated_after + if last_n is not None: + params["lastNObservations"] = str(positive_int(last_n, 1_000_000, "last_n")) + response = self._request("GET", f"/data/{flow}" + (f"/{key}" if key else ""), + params=params, headers={"Accept": "text/csv"}, refresh=refresh) + if not response.body.strip(): + return self.page(response, [], total=0, complete=True, records_seen=0, + warnings=("ECB returned no observations for the selected query.",)) + try: + reader = csv.DictReader(io.StringIO(response.body.decode("utf-8-sig")), strict=True) + fields = reader.fieldnames + if not fields or len(fields) != len(set(fields)) or "KEY" not in fields: + raise ValueError + if detail == "full" and not {"TIME_PERIOD", "OBS_VALUE"}.issubset(fields): + raise ValueError + rows = list(reader) + dataset = flow.split(",")[1] if "," in flow else flow + dimensions = key.split(".") if key else [] + def matches(series_key): + actual = series_key.removeprefix(dataset + ".").split(".") + return len(actual) >= len(dimensions) and all( + not wanted or value in wanted.split("+") + for wanted, value in zip(dimensions, actual)) + if any(None in row or any(value is None for value in row.values()) + or not row["KEY"].startswith(dataset + ".") or not matches(row["KEY"]) + or (detail == "full" and not row["TIME_PERIOD"]) + for row in rows): + raise ValueError + except (ValueError, UnicodeDecodeError, csv.Error): + raise InvalidResponse("ecb: invalid CSV or mismatched dataflow") from None + warnings = ["Values retain native units, UNIT_MULT, and observation status; no numeric coercion."] + if last_n is not None: + warnings.append("last_n selects the latest observations per series; earlier history is excluded.") + if updated_after: + warnings.append("Revision delta: additions, corrections, and deletions must be reconciled with prior data.") + if include_history: + warnings.append("Historical versions are included; do not collapse rows by series and period.") + return self.page(response, rows, total=len(rows), complete=True, + records_seen=len(rows), warnings=warnings) + + def observations(self, flow: str, key: str = "", **kwargs) -> Page: + """One complete selected query; narrow dates if the response size cap is hit.""" + return self._data(flow, key, **kwargs) + + def series(self, flow: str, key: str = "", *, refresh: bool = False) -> Page: + """Discover matching keys and all series attributes without observations.""" + return self._data(flow, key, detail="nodata", refresh=refresh) + + @staticmethod + def _query(query: str) -> tuple[str, str]: + if not isinstance(query, str) or query.count("/") > 1: + raise ValueError("query must be FLOW/SDMX.KEY or FLOW") + flow, separator, key = query.partition("/") + return flow, key if separator else "" + + def fetch(self, identifier: str, *, refresh: bool = False) -> Page: + return self.observations(*self._query(identifier), refresh=refresh) + + def search(self, query: str, *, refresh: bool = False) -> Iterator[Page]: + yield self.fetch(query, refresh=refresh) + + def fx(self, currencies: Iterable[str] | str | None = None, *, frequency: str = "D", + start_period: str | None = None, end_period: str | None = None, + last_n: int | None = None, refresh: bool = False) -> Page: + """Batch reference rates: OBS_VALUE is currency units per one EUR.""" + if frequency not in ("D", "M", "Q", "A"): + raise ValueError("frequency must be D, M, Q, or A") + values = [currencies] if isinstance(currencies, str) else list(currencies or ()) + if currencies is not None and (not values or any( + not isinstance(value, str) or not re.fullmatch(r"[A-Z]{3}", value) for value in values)): + raise ValueError("currencies must be uppercase ISO currency codes") + codes = "+".join(dict.fromkeys(values)) + if len(codes) > 1500: + raise ValueError("currency batch exceeds the local URL length limit") + return self.observations("EXR", f"{frequency}.{codes}.EUR.SP00.A", + start_period=start_period, end_period=end_period, + last_n=last_n, refresh=refresh) + + def dataflows(self, *, refresh: bool = False) -> Page: + """Discover dataset IDs, versions, and names; original XML is archived.""" + response = self._request("GET", "/dataflow/ECB/all/latest", params={"detail": "allstubs"}, + headers={"Accept": "application/vnd.sdmx.structure+xml;version=2.1"}, + refresh=refresh) + try: + root = ElementTree.fromstring(response.body) + rows = [] + for node in root.iter(): + if node.tag.rsplit("}", 1)[-1] == "Dataflow": + if not node.get("id") or node.get("agencyID") != "ECB": + raise ValueError + rows.append({**node.attrib, "names": [ + {**child.attrib, "text": child.text or ""} + for child in node if child.tag.rsplit("}", 1)[-1] == "Name"]}) + if not rows: + raise ValueError + except Exception: + raise InvalidResponse("ecb: invalid dataflow metadata XML") from None + return self.page(response, rows, total=len(rows), complete=True, records_seen=len(rows)) + + def quota(self) -> dict: + return {"api_key_required": False, "daily_provider_quota": None, + "local_requests_per_second": self.rate, "max_response_bytes": self.max_response_bytes, + "note": "No published daily allowance. Batch dimensions and use updated_after for deltas; narrow large queries by dates."} diff --git a/dedomena/sources/egress.py b/dedomena/sources/egress.py new file mode 100644 index 0000000..8779a56 --- /dev/null +++ b/dedomena/sources/egress.py @@ -0,0 +1,315 @@ +"""Owned egress routes and durable, weighted per-IP admission. + +The declared public IP identifies a quota; it does not configure routing. Each +route therefore needs a local interface, a proxy, or an already routed HTTP +client. No address is discovered automatically. Different local addresses +behind the same NAT must declare the same public IP and cannot form a pool. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +import hashlib +import ipaddress +import math +import time +import uuid +from typing import Callable, Iterable, TYPE_CHECKING + +import httpx + +if TYPE_CHECKING: + from .core import Store + + +def _address(value: str) -> str: + if not isinstance(value, str) or "%" in value: + raise ValueError("route IP must be an IPv4 or IPv6 address") + try: + address = ipaddress.ip_address(value) + except ValueError: + raise ValueError("route IP must be an IPv4 or IPv6 address") from None + if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped: + address = address.ipv4_mapped + return str(address) + + +@dataclass(frozen=True) +class IPRoute: + """An explicitly routed connection and its operator-declared public IP. + + Injected clients must already route through the declared address. They + cannot be combined with local_address or proxy, which configure a client + owned by the pool. Proxy credentials are deliberately absent from repr. + """ + public_ip: str + local_address: str | None = None + proxy: str | None = field(default=None, repr=False) + client: httpx.Client | None = field(default=None, repr=False, compare=False) + + def __post_init__(self): + object.__setattr__(self, "public_ip", _address(self.public_ip)) + if sum(value is not None for value in (self.local_address, self.proxy, self.client)) != 1: + raise ValueError("route requires exactly one local address, proxy or routed client") + if self.local_address is not None: + object.__setattr__(self, "local_address", _address(self.local_address)) + if self.proxy is not None: + if not isinstance(self.proxy, str): + raise ValueError("route proxy must be an HTTP or HTTPS URL") + try: + url = httpx.URL(self.proxy) + valid = (url.scheme in ("http", "https") and bool(url.host) + and not url.query and not url.fragment and url.path in ("", "/")) + except (httpx.InvalidURL, ValueError): + valid = False + if not valid: + raise ValueError("route proxy must be an HTTP or HTTPS origin") from None + if self.client is not None and not isinstance(self.client, httpx.Client): + raise ValueError("route client must be an httpx.Client") + + +@dataclass(frozen=True) +class IPLease: + """One admitted request, with a uniquely owned refundable IP estimate.""" + route: IPRoute + reservation_id: str | None + weight: int + scope: str + + @property + def client(self) -> httpx.Client: + return self.route.client + + @property + def public_ip(self) -> str: + return self.route.public_ip + + +class IPPool: + """Share a route pool across sources while keeping each provider's IP quota. + + Admission uses rolling weighted windows in Store, shared by threads, + processes and restarts using that database. Provider/key limits supplied + in limits are admitted in the same transaction; adding routes cannot + multiply those allowances. Quota charges represent dispatched attempts, + including failed requests. Only a verified response may reduce an estimate. + """ + def __init__(self, routes: Iterable[IPRoute]): + routes = tuple(routes) + if not routes or any(not isinstance(route, IPRoute) for route in routes): + raise ValueError("pool requires at least one IPRoute") + if len({route.public_ip for route in routes}) != len(routes): + raise ValueError("pool public IP addresses must be distinct, including NAT aliases") + self._owned = [] + self._closed = False + try: + configured = [] + for route in routes: + if route.client is None: + options = dict(timeout=30, follow_redirects=False, trust_env=False, + limits=httpx.Limits(max_connections=100, max_keepalive_connections=100)) + if route.local_address is not None: + options["transport"] = httpx.HTTPTransport(local_address=route.local_address, + trust_env=False, limits=options["limits"]) + else: + options["proxy"] = route.proxy + client = httpx.Client(**options) + self._owned.append(client) + configured.append(IPRoute(route.public_ip, client=client)) + else: + configured.append(route) + self.routes = tuple(configured) + except BaseException: + self.close() + raise + # The scheduling cursor is shared by pools with the same ordered routes. + self._identity = hashlib.sha256("\0".join(r.public_ip for r in self.routes).encode()).hexdigest() + + def close(self): + if not self._closed: + self._closed = True + for client in self._owned: + client.close() + + def __enter__(self): + if self._closed: + raise ValueError("IP pool is closed") + return self + + def __exit__(self, *args): + self.close() + + @staticmethod + def scope(source: str, route: IPRoute) -> str: + """Stable public-IP quota identity; excludes proxy and client credentials.""" + if not isinstance(source, str) or not source: + raise ValueError("source must be a nonempty string") + return source + ":ip:" + route.public_ip + + @staticmethod + def _schema(db): + db.execute("CREATE TABLE IF NOT EXISTS ip_events(" + "id TEXT PRIMARY KEY, scope TEXT NOT NULL, charged REAL NOT NULL, weight INTEGER NOT NULL)") + db.execute("CREATE INDEX IF NOT EXISTS ip_events_by_scope ON ip_events(scope,charged)") + db.execute("CREATE TABLE IF NOT EXISTS ip_windows(scope TEXT PRIMARY KEY, period REAL NOT NULL)") + db.execute("CREATE TABLE IF NOT EXISTS ip_cursors(scope TEXT PRIMARY KEY, position INTEGER NOT NULL)") + # Store owns cooldowns so admission and existing pacing see the same deferral. + db.execute("CREATE TABLE IF NOT EXISTS cooldowns(scope TEXT PRIMARY KEY, until REAL NOT NULL)") + + @staticmethod + def _limit(scope, weight, capacity, period): + if (not isinstance(scope, str) or not scope + or type(weight) is not int or weight < 1 + or type(capacity) is not int or capacity < 1 + or isinstance(period, bool) or not isinstance(period, (int, float)) + or not math.isfinite(period) or period <= 0): + raise ValueError("weighted admission requires a scope, positive integer cost and capacity, and finite period") + if weight > capacity: + from .core import BudgetExceeded + raise BudgetExceeded("request weight exceeds the configured window allowance") + return scope, weight, capacity, float(period) + + @staticmethod + def _ready(db, limit, now): + scope, weight, capacity, period = limit + row = db.execute("SELECT period FROM ip_windows WHERE scope=?", (scope,)).fetchone() + if row and row[0] != period: + # Changing a scope's period after pruning history could hide consumption. + raise ValueError("quota scope cannot change its rolling window period") + db.execute("INSERT OR IGNORE INTO ip_windows VALUES(?,?)", (scope, period)) + db.execute("DELETE FROM ip_events WHERE scope=? AND charged<=?", (scope, now - period)) + rows = db.execute("SELECT charged,weight FROM ip_events WHERE scope=? ORDER BY charged", + (scope,)).fetchall() + used = sum(amount for _, amount in rows) + due = now + if used + weight > capacity: + for charged, amount in rows: + used -= amount + due = charged + period + if used + weight <= capacity: + break + cooldown = db.execute("SELECT until FROM cooldowns WHERE scope=?", (scope,)).fetchone() + return max(due, cooldown[0] if cooldown else now) + + def acquire(self, store: Store, source: str, weight: int, capacity: int | None, period: float, + sleep: Callable[[float], None] = time.sleep, max_wait: float = 60, + *, limits: Iterable[tuple[str, int, int, float]] = (), + pacing: Iterable[tuple[str, float]] = (), before_admit=None) -> IPLease: + """Admit the next ready IP without reserving future dispatch slots. + + limits contains (scope, weight, capacity, period) for provider/key + ceilings. Scopes must be distinct from the IP scopes and each other. + Reuse a stable scope and period across every worker and pool. Existing + Store pacing scopes are admitted atomically too. capacity=None skips + weighted IP admission while retaining routing and cooldowns. Optional + before_admit(db, now) reserves provider budgets in the same transaction + immediately before admission; an exception rolls back every reservation. + """ + if self._closed: + raise ValueError("IP pool is closed") + if (isinstance(max_wait, bool) or not isinstance(max_wait, (int, float)) + or not math.isfinite(max_wait) or max_wait < 0): + raise ValueError("max_wait must be finite and nonnegative") + common = tuple(self._limit(*limit) for limit in limits) + rate_limits = tuple(pacing) + if any(not isinstance(scope, str) or not scope + or isinstance(rate, bool) or not isinstance(rate, (int, float)) + or not math.isfinite(rate) or rate <= 0 for scope, rate in rate_limits): + raise ValueError("pacing requires a scope and positive finite rate") + if len({scope for scope, _ in rate_limits}) != len(rate_limits): + raise ValueError("pacing scopes must be distinct") + # Validate weight and period even when this provider has no IP quota. + self._limit("validation", weight, capacity if capacity is not None else weight, period) + route_limits = tuple(self._limit(self.scope(source, route), weight, + capacity if capacity is not None else weight, period) + for route in self.routes) + common_scopes = [limit[0] for limit in common] + if (len(set(common_scopes)) != len(common_scopes) + or set(common_scopes).intersection(limit[0] for limit in route_limits)): + raise ValueError("admission scopes must be distinct") + began = store.clock() + scheduling = source + ":pool:" + self._identity + waited = False + while True: + if self._closed: + raise ValueError("IP pool is closed") + with store.transaction() as db: + self._schema(db) + now = store.clock() + db.execute("DELETE FROM throttles WHERE expires<=?", (now,)) + common_due = max([now] + [self._ready(db, limit, now) for limit in common]) + effective_rates = [] + for scope, rate in rate_limits: + throttled = db.execute("SELECT min(rate) FROM throttles WHERE scope=?", (scope,)).fetchone()[0] + effective_rates.append((scope, min(rate, throttled) if throttled else rate)) + next_at = db.execute("SELECT next_at FROM pacing WHERE scope=?", (scope,)).fetchone() + cooldown = db.execute("SELECT until FROM cooldowns WHERE scope=?", (scope,)).fetchone() + common_due = max(common_due, next_at[0] if next_at else now, + cooldown[0] if cooldown else now) + due = [] + for limit in route_limits: + if capacity is not None: + route_due = self._ready(db, limit, now) + else: + cooldown = db.execute("SELECT until FROM cooldowns WHERE scope=?", (limit[0],)).fetchone() + route_due = cooldown[0] if cooldown else now + due.append(max(common_due, route_due)) + row = db.execute("SELECT position FROM ip_cursors WHERE scope=?", (scheduling,)).fetchone() + first = row[0] % len(self.routes) if row else 0 + ordering = [(first + offset) % len(self.routes) for offset in range(len(self.routes))] + chosen = min(ordering, key=lambda index: due[index]) + delay = max(0, due[chosen] - now) + remaining = max_wait - max(0, now - began) + if delay > max(0, remaining) or (remaining < 0 and (waited or max_wait > 0)): + from .core import Throttled + raise Throttled(source, delay) + if delay == 0: + if before_admit is not None: + before_admit(db, now) + now = store.clock() + if max_wait > 0 and now - began > max_wait: + from .core import Throttled + raise Throttled(source, 0) + reservation = uuid.uuid4().hex if capacity is not None else None + for scope, amount, _, _ in common: + db.execute("INSERT INTO ip_events VALUES(?,?,?,?)", (uuid.uuid4().hex, scope, now, amount)) + if reservation: + db.execute("INSERT INTO ip_events VALUES(?,?,?,?)", + (reservation, route_limits[chosen][0], now, weight)) + for scope, rate in effective_rates: + db.execute("INSERT OR REPLACE INTO pacing VALUES(?,?)", (scope, now + 1 / rate)) + db.execute("INSERT OR REPLACE INTO ip_cursors VALUES(?,?)", + (scheduling, (chosen + 1) % len(self.routes))) + return IPLease(self.routes[chosen], reservation, weight, route_limits[chosen][0]) + waited = True + sleep(delay) + + @staticmethod + def settle(store: Store, lease: IPLease, actual_weight: int): + """Reduce only this request's known estimate; expired charges stay expired. + + Settle after a trustworthy response has established actual cost. An + uncertain send or malformed response must retain its full reservation. + """ + if not isinstance(lease, IPLease) or type(actual_weight) is not int or not 0 <= actual_weight <= lease.weight: + raise ValueError("actual weight must be a nonnegative integer within the reserved estimate") + if lease.reservation_id is None: + return + with store.transaction() as db: + IPPool._schema(db) + row = db.execute("SELECT weight FROM ip_events WHERE id=? AND scope=?", + (lease.reservation_id, lease.scope)).fetchone() + if row and row[0] != lease.weight and actual_weight != row[0]: + raise ValueError("settlement cannot revise an already reduced reservation") + db.execute("UPDATE ip_events SET weight=? WHERE id=? AND scope=?", + (actual_weight, lease.reservation_id, lease.scope)) + + def defer(self, store: Store, source: str, route: IPRoute | IPLease, seconds: float): + """Persist a 429 cooldown only for this provider and public IP.""" + if (isinstance(seconds, bool) or not isinstance(seconds, (int, float)) + or not math.isfinite(seconds) or seconds < 0): + raise ValueError("cooldown must be finite and nonnegative") + if isinstance(route, IPLease): + route = route.route + if route not in self.routes: + raise ValueError("cooldown route must belong to this pool") + store.defer(self.scope(source, route), seconds) diff --git a/dedomena/sources/fred.py b/dedomena/sources/fred.py new file mode 100644 index 0000000..cb101a9 --- /dev/null +++ b/dedomena/sources/fred.py @@ -0,0 +1,277 @@ +"""FRED/ALFRED series, dated vintages, and 500,000-observation release pages.""" +from __future__ import annotations + +from datetime import date, datetime, timezone +import os +import re +from typing import Iterator + +from .core import InvalidResponse, Page, Transport, positive_int + + +class FRED(Transport): + """A user's free FRED key; preserve native decimal strings and missing dots.""" + MAX_PAGE_SIZE = 100_000 + MAX_RELEASE_OBSERVATIONS = 500_000 + + def __init__(self, api_key: str | None = None, **kwargs): + self.api_key = api_key if api_key is not None else os.environ.get("FRED_API_KEY", "") + if not isinstance(self.api_key, str) or not re.fullmatch(r"[a-z0-9]{32}", self.api_key): + raise ValueError("FRED requires a 32-character lowercase alphanumeric FRED_API_KEY or api_key") + kwargs.setdefault("requests_per_second", 2) + if kwargs["requests_per_second"] > 2: + raise ValueError("FRED permits at most 2 requests per second / 120 per minute") + # A maximum release page can exceed the generic 16 MB response ceiling. + kwargs.setdefault("max_response_bytes", 128_000_000) + super().__init__("fred", "https://api.stlouisfed.org/fred", credential=self.api_key, + license="FRED terms; per-series copyright and attribution apply", **kwargs) + + @staticmethod + def _id(value: str) -> str: + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9_.-]+", value): + raise ValueError("series_id must be a FRED series identifier") + return value + + @staticmethod + def _date(value: str, name: str) -> str: + if not isinstance(value, str) or not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value): + raise ValueError(f"{name} must be YYYY-MM-DD") + try: + date.fromisoformat(value) + except ValueError: + raise ValueError(f"{name} must be a valid calendar date") from None + return value + + def _today(self) -> str: + return datetime.fromtimestamp(self.store.clock(), timezone.utc).date().isoformat() + + def _v1(self, path: str, params: dict, *, refresh: bool): + return self._request("GET", path, params={"file_type": "json", **params}, + private_params={"api_key": self.api_key}, refresh=refresh) + + def _pages(self, path: str, params: dict, rows_key: str, *, page_size: int, + offset: int, records_seen: int, refresh: bool, convert=None) -> Iterator[Page]: + if (type(offset) is not int or offset < 0 or type(records_seen) is not int + or records_seen < 0 or offset != records_seen): + raise ValueError("offset must match the nonnegative records_seen checkpoint") + params = {**params, "limit": page_size, "offset": offset, "sort_order": "asc"} + expected_total = None + while True: + response = self._v1(path, params, refresh=refresh) + body = response.json() + rows, total = body.get(rows_key), body.get("count") + if (not isinstance(rows, list) or type(total) is not int or total < 0 + or type(body.get("offset")) is not int or body["offset"] != params["offset"] + or len(rows) > page_size): + raise InvalidResponse("fred: invalid results, count or offset") + if expected_total is not None and total != expected_total: + raise InvalidResponse("fred: result count changed during pagination") + expected_total = total + for bound in ("realtime_start", "realtime_end"): + if bound in params and body.get(bound) != params[bound]: + raise InvalidResponse("fred: response does not match requested real-time bounds") + rows = [convert(row) for row in rows] if convert else rows + if any(not isinstance(row, dict) for row in rows): + raise InvalidResponse("fred: invalid record") + observed = records_seen + len(rows) + if observed > total or (observed < total and not rows): + raise InvalidResponse("fred: pagination disagrees with reported count") + complete = observed == total + yield self.page(response, rows, total=total, + next_cursor=None if complete else str(observed), + complete=complete, records_seen=observed) + if complete: + return + records_seen = observed + params["offset"] = observed + + def search(self, query: str, *, page_size: int = 1000, offset: int = 0, + records_seen: int = 0, as_of: str | None = None, + refresh: bool = False) -> Iterator[Page]: + """Search native FRED text; pin the metadata real-time period to one date.""" + positive_int(page_size, 1000, "page_size") + if not isinstance(query, str) or not query.strip(): + raise ValueError("query must be a nonempty FRED search") + vintage = self._date(as_of or self._today(), "as_of") + def record(row): + if not isinstance(row, dict) or not isinstance(row.get("id"), str) or not row["id"]: + raise InvalidResponse("fred: search result lacks series identifier") + return row + yield from self._pages("/series/search", {"search_text": query, + "search_type": "full_text", "order_by": "series_id", + "realtime_start": vintage, "realtime_end": vintage}, "seriess", + page_size=page_size, offset=offset, records_seen=records_seen, + refresh=refresh, convert=record) + + def fetch(self, series_id: str, *, as_of: str | None = None, refresh: bool = False) -> Page: + """Metadata keeps units, frequency, seasonality, notes, and source links.""" + series_id = self._id(series_id) + vintage = self._date(as_of or self._today(), "as_of") + response = self._v1("/series", {"series_id": series_id, + "realtime_start": vintage, "realtime_end": vintage}, refresh=refresh) + body = response.json() + if body.get("realtime_start") != vintage or body.get("realtime_end") != vintage: + raise InvalidResponse("fred: metadata does not match requested real-time bounds") + rows = body.get("seriess") + if not isinstance(rows, list) or len(rows) != 1 or not isinstance(rows[0], dict): + raise InvalidResponse("fred: invalid exact series metadata") + if rows[0].get("id") != series_id: + raise InvalidResponse("fred: exact lookup returned a different series") + return self.page(response, rows, total=1, complete=True, records_seen=1) + + def observations(self, series_id: str, *, as_of: str | None = None, + realtime_start: str | None = None, realtime_end: str | None = None, + observation_start: str | None = None, observation_end: str | None = None, + output_type: int = 1, units: str = "lin", frequency: str | None = None, + aggregation_method: str = "avg", page_size: int = 100_000, + offset: int = 0, records_seen: int = 0, + refresh: bool = False) -> Iterator[Page]: + """ALFRED as-of data or an explicit revision interval, with no float conversion. + + as_of sets both real-time bounds. Without explicit bounds, today's UTC date + is pinned in the request receipt. Fetch series metadata for native units. + output_type=4 requests initial releases; 1 preserves real-time periods. + """ + positive_int(page_size, self.MAX_PAGE_SIZE, "page_size") + positive_int(output_type, 4, "output_type") + series_id = self._id(series_id) + if as_of is not None and (realtime_start is not None or realtime_end is not None): + raise ValueError("provide as_of or explicit real-time bounds") + today = self._today() + start = self._date(as_of or realtime_start or today, "realtime_start") + end = self._date(as_of or realtime_end or today, "realtime_end") + if start > end: + raise ValueError("realtime_start must not exceed realtime_end") + if units not in ("lin", "chg", "ch1", "pch", "pc1", "pca", "cch", "cca", "log"): + raise ValueError("unsupported FRED units transformation") + if aggregation_method not in ("avg", "sum", "eop"): + raise ValueError("unsupported FRED aggregation method") + if frequency is not None and frequency not in ( + "d", "w", "bw", "m", "q", "sa", "a", "wef", "weth", "wew", + "wetu", "wem", "wesu", "wesa", "bwew", "bwem"): + raise ValueError("unsupported FRED frequency") + params = {"series_id": series_id, "realtime_start": start, "realtime_end": end, + "units": units, "output_type": output_type} + if observation_start is not None: + params["observation_start"] = self._date(observation_start, "observation_start") + if observation_end is not None: + params["observation_end"] = self._date(observation_end, "observation_end") + if (observation_start is not None and observation_end is not None + and observation_start > observation_end): + raise ValueError("observation_start must not exceed observation_end") + if frequency is not None: + params.update(frequency=frequency, aggregation_method=aggregation_method) + def record(row): + if not isinstance(row, dict) or not isinstance(row.get("date"), str): + raise InvalidResponse("fred: observation lacks date") + try: + self._date(row["date"], "observation date") + except ValueError: + raise InvalidResponse("fred: invalid observation date") from None + # Output types 2/3 name value columns by vintage; retain all native keys. + values = [value for key, value in row.items() + if key not in ("date", "realtime_start", "realtime_end")] + if not values or any(not isinstance(value, str) for value in values): + raise InvalidResponse("fred: observation values must be native strings") + return row + yield from self._pages("/series/observations", params, "observations", + page_size=page_size, offset=offset, records_seen=records_seen, + refresh=refresh, convert=record) + + def vintages(self, series_id: str, *, page_size: int = 10_000, offset: int = 0, + records_seen: int = 0, refresh: bool = False) -> Iterator[Page]: + """Enumerate dates on which ALFRED published new or revised observations.""" + positive_int(page_size, 10_000, "page_size") + series_id = self._id(series_id) + def record(value): + try: + self._date(value, "vintage_date") + except ValueError: + raise InvalidResponse("fred: invalid vintage date") from None + return {"series_id": series_id, "vintage_date": value} + yield from self._pages("/series/vintagedates", {"series_id": series_id}, "vintage_dates", + page_size=page_size, offset=offset, records_seen=records_seen, + refresh=refresh, convert=record) + + def release_observations(self, release_id: int, *, page_size: int = 500_000, + cursor: str | None = None, records_seen: int = 0, + refresh: bool = False) -> Iterator[Page]: + """Bulk latest data for every series in a release; preserve grouped series. + + page_size counts observations; Page.records counts native series fragments. + A series can span pages. ALFRED observations(as_of=...) is the dated route. + Resume with both next_cursor and records_seen from the last delivered page. + """ + positive_int(release_id, 2_147_483_647, "release_id") + positive_int(page_size, self.MAX_RELEASE_OBSERVATIONS, "page_size") + if (cursor is not None and (not isinstance(cursor, str) or not cursor) + or type(records_seen) is not int or records_seen < 0 + or (cursor is None and records_seen != 0) + or (cursor is not None and not records_seen)): + raise ValueError("cursor and positive records_seen must be supplied together") + params = {"release_id": release_id, "format": "json", "limit": page_size} + if cursor is not None: + params["next_cursor"] = cursor + cursors, versions = set(), {} + while True: + current = params.get("next_cursor") + if current in cursors: + raise InvalidResponse("fred: repeated release cursor") + cursors.add(current) + response = self._request("GET", "/v2/release/observations", params=params, + headers={"Authorization": "Bearer " + self.api_key}, + refresh=refresh) + body = response.json() + release, rows, more = body.get("release"), body.get("series"), body.get("has_more") + if (not isinstance(release, dict) or release.get("release_id") != release_id + or not isinstance(rows, list) or type(more) is not bool): + raise InvalidResponse("fred: invalid release identity or result envelope") + observations = 0 + ids = set() + for row in rows: + if (not isinstance(row, dict) or not isinstance(row.get("series_id"), str) + or not row["series_id"] or row["series_id"] in ids + or not isinstance(row.get("observations"), list) + or any(not isinstance(row.get(key), str) + for key in ("title", "frequency", "units", "last_updated", "copyright_id"))): + raise InvalidResponse("fred: release lacks series identity or metadata") + ident, version = row["series_id"], row["last_updated"] + ids.add(ident) + if ident in versions and versions[ident] != version: + raise InvalidResponse("fred: series revised during release pagination; restart") + versions[ident] = version + dates = set() + for observation in row["observations"]: + if (not isinstance(observation, dict) + or not isinstance(observation.get("date"), str) + or not isinstance(observation.get("value"), str) + or observation["date"] in dates): + raise InvalidResponse("fred: invalid or repeated release observation") + try: + self._date(observation["date"], "observation date") + except ValueError: + raise InvalidResponse("fred: invalid release observation date") from None + dates.add(observation["date"]) + observations += len(row["observations"]) + following = body.get("next_cursor") + if (observations > page_size or (more and (not observations + or not isinstance(following, str) or not following or following in cursors))): + raise InvalidResponse("fred: incomplete release pagination") + records_seen += len(rows) + self.store.record(self.scope, observations_delivered=observations, + upstream_observations=0 if response.cache_hit else observations) + yield self.page(response, rows, next_cursor=following if more else None, + complete=not more, records_seen=records_seen, + warnings=("Latest release data can contain mixed update times; use ALFRED as_of for historical knowledge.", + "Records are series fragments; series can span pages. Page size counts observations.")) + if not more: + return + params["next_cursor"] = following + + def quota(self) -> dict: + return {"api_key_required": True, "daily_provider_quota": None, + "requests_per_minute": 120, "max_requests_per_second": 2, + "local_requests_per_second": self.rate, + "max_series_observations": self.MAX_PAGE_SIZE, + "max_release_observations": self.MAX_RELEASE_OBSERVATIONS, + "note": "v1 uses private query authentication; v2 uses Bearer. Each user needs their own free key."} diff --git a/dedomena/sources/hyperliquid.py b/dedomena/sources/hyperliquid.py new file mode 100644 index 0000000..4cc3d5f --- /dev/null +++ b/dedomena/sources/hyperliquid.py @@ -0,0 +1,332 @@ +"""Hyperliquid public market data with native precision and replayable receipts.""" +from __future__ import annotations + +from decimal import Decimal, InvalidOperation +import json +import math +import re +from typing import Callable, Iterator + +from .core import InvalidResponse, Page, SearchLimitExceeded, Transport, canonical, digest + + +REQUEST_TYPES = frozenset(("allMids", "metaAndAssetCtxs", "spotMetaAndAssetCtxs", "l2Book", + "candleSnapshot", "fundingHistory")) + + +def info_policy(content: bytes) -> tuple[int, Callable[[bytes], int] | None]: + """Validate the fixed read-only info schema and derive its dispatch weight. + + Transport invokes this policy even for direct _request callers. Unsupported + actions and unexpected fields fail before cache lookup or network dispatch. + Response-sized weights reserve a conservative bound and settle by item count. + """ + try: + body = json.loads(content) + except (ValueError, TypeError, UnicodeDecodeError): + raise ValueError("Hyperliquid info body must be JSON") from None + if (not isinstance(body, dict) or not isinstance(body.get("type"), str) + or body["type"] not in REQUEST_TYPES): + raise ValueError("Hyperliquid request type is outside the read-only adapter") + kind = body["type"] + required = { + "allMids": {"type"}, "metaAndAssetCtxs": {"type"}, "spotMetaAndAssetCtxs": {"type"}, + "l2Book": {"type", "coin"}, + "candleSnapshot": {"type", "req"}, "fundingHistory": {"type", "coin", "startTime", "endTime"}, + }[kind] + optional = {"allMids": {"dex"}, "metaAndAssetCtxs": {"dex"}, + "l2Book": {"nSigFigs", "mantissa"}}.get(kind, set()) + if not required <= set(body) <= required | optional: + raise ValueError("Hyperliquid info body has unsupported or missing fields") + if "dex" in body: + Hyperliquid._dex(body["dex"]) + if "coin" in body: + Hyperliquid._coin(body["coin"]) + if kind == "l2Book": + Hyperliquid._aggregation(body.get("nSigFigs"), body.get("mantissa")) + if kind == "fundingHistory": + Hyperliquid._range(body["startTime"], body["endTime"]) + if kind == "candleSnapshot": + request = body["req"] + if not isinstance(request, dict) or set(request) != {"coin", "interval", "startTime", "endTime"}: + raise ValueError("Hyperliquid candle request has invalid fields") + Hyperliquid._coin(request["coin"]) + Hyperliquid._interval(request["interval"]) + Hyperliquid._range(request["startTime"], request["endTime"]) + if kind in ("allMids", "l2Book"): + return 2, None + if kind in ("metaAndAssetCtxs", "spotMetaAndAssetCtxs"): + return 20, None + maximum, per_unit = {"fundingHistory": (500, 20), + "candleSnapshot": (5000, 60)}[kind] + def actual_weight(content: bytes) -> int: + rows = json.loads(content) + if (not isinstance(rows, list) or len(rows) > maximum + or any(not isinstance(row, dict) for row in rows)): + raise ValueError("invalid Hyperliquid item-count response") + return 20 + math.ceil(len(rows) / per_unit) + return 20 + math.ceil(maximum / per_unit), actual_weight + + +class Hyperliquid(Transport): + """Restricted public /info queries; no account, signing, or exchange actions. + + Snapshot records preserve the native response envelope. Historical funding + pages remove exact inclusive-boundary repeats; candles retain the provider's + recent-history limitation rather than claiming complete historical coverage. + """ + + MAX_FUNDING_PAGE = 500 + MAX_CANDLES = 5000 + INTERVALS = frozenset(("1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", + "8h", "12h", "1d", "3d", "1w", "1M")) + + def __init__(self, **kwargs): + kwargs.setdefault("requests_per_second", 1000) + kwargs.setdefault("ip_limit", (1200, 60.0)) + limit = kwargs["ip_limit"] + if (not isinstance(limit, tuple) or len(limit) != 2 or type(limit[0]) is not int + or not 0 < limit[0] <= 1200 or isinstance(limit[1], bool) + or not isinstance(limit[1], (int, float)) or not math.isfinite(limit[1]) or limit[1] < 60): + raise ValueError("Hyperliquid IP policy cannot exceed 1,200 weight per minute") + kwargs["ip_throttle_only"] = True + kwargs.setdefault("cache_ttl", 0) + super().__init__("hyperliquid", "https://api.hyperliquid.xyz", + license="Hyperliquid public API; provider terms apply", **kwargs) + + @staticmethod + def _coin(coin: str) -> str: + if (not isinstance(coin, str) or len(coin) > 128 or not re.fullmatch( + r"(?:[A-Za-z0-9_]+:)?[A-Za-z0-9_@#]+(?:/[A-Za-z0-9_]+)?", coin)): + raise ValueError("coin must be a native Hyperliquid asset name") + return coin + + @staticmethod + def _dex(dex: str) -> str: + if not isinstance(dex, str) or not re.fullmatch(r"[A-Za-z0-9_-]{0,64}", dex): + raise ValueError("dex must be a Hyperliquid perpetual dex name") + return dex + + @staticmethod + def _time(value: int, name: str) -> int: + if type(value) is not int or not 0 <= value <= 9_007_199_254_740_991: + raise ValueError(f"{name} must be a nonnegative integer timestamp in milliseconds") + return value + + @classmethod + def _range(cls, start_time: int, end_time: int) -> tuple[int, int]: + start, end = cls._time(start_time, "start_time"), cls._time(end_time, "end_time") + if start > end: + raise ValueError("start_time must not follow end_time") + return start, end + + @classmethod + def _interval(cls, interval: str) -> str: + if not isinstance(interval, str) or interval not in cls.INTERVALS: + raise ValueError("interval must be a supported Hyperliquid candle interval") + return interval + + @staticmethod + def _aggregation(n_sig_figs, mantissa): + if n_sig_figs is not None and (type(n_sig_figs) is not int or n_sig_figs not in (2, 3, 4, 5)): + raise ValueError("n_sig_figs must be 2, 3, 4, 5 or None") + if mantissa is not None and (type(mantissa) is not int or mantissa not in (1, 2, 5) or n_sig_figs != 5): + raise ValueError("mantissa must be 1, 2 or 5 and requires n_sig_figs=5") + + @staticmethod + def _decimal(value, *, nonnegative: bool = False) -> bool: + if not isinstance(value, str) or not value or len(value) > 256: + return False + try: + number = Decimal(value) + return number.is_finite() and (not nonnegative or number >= 0) + except InvalidOperation: + return False + + def _info(self, body: dict, *, refresh: bool): + content = canonical(body).encode() + weight, actual = info_policy(content) + response = self._request("POST", "/info", content=content, + headers={"Content-Type": "application/json"}, + rate_weight=weight, actual_weight=actual, refresh=refresh) + try: + data = json.loads(response.body) + except (ValueError, UnicodeDecodeError): + raise InvalidResponse("hyperliquid: invalid JSON response") from None + return response, data + + def all_mids(self, *, dex: str = "", refresh: bool = False) -> Page: + """One native coin-to-price map; spot mids belong to the default dex.""" + dex = self._dex(dex) + response, data = self._info({"type": "allMids", "dex": dex}, refresh=refresh) + if (not isinstance(data, dict) or any(not isinstance(coin, str) or not coin + or not self._decimal(price, nonnegative=True) for coin, price in data.items())): + raise InvalidResponse("hyperliquid: invalid mids response") + return self.page(response, [data], total=1, complete=True, records_seen=1, + warnings=("Point-in-time mids; an empty book can fall back to the last trade price.",)) + + def markets(self, *, spot: bool = False, dex: str = "", refresh: bool = False) -> Page: + """One native metadata/context envelope; spot contexts can outnumber pairs.""" + dex = self._dex(dex) + if type(spot) is not bool or (spot and dex): + raise ValueError("spot must be boolean; spot metadata does not accept a dex") + body = {"type": "spotMetaAndAssetCtxs" if spot else "metaAndAssetCtxs"} + if not spot: + body["dex"] = dex + response, data = self._info(body, refresh=refresh) + if (not isinstance(data, list) or len(data) != 2 or not isinstance(data[0], dict) + or not isinstance(data[1], list) or not isinstance(data[0].get("universe"), list)): + raise InvalidResponse("hyperliquid: invalid market metadata envelope") + meta, contexts = data + universe = meta["universe"] + if ((not spot and len(universe) != len(contexts)) or any(not isinstance(row, dict) + or not isinstance(row.get("name"), str) or not row["name"] for row in universe) + or len({row["name"] for row in universe}) != len(universe) + or any(not isinstance(row, dict) for row in contexts) + or (spot and (not isinstance(meta.get("tokens"), list) + or any(not isinstance(row, dict) for row in meta["tokens"])))): + raise InvalidResponse("hyperliquid: inconsistent market metadata or asset contexts") + return self.page(response, [{"meta": meta, "assetCtxs": contexts}], total=1, + complete=True, records_seen=1, + warnings=(("Spot contexts and pair metadata have independent lengths; native coin fields and token indices are retained." + if spot else "Perpetual asset contexts match universe positions; native decimal strings are retained."),)) + + def order_book(self, coin: str, *, n_sig_figs: int | None = None, + mantissa: int | None = None, refresh: bool = False) -> Page: + """Native L2 snapshot, at most twenty levels on each side.""" + coin = self._coin(coin) + self._aggregation(n_sig_figs, mantissa) + body = {"type": "l2Book", "coin": coin} + if n_sig_figs is not None: + body["nSigFigs"] = n_sig_figs + if mantissa is not None: + body["mantissa"] = mantissa + response, data = self._info(body, refresh=refresh) + if (not isinstance(data, dict) or data.get("coin") != coin + or type(data.get("time")) is not int or data["time"] < 0 + or not isinstance(data.get("levels"), list) or len(data["levels"]) != 2 + or any(not isinstance(side, list) or len(side) > 20 for side in data["levels"])): + raise InvalidResponse("hyperliquid: invalid book identity or levels") + for side in data["levels"]: + if any(not isinstance(row, dict) or not self._decimal(row.get("px"), nonnegative=True) + or not self._decimal(row.get("sz"), nonnegative=True) + or type(row.get("n")) is not int or row["n"] < 0 for row in side): + raise InvalidResponse("hyperliquid: invalid book level") + return self.page(response, [data], total=1, complete=True, records_seen=1, + warnings=("Point-in-time L2 snapshot; the provider returns at most 20 levels per side.",)) + + def candles(self, coin: str, interval: str, start_time: int, end_time: int, + *, refresh: bool = False) -> Page: + """Return available candles; only the provider's latest 5,000 exist here.""" + coin, interval = self._coin(coin), self._interval(interval) + start, end = self._range(start_time, end_time) + response, rows = self._info({"type": "candleSnapshot", "req": { + "coin": coin, "interval": interval, "startTime": start, "endTime": end}}, refresh=refresh) + if not isinstance(rows, list) or len(rows) > self.MAX_CANDLES: + raise InvalidResponse("hyperliquid: invalid candle response") + previous = -1 + for row in rows: + if (not isinstance(row, dict) or row.get("s") != coin or row.get("i") != interval + or type(row.get("t")) is not int or type(row.get("T")) is not int + or row["t"] < 0 or row["T"] < row["t"] or row["t"] > end or row["T"] < start + or row["t"] <= previous or type(row.get("n")) is not int or row["n"] < 0 + or any(not self._decimal(row.get(key), nonnegative=True) for key in ("o", "h", "l", "c", "v"))): + raise InvalidResponse("hyperliquid: invalid candle identity, range or values") + previous = row["t"] + return self.page(response, rows, complete=False, records_seen=len(rows), + warnings=("Only the latest 5,000 candles are retained; a short or empty response does not prove historical coverage.", + "The current interval can be unfinished; OHLCV values remain native decimal strings.")) + + def funding_history(self, coin: str, start_time: int, end_time: int | None = None, + *, cursor: str | None = None, refresh: bool = False) -> Iterator[Page]: + """Enumerate a pinned inclusive range with safe timestamp overlap. + + Resume with the same coin/start/end and the preceding Page.next_cursor. + If end_time is omitted, the cursor retains the initial retrieval cutoff. + The opaque cursor contains the count and hashes of already emitted rows + at the inclusive boundary. A saturated timestamp cannot be enumerated + safely and raises SearchLimitExceeded before that page is delivered. + """ + coin = self._coin(coin) + start = self._time(start_time, "start_time") + if end_time is not None: + self._time(end_time, "end_time") + current, seen, boundary = start, 0, set() + if cursor is not None: + if not isinstance(cursor, str) or len(cursor) > 50_000: + raise ValueError("cursor must be a funding checkpoint for this exact query") + try: + state = json.loads(cursor) + if (not isinstance(state, dict) or set(state) != {"coin", "start", "end", "after", "seen", "boundary"} + or state["coin"] != coin or state["start"] != start + or (end_time is not None and state["end"] != end_time) + or type(state["seen"]) is not int or state["seen"] <= 0 + or not isinstance(state["boundary"], list) or not state["boundary"] + or len(state["boundary"]) > self.MAX_FUNDING_PAGE + or any(not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) + for value in state["boundary"])): + raise ValueError + end = self._time(state["end"], "cursor end") + current = self._time(state["after"], "cursor after") + if not start < current <= end: + raise ValueError + seen, boundary = state["seen"], set(state["boundary"]) + except (ValueError, TypeError, KeyError): + raise ValueError("cursor must be a funding checkpoint for this exact query") from None + else: + end = self._time(end_time if end_time is not None else int(self.store.clock() * 1000), "end_time") + self._range(start, end) + while True: + response, rows = self._info({"type": "fundingHistory", "coin": coin, + "startTime": current, "endTime": end}, refresh=refresh) + if not isinstance(rows, list) or len(rows) > self.MAX_FUNDING_PAGE: + raise InvalidResponse("hyperliquid: invalid funding history response") + previous, hashes = current, set() + emitted = [] + for row in rows: + if (not isinstance(row, dict) or row.get("coin") != coin + or type(row.get("time")) is not int or not current <= row["time"] <= end + or row["time"] < previous or not self._decimal(row.get("fundingRate")) + or not self._decimal(row.get("premium"))): + raise InvalidResponse("hyperliquid: invalid funding identity, order or range") + ident = digest(canonical(row).encode()) + if ident in hashes: + raise InvalidResponse("hyperliquid: duplicate funding row in response") + hashes.add(ident) + previous = row["time"] + if not (row["time"] == current and ident in boundary): + emitted.append(row) + more = len(rows) == self.MAX_FUNDING_PAGE + if more and previous == current: + raise SearchLimitExceeded("hyperliquid: funding timestamp reaches the page cap; narrow the range") + seen += len(emitted) + following = None + if more: + boundary = {digest(canonical(row).encode()) for row in rows if row["time"] == previous} + following = canonical({"coin": coin, "start": start, "end": end, + "after": previous, "seen": seen, "boundary": sorted(boundary)}) + if len(rows) > len(emitted): + self.store.record(self.scope, upstream_records=0 if response.cache_hit else len(rows) - len(emitted), + boundary_repeats=len(rows) - len(emitted)) + yield self.page(response, emitted, complete=not more, next_cursor=following, + records_seen=seen, + warnings=("Inclusive funding timestamp overlap is deduplicated by exact row hash; this is provider-available history.",)) + if not more: + return + current = previous + + def fetch(self, identifier: str, *, refresh: bool = False) -> Page: + return self.order_book(identifier, refresh=refresh) + + def search(self, query: str, *, refresh: bool = False) -> Iterator[Page]: + yield self.fetch(query, refresh=refresh) + + def quota(self) -> dict: + return {"api_key_required": False, "daily_provider_quota": None, + "ip_weight_per_minute": 1200, "ip_weight_per_second_average": 20, + "local_requests_per_second": self.rate, + "reserved_request_weights": {"all_mids": 2, "order_book": 2, "markets": 20, + "funding_history": 45, "candles": 104}, + "max_candles_retained": self.MAX_CANDLES, + "funding_page_cap": self.MAX_FUNDING_PAGE, + "note": "Response-sized requests reserve maximum weight, then use rounded-up returned counts for local accounting; provider billing is not reported. IP pools do not expand address limits."} diff --git a/dedomena/sources/openalex.py b/dedomena/sources/openalex.py index 08d7b28..88ff4c5 100644 --- a/dedomena/sources/openalex.py +++ b/dedomena/sources/openalex.py @@ -45,6 +45,11 @@ def __init__(self, api_key: str | None = None, *, daily_credit_limit: int = 10_0 kwargs.setdefault("requests_per_second", 100) if kwargs["requests_per_second"] > 100: raise ValueError("OpenAlex permits at most 100 requests per second") + ip_limit = kwargs.setdefault("ip_limit", (100, 1.0)) + if (not isinstance(ip_limit, tuple) or len(ip_limit) != 2 + or type(ip_limit[0]) is not int or not 1 <= ip_limit[0] <= 100 + or type(ip_limit[1]) not in (int, float) or ip_limit[1] != 1): + raise ValueError("OpenAlex IP policy permits at most 100 requests per second") super().__init__("openalex", "https://api.openalex.org", credential=self.api_key, license="CC0 metadata; article text has separate rights", **kwargs) diff --git a/dedomena/sources/sec.py b/dedomena/sources/sec.py new file mode 100644 index 0000000..f6fbe67 --- /dev/null +++ b/dedomena/sources/sec.py @@ -0,0 +1,273 @@ +"""SEC EDGAR: complete filing histories and native XBRL units and vintages.""" +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from datetime import date +import math +import os +import re +from typing import Iterable, Iterator + +from .core import InvalidResponse, Page, Response, Transport, chunks, positive_int + + +class SEC(Transport): + """Keyless EDGAR JSON with shared 10 requests/s pacing and declared contact.""" + + MAX_REQUESTS_PER_SECOND = 10 + BULK_ARCHIVES = { + "companyfacts": "https://www.sec.gov/Archives/edgar/daily-index/xbrl/companyfacts.zip", + "submissions": "https://www.sec.gov/Archives/edgar/daily-index/bulkdata/submissions.zip", + } + + def __init__(self, user_agent: str | None = None, **kwargs): + user_agent = user_agent if user_agent is not None else os.environ.get("SEC_USER_AGENT", "") + if (not isinstance(user_agent, str) or "\n" in user_agent or "\r" in user_agent + or not re.search(r"\S+\s+[^\s@]+@[^\s@]+[.][^\s@]+", user_agent)): + raise ValueError("SEC requires SEC_USER_AGENT with an organization and contact email") + self.user_agent = user_agent + if kwargs.pop("credential", ""): + raise ValueError("SEC keyless clients must share the aggregate pacing scope") + kwargs.setdefault("requests_per_second", 10) + if kwargs["requests_per_second"] > 10: + raise ValueError("SEC permits at most 10 requests per second across all clients") + # No credential or User-Agent namespaces: every SEC client in this store shares pacing. + kwargs.setdefault("max_response_bytes", 64_000_000) + super().__init__("sec", "https://data.sec.gov", + credential="", + license="Public EDGAR data; SEC terms and underlying filing rights apply", **kwargs) + + @staticmethod + def _cik(value: str | int) -> str: + if isinstance(value, str): + value = value.removeprefix("CIK") + if not re.fullmatch(r"[0-9]{1,10}", value): + raise ValueError("CIK must contain 1-10 digits") + value = int(value) + if type(value) is not int or not 1 <= value <= 9_999_999_999: + raise ValueError("CIK must be a positive integer with at most 10 digits") + return f"{value:010d}" + + @staticmethod + def _name(value: str, kind: str) -> str: + if not isinstance(value, str) or not re.fullmatch(r"[A-Za-z][A-Za-z0-9_.-]*", value): + raise ValueError(f"invalid SEC {kind}") + return value + + def _get(self, path: str, *, refresh: bool = False) -> Response: + return self._request("GET", path, headers={"User-Agent": self.user_agent, + "Accept": "application/json"}, refresh=refresh) + + @staticmethod + def _identity(body: dict, cik: str): + value = body.get("cik") + valid = type(value) is int or (isinstance(value, str) and re.fullmatch(r"[0-9]{1,10}", value)) + if not valid or int(value) != int(cik): + raise InvalidResponse("sec: response does not match requested CIK") + + @staticmethod + def _facts(units: object): + if not isinstance(units, dict): + raise InvalidResponse("sec: XBRL concept lacks units") + for unit, rows in units.items(): + if not isinstance(unit, str) or not unit or not isinstance(rows, list): + raise InvalidResponse("sec: invalid XBRL units") + for row in rows: + SEC._fact(row) + + @staticmethod + def _fact(row: object): + if (not isinstance(row, dict) + or not isinstance(row.get("accn"), str) + or not re.fullmatch(r"[0-9]{10}-[0-9]{2}-[0-9]{6}", row["accn"]) + or type(row.get("val")) not in (int, float) + or not math.isfinite(row["val"])): + raise InvalidResponse("sec: fact lacks accession number or finite numeric value") + for key in ("end", "start", "filed"): + if key == "start" and key not in row: + continue + if key == "filed" and key not in row: + continue # Frames expose accession and period, but no filing date. + value = row.get(key) + try: + if not isinstance(value, str): + raise ValueError + date.fromisoformat(value) + except ValueError: + raise InvalidResponse("sec: invalid XBRL observation date") from None + + def company_facts(self, cik: str | int, *, refresh: bool = False) -> Page: + """One call retains every taxonomy, concept, unit and filed observation.""" + ident = self._cik(cik) + response = self._get(f"/api/xbrl/companyfacts/CIK{ident}.json", refresh=refresh) + body = response.json() + self._identity(body, ident) + taxonomies = body.get("facts") + if not isinstance(taxonomies, dict): + raise InvalidResponse("sec: company facts lack taxonomy data") + for taxonomy, concepts in taxonomies.items(): + if not isinstance(taxonomy, str) or not isinstance(concepts, dict): + raise InvalidResponse("sec: invalid company taxonomy") + for tag, concept in concepts.items(): + if not isinstance(tag, str) or not isinstance(concept, dict): + raise InvalidResponse("sec: invalid company concept") + self._facts(concept.get("units")) + return self.page(response, [body], total=1, complete=True, records_seen=1, + warnings=("Units and filing vintages remain native; no latest-value selection or currency conversion.",)) + + def fetch(self, cik: str | int, *, refresh: bool = False) -> Page: + return self.company_facts(cik, refresh=refresh) + + def fetch_many(self, ciks: Iterable[str | int], *, workers: int = 4, + refresh: bool = False) -> Iterator[Page]: + """Bounded company reads preserve caller order and shared aggregate pacing.""" + positive_int(workers, 10, "workers") + with ThreadPoolExecutor(max_workers=workers) as pool: + for batch in chunks(ciks, workers): + pending = {} + for cik in batch: + ident = self._cik(cik) + if ident not in pending: + pending[ident] = pool.submit(self.company_facts, ident, refresh=refresh) + for cik in batch: + yield pending[self._cik(cik)].result() + + def company_concept(self, cik: str | int, taxonomy: str, tag: str, + *, refresh: bool = False) -> Page: + """A lighter exact concept read, retaining distinct units and amended filings.""" + ident = self._cik(cik) + taxonomy, tag = self._name(taxonomy, "taxonomy"), self._name(tag, "tag") + response = self._get(f"/api/xbrl/companyconcept/CIK{ident}/{taxonomy}/{tag}.json", + refresh=refresh) + body = response.json() + self._identity(body, ident) + if body.get("taxonomy") != taxonomy or body.get("tag") != tag: + raise InvalidResponse("sec: company concept does not match requested taxonomy and tag") + self._facts(body.get("units")) + return self.page(response, [body], total=1, complete=True, records_seen=1) + + def frame(self, taxonomy: str, tag: str, unit: str, period: str, + *, refresh: bool = False) -> Page: + """Native cross-company calendar frame; this is SEC's latest matching filing view.""" + taxonomy, tag = self._name(taxonomy, "taxonomy"), self._name(tag, "tag") + unit = self._name(unit, "unit") + if not isinstance(period, str) or not re.fullmatch(r"CY[0-9]{4}(?:Q[1-4]I?)?", period): + raise ValueError("period must be CYyyyy, CYyyyyQq, or CYyyyyQqI") + response = self._get(f"/api/xbrl/frames/{taxonomy}/{tag}/{unit}/{period}.json", + refresh=refresh) + body = response.json() + if (body.get("taxonomy"), body.get("tag"), body.get("uom"), body.get("ccp")) != (taxonomy, tag, unit.replace("-per-", "/"), period): + raise InvalidResponse("sec: frame does not match requested concept, unit and period") + rows, count = body.get("data"), body.get("pts") + if not isinstance(rows, list) or type(count) is not int or count != len(rows): + raise InvalidResponse("sec: frame lacks its reported observations") + seen = set() + for row in rows: + self._fact(row) + cik = row.get("cik") + if type(cik) is not int or not 1 <= cik <= 9_999_999_999 or cik in seen: + raise InvalidResponse("sec: frame contains an invalid or repeated CIK") + seen.add(cik) + return self.page(response, [body], total=1, complete=True, records_seen=1, + warnings=("Frame selects the latest filing matching a calendar period; company fiscal dates can differ.",)) + + @staticmethod + def _filing_rows(columns: object) -> list[dict]: + if not isinstance(columns, dict) or not isinstance(columns.get("accessionNumber"), list): + raise InvalidResponse("sec: filings lack accession numbers") + count = len(columns["accessionNumber"]) + if any(not isinstance(value, list) or len(value) != count for value in columns.values()): + raise InvalidResponse("sec: filing columns have unequal lengths") + rows = [{name: values[index] for name, values in columns.items()} for index in range(count)] + for row in rows: + accession = row["accessionNumber"] + if not isinstance(accession, str) or not re.fullmatch(r"[0-9]{10}-[0-9]{2}-[0-9]{6}", accession): + raise InvalidResponse("sec: invalid filing accession number") + if len({row["accessionNumber"] for row in rows}) != len(rows): + raise InvalidResponse("sec: repeated filing accession number") + return rows + + def submissions(self, cik: str | int, *, include_history: bool = True, + cursor: str = "recent", records_seen: int = 0, + refresh: bool = False) -> Iterator[Page]: + """Enumerate recent AND additional filing files; mark only the final page complete. + + Resume with a returned filename cursor and records_seen checkpoint. SEC's live + manifest may change; a changed checkpoint fails instead of claiming completeness. + """ + ident = self._cik(cik) + if type(include_history) is not bool: + raise ValueError("include_history must be boolean") + if type(records_seen) is not int or records_seen < 0 or not isinstance(cursor, str): + raise ValueError("invalid SEC filing checkpoint") + response = self._get(f"/submissions/CIK{ident}.json", refresh=refresh) + body = response.json() + self._identity(body, ident) + filings = body.get("filings") + if not isinstance(filings, dict): + raise InvalidResponse("sec: submissions lack filings") + recent = self._filing_rows(filings.get("recent")) + files = filings.get("files") + if not isinstance(files, list): + raise InvalidResponse("sec: submissions lack historical file manifest") + names, counts = [], [] + for descriptor in files: + if (not isinstance(descriptor, dict) + or not isinstance(descriptor.get("name"), str) + or not re.fullmatch(f"CIK{ident}-submissions-[0-9]+[.]json", descriptor["name"]) + or type(descriptor.get("filingCount")) is not int + or descriptor["filingCount"] < 0): + raise InvalidResponse("sec: invalid historical filing file descriptor") + names.append(descriptor["name"]) + counts.append(descriptor["filingCount"]) + if len(set(names)) != len(names): + raise InvalidResponse("sec: repeated historical filing file") + total = len(recent) + sum(counts) + metadata_warning = "Filing rows retain native columns; the full company metadata and columnar envelope are archived in the raw snapshot." + if cursor == "recent": + if records_seen: + raise ValueError("initial SEC filings request cannot have a records_seen checkpoint") + observed, begin = len(recent), 0 + next_cursor = names[0] if names else None + warnings = (metadata_warning,) + if names and not include_history: + warnings += ("Historical filing files were not requested; enumeration is incomplete.",) + yield self.page(response, recent, total=total, next_cursor=next_cursor, + complete=not names, records_seen=observed, warnings=warnings) + else: + if cursor not in names: + raise ValueError("SEC cursor is absent from the current historical manifest") + begin = names.index(cursor) + expected = len(recent) + sum(counts[:begin]) + if records_seen != expected: + raise ValueError("SEC checkpoint differs from the current historical manifest") + observed = records_seen + if not include_history: + return + all_seen = {row["accessionNumber"] for row in recent} if cursor == "recent" else set() + for index in range(begin, len(names)): + historical = self._get("/submissions/" + names[index], refresh=refresh) + rows = self._filing_rows(historical.json()) + if len(rows) != counts[index]: + raise InvalidResponse("sec: historical filing count differs from its manifest") + if any(row["accessionNumber"] in all_seen for row in rows): + raise InvalidResponse("sec: filing accession repeated across history pages") + all_seen.update(row["accessionNumber"] for row in rows) + observed += len(rows) + complete = index + 1 == len(names) + yield self.page(historical, rows, total=total, + next_cursor=None if complete else names[index + 1], + complete=complete, records_seen=observed, + warnings=("Historical filing rows retain native columns; company identity is the CIK in the source URL.",)) + + def search(self, cik: str | int, *, refresh: bool = False) -> Iterator[Page]: + """The consistent discovery operation is complete filing history for an exact CIK.""" + return self.submissions(cik, refresh=refresh) + + def quota(self) -> dict: + return {"api_key_required": False, "identifying_user_agent_required": True, + "provider_max_requests_per_second": 10, + "local_requests_per_second": self.rate, + "daily_provider_quota": None, + "bulk_archives": dict(self.BULK_ARCHIVES), + "note": "SEC's 10 requests/s limit aggregates all clients; one store shares pacing across User-Agents. Bulk ZIPs are nightly alternatives for corpus-scale ingestion."} diff --git a/dedomena/sources/worldbank.py b/dedomena/sources/worldbank.py new file mode 100644 index 0000000..42e3d20 --- /dev/null +++ b/dedomena/sources/worldbank.py @@ -0,0 +1,196 @@ +"""World Bank Indicators v2: 60-indicator batches and large, counted pages.""" +from __future__ import annotations + +import json +import math +import re +from typing import Iterable, Iterator + +import httpx + +from .core import InvalidResponse, Page, Transport, positive_int + + +class WorldBank(Transport): + # Documentation publishes no page ceiling. 32,767 worked in bounded probes; + # 50,000 was rejected with HTTP 400. Keep the verified signed-short ceiling. + MAX_PAGE_SIZE = 32_767 + MAX_INDICATORS = 60 + + def __init__(self, **kwargs): + # Configurable local pacing, not a claimed provider allowance. + kwargs.setdefault("requests_per_second", 5) + kwargs.setdefault("max_response_bytes", 32_000_000) + super().__init__("worldbank", "https://api.worldbank.org/v2", + license="World Bank CC BY 4.0 with additional terms; metadata exceptions apply", + **kwargs) + + @staticmethod + def _codes(values: str | Iterable[str], name: str, *, maximum: int | None = None) -> list[str]: + codes = values.split(";") if isinstance(values, str) else list(values) + pattern = r"[A-Za-z0-9_.-]+" if name == "indicators" else r"[A-Za-z0-9]+" + if (not codes or any(not isinstance(code, str) or not re.fullmatch(pattern, code) for code in codes) + or any(code in (".", "..") for code in codes) + or len(set(codes)) != len(codes)): + raise ValueError(f"{name} must contain distinct World Bank codes") + if maximum is not None and len(codes) > maximum: + raise ValueError(f"at most {maximum} {name} fit one World Bank query") + if len(";".join(codes)) > 1500: + raise ValueError(f"{name} exceeds World Bank's 1,500-character path segment limit") + return codes + + @staticmethod + def _dates(value: str) -> str: + if not isinstance(value, str): + raise ValueError("date must be a World Bank year, month, quarter or range") + parts = value.split(":") + if len(parts) not in (1, 2): + raise ValueError("date must be a World Bank period or range") + parsed = [] + for part in parts: + match = re.fullmatch(r"([0-9]{4})(?:(M)([0-9]{2})|(Q)([1-4]))?", part) + if not match or match[1] == "0000": + raise ValueError("date must contain valid years, YYYYQn or YYYYMnn") + if match[2] and not 1 <= int(match[3]) <= 12: + raise ValueError("date contains an invalid month") + parsed.append((int(match[1]), match[2] or match[4] or "Y", int(match[3] or match[5] or 1))) + if len(parsed) == 2 and (parsed[0][1] != parsed[1][1] or parsed[0] > parsed[1]): + raise ValueError("date range must use one frequency with start no later than end") + return value + + def _pages(self, path: str, params: dict, *, page_size: int, page: int, + records_seen: int, refresh: bool, validate=None, warnings=()) -> Iterator[Page]: + positive_int(page_size, self.MAX_PAGE_SIZE, "page_size") + positive_int(page, 2_147_483_647, "page") + if (type(records_seen) is not int or records_seen < 0 + or records_seen != (page - 1) * page_size): + raise ValueError("page resume requires records_seen=(page-1)*page_size") + params = {**params, "format": "json", "per_page": page_size, "page": page} + if len(str(httpx.Request("GET", self.base_url + path, params=params).url)) > 4000: + raise ValueError("World Bank query exceeds its 4,000-character URL limit") + expected_total, update = None, None + while True: + response = self._request("GET", path, params=params, refresh=refresh) + try: + body = json.loads(response.body) + except (ValueError, UnicodeDecodeError): + raise InvalidResponse("worldbank: invalid JSON response") from None + if (not isinstance(body, list) or len(body) != 2 or not isinstance(body[0], dict) + or not (isinstance(body[1], list) or body[1] is None)): + raise InvalidResponse("worldbank: invalid result envelope") + meta, rows = body[0], body[1] or [] + total, pages = meta.get("total"), meta.get("pages") + # Observation endpoints use an integer; metadata endpoints use a + # decimal string for this same documented pagination field. + returned_size = meta.get("per_page") + if isinstance(returned_size, str) and returned_size.isascii() and returned_size.isdigit(): + returned_size = int(returned_size) + if (type(total) is not int or total < 0 or type(pages) is not int or pages < 0 + or type(meta.get("page")) is not int or meta["page"] != params["page"] + or type(returned_size) is not int or returned_size != page_size + or len(rows) > page_size): + raise InvalidResponse("worldbank: invalid count or page metadata") + if pages != math.ceil(total / page_size): + raise InvalidResponse("worldbank: page count disagrees with total") + if expected_total is not None and (total != expected_total or meta.get("lastupdated") != update): + raise InvalidResponse("worldbank: dataset changed during pagination; restart") + expected_total, update = total, meta.get("lastupdated") + if any(not isinstance(row, dict) for row in rows): + raise InvalidResponse("worldbank: invalid record") + if validate: + for row in rows: + validate(row) + observed = records_seen + len(rows) + complete = observed == total + if observed > total or (not complete and len(rows) != page_size): + raise InvalidResponse("worldbank: pagination ended before reported count") + following = params["page"] + 1 + if not complete and following > pages: + raise InvalidResponse("worldbank: no next page before reported total") + yield self.page(response, rows, total=total, + next_cursor=None if complete else str(following), complete=complete, + records_seen=observed, warnings=warnings) + if complete: + return + records_seen = observed + params["page"] = following + + def search(self, query: str | Iterable[str], *, countries: str | Iterable[str] = "all", + source: int = 2, date: str | None = None, footnotes: bool = False, + page_size: int = 32_767, page: int = 1, records_seen: int = 0, + refresh: bool = False) -> Iterator[Page]: + """Fetch observations for up to 60 exact indicator codes in one dataset. + + all includes countries and regional/income aggregates. Null observations, + country/indicator IDs, dates, units, decimals and footnotes remain native. + """ + indicators = self._codes(query, "indicators", maximum=self.MAX_INDICATORS) + countries = self._codes(countries, "countries") + all_countries = any(code.upper() == "ALL" for code in countries) + if all_countries: + if len(countries) != 1: + raise ValueError("all cannot be combined with country codes") + countries = ["all"] + positive_int(source, 2_147_483_647, "source") + if type(footnotes) is not bool: + raise ValueError("footnotes must be boolean") + params = {"source": source} + if date is not None: + params["date"] = self._dates(date) + if footnotes: + params["footnote"] = "y" + wanted = {value.upper() for value in indicators} + selected_countries = {value.upper() for value in countries} + def validate(row): + indicator, country = row.get("indicator"), row.get("country") + if (not isinstance(indicator, dict) or not isinstance(indicator.get("id"), str) + or indicator["id"].upper() not in wanted + or not isinstance(country, dict) or not isinstance(country.get("id"), str) + or not country["id"] or not isinstance(row.get("date"), str) + or "value" not in row): + raise InvalidResponse("worldbank: observation lacks requested indicator or identity") + if not all_countries: + actual = {country["id"].upper(), str(row.get("countryiso3code", "")).upper()} + if not actual.intersection(selected_countries): + raise InvalidResponse("worldbank: response contains an unrequested country") + path = "/country/" + ";".join(countries) + "/indicator/" + ";".join(indicators) + yield from self._pages(path, params, page_size=page_size, page=page, + records_seen=records_seen, refresh=refresh, validate=validate, + warnings=("Latest observations can be revised; historical vintages are not supplied by this endpoint.",)) + + def observations(self, indicators: str | Iterable[str], **kwargs) -> Iterator[Page]: + return self.search(indicators, **kwargs) + + def indicators(self, *, source: int = 2, page_size: int = 32_767, + page: int = 1, records_seen: int = 0, refresh: bool = False) -> Iterator[Page]: + """Discover indicator metadata including units, agency and source notes.""" + positive_int(source, 2_147_483_647, "source") + def validate(row): + if not isinstance(row.get("id"), str) or not row["id"]: + raise InvalidResponse("worldbank: indicator metadata lacks identifier") + if not isinstance(row.get("source"), dict) or str(row["source"].get("id")) != str(source): + raise InvalidResponse("worldbank: indicator metadata belongs to a different dataset") + yield from self._pages("/indicator", {"source": source}, page_size=page_size, + page=page, records_seen=records_seen, refresh=refresh, validate=validate) + + def fetch(self, indicator: str, *, source: int = 2, refresh: bool = False) -> Page: + """Exact indicator metadata; source defaults to World Development Indicators.""" + ident = self._codes(indicator, "indicators", maximum=1)[0] + positive_int(source, 2_147_483_647, "source") + def validate(row): + if not isinstance(row.get("id"), str) or row["id"].upper() != ident.upper(): + raise InvalidResponse("worldbank: exact lookup returned a different indicator") + if not isinstance(row.get("source"), dict) or str(row["source"].get("id")) != str(source): + raise InvalidResponse("worldbank: indicator metadata belongs to a different dataset") + pages = self._pages("/indicator/" + ident, {"source": source}, page_size=self.MAX_PAGE_SIZE, + page=1, records_seen=0, refresh=refresh, validate=validate) + result = next(pages) + if not result.complete or len(result.records) != 1: + raise InvalidResponse("worldbank: exact indicator metadata is missing or ambiguous") + return result + + def quota(self) -> dict: + return {"api_key_required": False, "daily_provider_quota": None, + "local_requests_per_second": self.rate, "max_indicators_per_query": 60, + "local_max_page_size": self.MAX_PAGE_SIZE, + "note": "No published daily/rate/page ceiling. 32,767 per page was accepted and 50,000 rejected in bounded probes."} diff --git a/docs/FINANCE.md b/docs/FINANCE.md new file mode 100644 index 0000000..be97d03 --- /dev/null +++ b/docs/FINANCE.md @@ -0,0 +1,222 @@ +# Finance sources + +Four official APIs cover corporate fundamentals, dated macroeconomic data, +monetary statistics, reference exchange rates and international indicators. +Every adapter shares the research-source transport, native records, durable +snapshots, verified replay and JSON CLI. [Hyperliquid](HYPERLIQUID.md) adds public +crypto market data with [weighted IP routing](IP_ROUTING.md). + +## Access and retrieval weight + +| Source | Access | Maximum retrieval unit | Rate policy | +| --- | --- | --- | --- | +| SEC EDGAR | No key; identifying organization/contact User-Agent | Company facts, exact concept, cross-company frame, complete filing history | Provider maximum 10 requests/s across clients and machines | +| FRED/ALFRED | Free personal API key | 100,000 series observations or 500,000 release observations/request | v1 120/min; v2 2/s | +| ECB SDMX | No key | Batched dimension values, selected series and history/revision deltas | No published numerical allowance found; local default 2/s | +| World Bank Indicators v2 | No key | 60 indicators/query; 32,767 records/page accepted in live probes | No published numerical allowance found; local default 5/s | + +Rates are ceilings or local policies, not guaranteed daily entitlements. +SEC counts other clients and machines too; coordinate traffic and use a shared +store. SEC, ECB and World Bank pacing shares a source namespace; FRED shares a +key namespace. All adapters honor 429/Retry-After and bounded retries. + +Default decoded-response ceilings: SEC 64 MB, FRED 128 MB, ECB 16 MB, +World Bank 32 MB. Set `max_response_bytes` if needed; narrow dates, concepts or +page sizes to reduce memory/transfer weight. A response exceeding its cap fails +explicitly. Each response is decoded, parsed and archived in memory before +delivery; page streaming bounds acquisition between requests. + +Primary references: [SEC APIs](https://www.sec.gov/search-filings/edgar-application-programming-interfaces), +[SEC fair access](https://www.sec.gov/about/developer-resources), +[FRED v2 bulk](https://fred.stlouisfed.org/docs/api/fred/v2/release_observations.html), +[FRED v2 rates](https://fred.stlouisfed.org/docs/api/fred/v2/errors.html), +[FRED v1 rates](https://fred.stlouisfed.org/docs/api/fred/errors.html), +[ECB API](https://data.ecb.europa.eu/help/api/data), +[World Bank access](https://datahelpdesk.worldbank.org/knowledgebase/articles/889392), +[World Bank paging/batches](https://datahelpdesk.worldbank.org/knowledgebase/articles/898581). + +## SEC: scope the read to the question + +Set `SEC_USER_AGENT` to your organization/application and actual contact email. +The adapter requires an explicit identity; it does not borrow a default contact. + +~~~python +from dedomena.sources import SEC +with SEC() as sec: + facts = sec.company_facts(320193) + assets = sec.company_concept(320193, "us-gaap", "Assets") + frame = sec.frame("us-gaap", "Assets", "USD", "CY2025Q4I") + for page in sec.submissions(320193): + consume(page.records, page.provenance.to_dict()) +~~~ + +`fetch` aliases company facts; `search` enumerates all filings for an exact CIK. +`fetch_many` offers bounded parallel company reads, stable caller order and +window deduplication. All SEC clients in one store share pacing across User-Agents. + +Facts, concepts and frames are one native envelope per Page record, preserving +taxonomies, units, accession numbers, fiscal/report periods and amendments. +Filing pages become rows of native columns; original company metadata and +columnar envelopes remain in snapshots. Additional history files are traversed +by default; requesting recent filings alone is explicitly incomplete when +history exists. Resume with filename `next_cursor` and `records_seen`. + +Frames select SEC's latest filing matching a calendar period. Fiscal dates can +differ; a frame is not a historical knowledge-date query. Selecting an amendment +or latest value from company facts remains a downstream research decision. +Standard taxonomy facts do not represent every custom filing concept/full document. + +For corpus scale, SEC recommends its +[nightly companyfacts/submissions ZIP archives](https://www.sec.gov/search-filings/edgar-application-programming-interfaces). +`quota()` exposes their official URLs; this adapter implements bounded JSON reads. + +## FRED: bulk acquisition and dated evidence + +Set `FRED_API_KEY` using your own free FRED account key. +[Authentication](https://fred.stlouisfed.org/docs/api/fred/v2/api_key.html) +requires a key for each user. + +~~~python +from dedomena.sources import FRED +with FRED() as fred: + metadata = fred.fetch("GDP", as_of="2020-01-01") + for page in fred.observations("GDP", as_of="2020-01-01"): + consume(page.records, page.provenance.to_dict()) + for page in fred.release_observations(53): + consume(page.records, page.provenance.to_dict()) +~~~ + +V1 search, metadata and observations pin default real-time bounds to today's +UTC date in the receipt. Use `as_of` for ALFRED knowledge-date evidence, or explicit +`realtime_start`/`realtime_end` for revision intervals. Observation dates differ +from knowledge dates. `vintages` enumerates publication/revision dates. +The [observations contract](https://fred.stlouisfed.org/docs/api/fred/series_observations.html) +defines transformations and aggregation; raw levels are the default. +Decimal strings and missing `"."` remain native. Fetch metadata for units, +frequency, seasonality, rights and source notes. + +V2 release pages contain series fragments, metadata and observations. A series +may span pages. `page_size` counts observations; `Page.records`, `records_seen` +and CLI `records` count series fragments. Usage separately tracks +`observations_delivered`. Resume v1 with offset/records_seen, v2 with cursor/records_seen. +V2 requires both checkpoint parts together; initial retrieval uses no cursor and zero +records seen. +A repeated series changing version during traversal fails explicitly; different +series can still have mixed update times. V2 serves latest release data rather +than an ALFRED vintage. The raw envelope retains release/source metadata. + +V1 mandates a query-string key. Only the HTTP send receives it; public provenance +and archived requests exclude it, and caches use hashed key namespaces. +A stable filter redacts `api_key` from HTTPX's URL response log without retaining +a credential registry. V2 uses Bearer authentication. Caller-supplied hooks and +logging remain the caller's responsibility. Echoed credentials are rejected. + +## ECB: batched dimensions and statistical meaning + +~~~python +from dedomena.sources import ECB +with ECB() as ecb: + datasets = ecb.dataflows() + metadata = ecb.series("EXR", "M..EUR.SP00.A") + rates = ecb.fx(["USD", "GBP", "JPY"], frequency="M", + start_period="2025-01", end_period="2025-12") + delta = ecb.observations("EXR", "D..EUR.SP00.A", + updated_after="2026-10-01T00:00:00Z") +~~~ + +`fx` batches ISO currencies: `OBS_VALUE` is currency units per one EUR. +General `observations(flow, key)` preserves native SDMX keys, blank wildcards +and `+` unions. CSV strings preserve precision, units, `UNIT_MULT`, status and +every returned attribute. No scaling, currency inversion or imputation occurs. +`series` returns keys/attributes without observation payloads. Date bounds retain +their native SDMX formats. Reversed bounds of the same format fail locally; mixed +formats are forwarded unchanged for the provider to interpret. + +`last_n` selects observations separately per series. `updated_after` deltas can +include additions, revisions and deletions: reconcile with prior data. +`include_history=True` requests prior versions where supported; avoid collapsing +rows solely by series and period. [Parameter semantics](https://data.ecb.europa.eu/help/api/data). + +There is no documented cursor pagination. A Page covers the selected request; +narrow explicit date windows when the response cap is exceeded. A valid future +interval returned HTTP 200 with an empty body in live probes: complete empty +result, receipt and warning. HTTP 404 stays an explicit error because invalid +series and no matching data cannot be distinguished. Missing calendar dates +do not themselves establish truncated retrieval. + +## World Bank: counted global indicators + +~~~python +from dedomena.sources import WorldBank +with WorldBank() as wb: + metadata = wb.fetch("NY.GDP.MKTP.CD") + for page in wb.search(["NY.GDP.MKTP.CD", "FP.CPI.TOTL.ZG"], + countries=["USA", "FIN"], date="1970:2024", + footnotes=True): + consume(page.records, page.provenance.to_dict()) +~~~ + +`indicators()` discovers metadata; `fetch` returns exact indicator metadata. +`search`/`observations` return values, defaulting to WDI (source 2); select other +datasets with numeric `source`. IDs, dates, units, decimals, nulls and requested +footnotes remain native. `countries="all"` includes regional/income aggregates; +those are not independent countries. The wildcard is case-insensitive and cannot +be combined with country codes. Fetch units/source notes before comparison. +Current revised indicators do not supply historical vintages. + +Count, page size and update metadata are checked across pages. Resume with numeric +page cursor and `records_seen=(page-1)*page_size`. 32,767/page succeeded live; +32,768 and 50,000 returned HTTP 400. This local cap records observed behavior, +not a published provider promise. + +## Agent CLI and replay + +~~~sh +python -m dedomena.sources fetch sec 320193 +python -m dedomena.sources search sec 320193 --max-pages 1 +python -m dedomena.sources benchmark fred 53 --operation release +python -m dedomena.sources observations fred GDP --as-of 2020-01-01 +python -m dedomena.sources catalogue ecb +python -m dedomena.sources benchmark ecb EXR/M.USD+GBP.EUR.SP00.A --last-n 12 +python -m dedomena.sources catalogue worldbank +python -m dedomena.sources benchmark worldbank 'NY.GDP.MKTP.CD;FP.CPI.TOTL.ZG' --period 1970:2024 +python -m dedomena.sources replay fred SNAPSHOT_ID --store /path/to/sources.sqlite3 +~~~ + +Benchmarks default to one page, emitting metrics/checkpoints without records. +Other collection operations emit pages and a final receipt. `--max-pages` keeps +the real completion state. SDK methods accept the reported resume checkpoints; +the CLI reports checkpoints but does not yet expose resume options. +Replay needs no provider credentials and rejects source mismatches. + +Every successful response includes public query semantics, adapter version, +HTTP status, source URL, retrieval/completion times, hashes, license context and +snapshot ID. Replay verifies saved bytes offline. Fresh queries against evolving +providers cannot guarantee identical results; deterministic evidence uses snapshots. + +## Rights and bounded verification + +Free access does not establish universal redistribution rights. +[FRED terms](https://fred.stlouisfed.org/legal/terms/), native copyright fields, +[ECB reuse](https://www.ecb.europa.eu/stats/ecb_statistics/governance_and_quality_framework/html/usage_policy.en.html), +[World Bank data terms](https://data.worldbank.org/summary-terms-of-use), +and underlying SEC filing rights govern reuse. Code is MIT; data rights differ. +Coverage is fundamentals, macro and reference rates; it is not a comprehensive +exchange-level real-time securities feed. + +Bounded probes on 2026-10-02 verified: + +- SEC: 2,257 Apple filings across both files; 25,135 company-fact observations. + All facts: 3,789,099 decoded / 271,819 wire bytes. Assets concept: 19,972 / + 2,472 bytes, 99.47% smaller decoded payload with narrower coverage. + Assets frame: 6,157 companies; USD-per-shares frame also verified. +- ECB: 36 monthly observations across three currencies, 44 series metadata + records, 104 dataflows and a complete empty future interval; raw replay verified. +- World Bank: 29,150 GDP/inflation observations for 1970–2024 in one request; + 6,472,841 decoded / 363,141 wire bytes. Cache used zero upstream bytes; raw + replay, metadata and the 1,498-indicator WDI catalog verified. +- FRED: official v1/v2 protocol fixtures pass offline. No key was configured; + authenticated live acquisition and sustained key yield remain unmeasured. + +These are selected-query probes, not daily harvest measurements. Sanitized local +metrics/snapshots live under ignored `out/finance-probes-2026-10-02/`. diff --git a/docs/HYPERLIQUID.md b/docs/HYPERLIQUID.md new file mode 100644 index 0000000..5d6841b --- /dev/null +++ b/docs/HYPERLIQUID.md @@ -0,0 +1,123 @@ +# Hyperliquid market data + +`Hyperliquid` retrieves public market data from the official +[POST `/info` endpoint](https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/info-endpoint). +No key or wallet is required. The transport allows only the supported public +query schemas; account queries, signing and `/exchange` are excluded. + +## Choose the retrieval unit + +| SDK method | Native result | Request weight | +| --- | --- | --- | +| `all_mids(dex="")` | One coin-to-price map | 2 | +| `markets(spot=False, dex="")` | One metadata/context envelope | 20 | +| `order_book(coin)`; `fetch(coin)` alias | One L2 snapshot, at most 20 levels/side | 2 | +| `candles(coin, interval, start_time, end_time)` | Available OHLCV rows | Reserve 104; settle 20 + ceil(rows/60) | +| `funding_history(coin, start_time, end_time=None, cursor=None)` | Funding rows in timestamp pages | Reserve 45; settle 20 + ceil(rows/20) | + +Metadata retains native universe, token indices, margin tables and asset +contexts. Perpetual contexts follow universe positions. Spot contexts and pair +metadata can have different lengths; both remain intact without an inferred join. +Decimals remain strings; units, nulls and additional native fields are unchanged. +Native schemas: [perpetuals](https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/info-endpoint/perpetuals), +[spot](https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/info-endpoint/spot). + +Use provider asset names: `BTC` for a perpetual, `xyz:XYZ100` for a builder DEX, +`PURR/USDC` or `@` for spot. Resolve spot indices from the native universe; +UI labels can differ from API names. `dex` selects a perpetual DEX for mids or +metadata; spot metadata does not accept it. +[Asset naming](https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/info-endpoint#perpetuals-vs-spot). + +~~~python +import time +from dedomena.sources import Hyperliquid + +end = int(time.time() * 1000) +start = end - 86_400_000 +with Hyperliquid() as api: + mids = api.all_mids() + perpetuals = api.markets() + spot = api.markets(spot=True) + book = api.order_book("BTC") + candles = api.candles("BTC", "1h", start, end) + for page in api.funding_history("BTC", start, end): + consume(page.records, page.provenance.to_dict()) +~~~ + +Times are inclusive epoch milliseconds. Candle intervals are `1m`, `3m`, `5m`, +`15m`, `30m`, `1h`, `2h`, `4h`, `8h`, `12h`, `1d`, `3d`, `1w`, `1M`. +Only the latest 5,000 candles are available. Every candle Page has +`complete=False`: short or empty results cannot prove historical coverage. +The current candle can be unfinished. [Candle contract](https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/info-endpoint#candle-snapshot). + +## Funding checkpoints and receipts + +Funding uses the provider's 500-item time-range ceiling. Each following request +starts at the preceding page's final timestamp; exact boundary-row hashes remove +inclusive repeats. A full page that cannot advance past its starting timestamp +raises `SearchLimitExceeded` before delivering that page. Malformed identities, +reordered timestamps and duplicate rows fail explicitly. +[Provider pagination](https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/info-endpoint#pagination). + +The opaque `next_cursor` contains the exact coin, start/end bounds, emitted +count and boundary hashes. Resume with the same coin/start and cursor; supply +the same end or omit it to use the cutoff stored in the checkpoint. An omitted +initial end is pinned once to retrieval time. `complete=True` finishes enumeration +of provider-available funding for that range. + +~~~python +with Hyperliquid() as api: + start = end - 30 * 86_400_000 + stream = api.funding_history("BTC", start, end) + checkpoint = next(stream) + consume(checkpoint.records, checkpoint.provenance.to_dict()) + stream.close() + if checkpoint.next_cursor is not None: + for page in api.funding_history("BTC", start, cursor=checkpoint.next_cursor): + consume(page.records, page.provenance.to_dict()) +~~~ + +Every Page carries the canonical public JSON request body, hashes, retrieval +and completion times, HTTP status and durable snapshot ID. Replay verifies the +stored response hash. Mutable snapshots default to `cache_ttl=0`; explicit cache +TTL or `refresh=True` follows the [shared source contract](SOURCES.md). + +## CLI + +~~~sh +python -m dedomena.sources markets hyperliquid +python -m dedomena.sources markets hyperliquid --spot +python -m dedomena.sources mids hyperliquid --dex xyz +python -m dedomena.sources book hyperliquid BTC +python -m dedomena.sources fetch hyperliquid BTC +python -m dedomena.sources candles hyperliquid BTC --interval 1h --start-time 1790812800000 --end-time 1790899200000 +python -m dedomena.sources funding hyperliquid BTC --start-time 1790812800000 --end-time 1790899200000 +python -m dedomena.sources quota hyperliquid +~~~ + +The CLI emits JSON Pages followed by retrieval totals and checkpoint fields. +`--max-pages` bounds funding acquisition; checkpoint continuation uses the SDK. + +## IP capacity and measured validation + +Hyperliquid publishes an aggregate REST limit of **1,200 weight/minute per IP**. +The shared limiter reserves each request's conservative maximum, then settles +valid list responses by returned item count. Failed or uncertain sends retain +the reservation; 429 cooldowns apply to the affected IP. The local 1,000 +requests/s dispatch ceiling is not a provider allowance. Other clients outside +the shared store still consume the provider's quota. +[Official rate limits](https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/rate-limits-and-user-limits). + +Use the same [generic IPPool](IP_ROUTING.md) with Hyperliquid and OpenAlex. +Ten distinct owned egress IPs have a theoretical ceiling of 6,000 weight-2 +requests/minute, subject to other traffic, latency and provider availability. +This is capacity arithmetic, not a sustained throughput measurement or daily +entitlement. + +Bounded live adapter probes on **2026-10-02**: five successful queries covered +mids, perpetual metadata, spot metadata, candles and funding. All returned HTTP +200; all five snapshots passed verified replay. Combined transfer was 399,814 +decoded bytes, with 191 weight reserved and 85 locally accounted weight. +Item-count costs are rounded up; provider billing was not reported. Candles +returned three rows with `complete=False`; funding returned 24 rows with +`complete=True`. No sustained high-volume or multi-IP benchmark was performed. diff --git a/docs/IP_ROUTING.md b/docs/IP_ROUTING.md new file mode 100644 index 0000000..c92d44f --- /dev/null +++ b/docs/IP_ROUTING.md @@ -0,0 +1,121 @@ +# Shared IP routing + +`IPPool` connects any source adapter to the same owned egress routes. Each route +pairs its **actual public IP** with a local interface address, an HTTP(S) proxy, +or an already routed `httpx.Client`. Declaring an IP identifies its allowance; +it does not configure or verify network routing. Dedomena never discovers IPs +through an external service. + +Two local addresses behind the same NAT share one public IP and one allowance. +Duplicate public IPs, including IPv4-mapped IPv6 aliases, are rejected within a +pool. Configure every worker with the same declared IP for the same egress and +use the same SQLite `Store` to coordinate across threads, processes and restarts. +Other applications using that IP still consume the provider's allowance. + +## One pool, multiple sources + +Replace all example addresses with interfaces and public IPs you own. The +`203.0.113.*` addresses below are reserved documentation examples; local addresses +must exist on the machine running Dedomena. + +~~~python +from dedomena.sources import Hyperliquid, IPPool, IPRoute, OpenAlex, Store + +store = Store("sources.sqlite3") +try: + with IPPool([ + IPRoute("203.0.113.10", local_address="10.0.0.10"), + IPRoute("203.0.113.11", local_address="10.0.0.11"), + ]) as routes: + with OpenAlex(store=store, ip_pool=routes) as papers, \ + Hyperliquid(store=store, ip_pool=routes) as markets: + paper = papers.fetch("W2741809807") + book = markets.order_book("BTC") +finally: + store.close() +~~~ + +`OPENALEX_API_KEY` supplies the paper source's key. Hyperliquid public market +reads require no key. Closing a source leaves the caller's pool and store open; +closing the pool closes only clients it created. Local bindings and proxies use +pooled connections with `trust_env=False`, so environment proxy settings cannot +replace the configured route. An injected client must already implement that +route and cannot be combined with `local_address` or `proxy`. + +For a proxy, keep its URL and credentials in the caller's environment: + +~~~python +import os + +route = IPRoute("203.0.113.10", proxy=os.environ["RESEARCH_PROXY_1"]) +# RESEARCH_PROXY_1 is an http:// or https:// proxy origin, optionally with auth. +~~~ + +## Allowances and retrieval weight + +The pool admits weighted rolling windows, shared provider/key pacing and +cooldowns atomically. It chooses a ready route in deterministic round-robin +order, or waits for the earliest eligible route within `max_wait`. Waiting does +not reserve a future dispatch slot. A positive `max_wait` bounds admission, +including budget work and lock wait; zero disables scheduled quota sleeps. A rolling window's duration remains fixed +for its quota scope so changing a worker's configuration cannot hide prior use. + +- **Hyperliquid:** 1,200 aggregate REST weight per minute per public IP. + Mid-price and order-book requests cost 2; market metadata costs 20. + Ten distinct IPs therefore have a theoretical combined ceiling of 6,000 + cheap requests/minute, versus 600 for one IP. This is allowance arithmetic, + subject to RTT, local pacing and other traffic; it is not a throughput benchmark. + [Official rate policy](https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/rate-limits-and-user-limits). +- **OpenAlex:** Dedomena retains the 100 requests/second ceiling across the key, + plus a conservative IP guard and the shared daily credit budget. The current + help page does not establish an IP-only scope for that rate, so adding IPs + does not increase the key's request or credit allowance. + [Official authentication and limits](https://help.openalex.org/api/authentication/). + +Response-sized Hyperliquid reads reserve their maximum estimated weight before +sending, then reduce only that request's reservation after a verified response +establishes its returned item count. Uncertain sends and failed responses retain +the reservation. Budget rejection before a send consumes no IP allowance. +A 429 persists a cooldown on the affected provider/IP; shared provider or key +exhaustion still applies across the whole pool. Provider errors and retries never +supply extra allowances. + +Without explicit routes, adapters with IP limits use the conservative unknown-IP +scope `0.0.0.0`. All such instances of that provider share its IP guard in the +same store. No extra IP allowance is inferred from separate clients or keys. + +For another adapter, pass `ip_pool=routes` and `ip_limit=(capacity, seconds)` through +its transport configuration. The provider's own key, daily, service and global +limits remain in effect. `ip_throttle_only=True` is appropriate only when that +provider's 429 is known to describe an IP allowance; otherwise the transport also +retains its shared cooldown. A pool with `ip_limit=None` provides routing and IP +cooldowns while keeping the adapter's existing shared pacing. + +Request hashes and cache identity exclude routing, so a query can reuse the same +snapshot across IPs. Network receipts carry optional `egress_ip_sha256` for the +chosen declared public IP; unknown-IP and older receipts leave it unset. Public +provenance, pool representations and admission state exclude proxy credentials. + +## CLI route file + +Save the same routes as a JSON array in `ip-routes.json`: + +~~~json +[ + {"public_ip": "203.0.113.10", "local_address": "10.0.0.10"}, + {"public_ip": "203.0.113.11", "local_address": "10.0.0.11"} +] +~~~ + +~~~sh +python -m dedomena.sources fetch hyperliquid BTC --store sources.sqlite3 --ip-routes ip-routes.json +python -m dedomena.sources fetch openalex W2741809807 --store sources.sqlite3 --ip-routes ip-routes.json +~~~ + +Each JSON route accepts `public_ip` and exactly one of `local_address` or `proxy`; +Python client objects cannot appear in the file. Use the same route identities +and store path across CLI workers. + +Routing admission does not schedule parallel work. A generic bounded executor, +with stable ordering, shared admission and durable checkpoints, is tracked in +[issue #18](https://github.com/autonomio/dedomena/issues/18). diff --git a/setup.py b/setup.py index 5d2399e..3e5f0dc 100755 --- a/setup.py +++ b/setup.py @@ -4,8 +4,10 @@ DESCRIPTION = "Reproducible research data sources for agents and scientists" LONG_DESCRIPTION = """\ -Dedomena provides compact, high-throughput access to OpenAlex, Europe PMC -and EPO patent data, with consistent provenance, durable caching and replay. +Dedomena provides compact, high-throughput access to research and finance data: +OpenAlex, Europe PMC, EPO, SEC EDGAR, FRED/ALFRED, ECB, World Bank and Hyperliquid. +Every source shares consistent provenance, durable caching and replay. +Reusable IP pools coordinate weighted egress limits across sources. Legacy dataset and API functions remain available through the legacy extra. """ @@ -15,7 +17,7 @@ URL = 'http://autonom.io' LICENSE = 'MIT' DOWNLOAD_URL = 'https://github.com/autonomio/dedomena/' -VERSION = '0.4.0' +VERSION = '0.6.0' try: from setuptools import setup diff --git a/tests/test_egress.py b/tests/test_egress.py new file mode 100644 index 0000000..17148e4 --- /dev/null +++ b/tests/test_egress.py @@ -0,0 +1,362 @@ +from concurrent.futures import ThreadPoolExecutor +import math + +import httpx +import pytest + +from dedomena.sources.core import BudgetExceeded, Store, Throttled +from dedomena.sources.egress import IPLease, IPPool, IPRoute + + +class Clock: + def __init__(self): + self.now = 1000.0 + self.waits = [] + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.waits.append(seconds) + self.now += seconds + + +def pool(*ips): + return IPPool(IPRoute(ip, client=httpx.Client(transport=httpx.MockTransport( + lambda _: httpx.Response(200)))) for ip in ips) + + +def acquire(routes, store, **kwargs): + return routes.acquire(store, "provider", 1, 2, 60, max_wait=0, **kwargs) + + +def test_canonical_nat_aliases_do_not_multiply_quota(): + with pool("2001:db8::1") as routes: + assert routes.routes[0].public_ip == "2001:db8::1" + client = httpx.Client(transport=httpx.MockTransport(lambda _: httpx.Response(200))) + assert IPRoute("::ffff:192.0.2.1", client=client).public_ip == "192.0.2.1" + with pytest.raises(ValueError, match="distinct"): + IPPool([IPRoute("192.0.2.1", client=client), IPRoute("::ffff:c000:201", client=client)]) + with pytest.raises(ValueError, match="distinct"): + IPPool([IPRoute("2001:db8::1", client=client), IPRoute("2001:0DB8:0:0:0:0:0:1", client=client)]) + client.close() + + +@pytest.mark.parametrize("ip", ["host.example", "", "192.0.2.999", "fe80::1%en0", None]) +def test_invalid_ip_is_not_disclosed_in_error(ip): + with pytest.raises(ValueError, match="IPv4 or IPv6"): + IPRoute(ip, local_address="127.0.0.1") + + +def test_routing_is_real_and_cannot_be_overridden_by_injected_client(monkeypatch): + clients, transports = [], [] + original_client, original_transport = httpx.Client, httpx.HTTPTransport + + def transport(**kwargs): + transports.append(kwargs) + return original_transport(**kwargs) + + # IPRoute validates injected clients against the real class. + route = IPRoute("192.0.2.1", local_address="127.0.0.1") + monkeypatch.setattr(httpx, "HTTPTransport", transport) + # The constructor creates a bound client, then validates its immutable route. + # Restore the class while constructing the post-bind route. + def make_client(**kwargs): + clients.append(kwargs) + result = original_client(**kwargs) + monkeypatch.setattr(httpx, "Client", original_client) + return result + monkeypatch.setattr(httpx, "Client", make_client) + routes = IPPool([route]) + owned = routes.routes[0].client + assert transports[0]["local_address"] == "127.0.0.1" + assert transports[0]["trust_env"] is False + assert transports[0]["limits"].max_keepalive_connections == 100 + assert clients[0]["trust_env"] is False + assert clients[0]["follow_redirects"] is False + routes.close() + assert owned.is_closed + with pytest.raises(ValueError, match="exactly one"): + IPRoute("192.0.2.1", local_address="127.0.0.1", client=original_client()) + with pytest.raises(ValueError, match="exactly one"): + IPRoute("192.0.2.1") + + +def test_proxy_construction_and_secret_free_representations(monkeypatch): + calls = [] + original_client = httpx.Client + secret = "proxy-password-token" + route = IPRoute("192.0.2.1", proxy=f"http://user:{secret}@proxy.example:8080") + + def client(**kwargs): + calls.append(kwargs) + result = original_client(**kwargs) + monkeypatch.setattr(httpx, "Client", original_client) + return result + + monkeypatch.setattr(httpx, "Client", client) + with IPPool([route]) as routes: + assert calls[0]["proxy"] == route.proxy + assert calls[0]["trust_env"] is False + store = Store() + lease = acquire(routes, store) + assert secret not in repr(route) + repr(routes.routes[0]) + repr(lease) + with store.transaction() as db: + assert secret not in "\n".join(db.iterdump()) + with pytest.raises(ValueError) as error: + IPRoute("192.0.2.1", proxy=f"file://user:{secret}@host") + assert secret not in str(error.value) + + +def test_pool_does_not_close_injected_clients(): + routes = pool("192.0.2.1") + client = routes.routes[0].client + routes.close() + routes.close() + assert not client.is_closed + with pytest.raises(ValueError, match="closed"): + acquire(routes, Store()) + client.close() + + +def test_round_robin_is_deterministic_and_shared_across_pool_instances(): + store = Store() + with pool("192.0.2.1", "192.0.2.2") as first, pool("192.0.2.1", "192.0.2.2") as second: + assert [acquire(routes, store).public_ip for routes in [first, second, first, second]] == [ + "192.0.2.1", "192.0.2.2", "192.0.2.1", "192.0.2.2"] + with pytest.raises(Throttled): + acquire(first, store) + + +def test_ip_consumption_shared_across_restarts_and_subsets(tmp_path): + path = tmp_path / "shared.sqlite3" + clock = Clock() + with pool("192.0.2.1", "192.0.2.2") as first: + store = Store(path, clock=clock) + first.acquire(store, "provider", 2, 2, 60, max_wait=0) + store.close() + with pool("192.0.2.1") as restarted: + store = Store(path, clock=clock) + with pytest.raises(Throttled): + acquire(restarted, store) + # The same IP's unrelated provider remains independent. + assert restarted.acquire(store, "other", 2, 2, 60, max_wait=0).public_ip == "192.0.2.1" + store.close() + + +def test_weighted_rolling_window_waits_for_exact_capacity(): + clock = Clock() + store = Store(clock=clock) + with pool("192.0.2.1") as routes: + routes.acquire(store, "provider", 2, 3, 10, clock.sleep, 0) + clock.now += 3 + routes.acquire(store, "provider", 1, 3, 10, clock.sleep, 0) + lease = routes.acquire(store, "provider", 2, 3, 10, clock.sleep, 7) + assert lease.public_ip == "192.0.2.1" + assert clock.waits == [7] + with pytest.raises(Throttled) as error: + routes.acquire(store, "provider", 2, 3, 10, clock.sleep, 0) + assert error.value.retry_after == 10 + + +def test_waiters_charge_no_future_slot_and_recheck_cooldown(): + clock = Clock() + store = Store(clock=clock) + with pool("192.0.2.1", "192.0.2.2") as routes: + first = routes.acquire(store, "provider", 1, 1, 10, max_wait=0) + second = routes.acquire(store, "provider", 1, 1, 10, max_wait=0) + + def sleep(seconds): + clock.sleep(seconds) + if len(clock.waits) == 1: + routes.defer(store, "provider", first, 15) + # Another worker acquires the newly available route before this waiter. + assert routes.acquire(store, "provider", 1, 1, 10, max_wait=0).public_ip == second.public_ip + + admitted = routes.acquire(store, "provider", 1, 1, 10, sleep, 20) + assert admitted.public_ip == second.public_ip + assert clock.waits == [10, 10] + + +def test_one_ip_cooldown_does_not_poison_other_routes_or_providers(tmp_path): + clock = Clock() + first = Store(tmp_path / "shared.sqlite3", clock=clock) + second = Store(tmp_path / "shared.sqlite3", clock=clock) + with pool("192.0.2.1", "192.0.2.2") as routes, pool("192.0.2.1") as subset: + lease = acquire(routes, first) + routes.defer(first, "provider", lease, 25) + routes.defer(first, "provider", lease, 10) # cannot shorten an existing cooldown + assert acquire(routes, second).public_ip == "192.0.2.2" + with pytest.raises(Throttled) as error: + acquire(subset, second) + assert error.value.retry_after == 25 + assert subset.acquire(second, "other", 1, 2, 60, max_wait=0).public_ip == "192.0.2.1" + first.close() + second.close() + + +def test_shared_provider_limit_cannot_be_multiplied_by_more_ips(): + clock = Clock() + store = Store(clock=clock) + limits = (("provider:key:hashed", 1, 2, 60),) + with pool("192.0.2.1", "192.0.2.2", "192.0.2.3") as routes: + acquire(routes, store, limits=limits) + acquire(routes, store, limits=limits) + with pytest.raises(Throttled): + acquire(routes, store, limits=limits) + with store.transaction() as db: + assert db.execute("SELECT sum(weight) FROM ip_events WHERE scope=?", (limits[0][0],)).fetchone()[0] == 2 + assert db.execute("SELECT sum(weight) FROM ip_events WHERE scope LIKE 'provider:ip:%'").fetchone()[0] == 2 + + +def test_existing_core_pacing_is_atomic_with_ip_admission(): + clock = Clock() + store = Store(clock=clock) + pacing = (("provider:key", 2),) + with pool("192.0.2.1", "192.0.2.2") as routes: + store.pace("provider:key", 2, clock.sleep, 0) + with pytest.raises(Throttled) as error: + acquire(routes, store, pacing=pacing) + assert error.value.retry_after == 0.5 + with store.transaction() as db: + exists = db.execute("SELECT 1 FROM sqlite_master WHERE name='ip_events'").fetchone() + assert not exists or db.execute("SELECT count(*) FROM ip_events").fetchone()[0] == 0 + lease = routes.acquire(store, "provider", 1, 2, 60, clock.sleep, 1, pacing=pacing) + assert clock.waits == [0.5] + assert lease.public_ip == "192.0.2.1" + with pytest.raises(Throttled): + store.pace("provider:key", 2, clock.sleep, 0) + + +def test_core_throttle_and_global_cooldown_survive_route_selection(): + clock = Clock() + store = Store(clock=clock) + store.throttle("provider:key", 1) + store.defer("provider:key", 3) + with pool("192.0.2.1", "192.0.2.2") as routes: + routes.acquire(store, "provider", 1, 10, 60, clock.sleep, 3, pacing=(("provider:key", 10),)) + assert clock.waits == [3] + with pytest.raises(Throttled) as error: + routes.acquire(store, "provider", 1, 10, 60, clock.sleep, 0, pacing=(("provider:key", 10),)) + assert error.value.retry_after == 1 + + +def test_route_only_mode_still_observes_cooldown_and_pacing(): + clock = Clock() + store = Store(clock=clock) + with pool("192.0.2.1", "192.0.2.2") as routes: + lease = routes.acquire(store, "provider", 1, None, 60, max_wait=0) + assert lease.reservation_id is None + routes.defer(store, "provider", lease, 10) + assert routes.acquire(store, "provider", 1, None, 60, max_wait=0).public_ip == "192.0.2.2" + with store.transaction() as db: + assert db.execute("SELECT count(*) FROM ip_events").fetchone()[0] == 0 + + +def test_settlement_releases_only_this_lease_and_never_reinserts_expired_events(): + clock = Clock() + store = Store(clock=clock) + with pool("192.0.2.1") as routes: + first = routes.acquire(store, "provider", 4, 8, 10, max_wait=0) + second = routes.acquire(store, "provider", 4, 8, 10, max_wait=0) + routes.settle(store, second, 1) + assert routes.acquire(store, "provider", 3, 8, 10, max_wait=0).public_ip == first.public_ip + with pytest.raises(Throttled): + routes.acquire(store, "provider", 1, 8, 10, max_wait=0) + routes.settle(store, second, 1) # same-cost settlement is idempotent + with pytest.raises(ValueError): + routes.settle(store, second, 0) + with pytest.raises(ValueError): + routes.settle(store, second, 4) + with pytest.raises(ValueError): + routes.settle(store, first, 5) + clock.now += 10 + routes.acquire(store, "provider", 1, 8, 10, max_wait=0) + routes.settle(store, first, 0) + with store.transaction() as db: + assert db.execute("SELECT sum(weight) FROM ip_events").fetchone()[0] == 1 + + +def test_oversleep_cannot_dispatch_after_deadline(): + clock = Clock() + store = Store(clock=clock) + with pool("192.0.2.1") as routes: + routes.acquire(store, "provider", 1, 1, 10, max_wait=0) + with pytest.raises(Throttled): + routes.acquire(store, "provider", 1, 1, 10, lambda seconds: clock.sleep(seconds + 1), 10) + with store.transaction() as db: + assert db.execute("SELECT count(*) FROM ip_events").fetchone()[0] == 1 + + +def test_atomic_admission_across_concurrent_stores(tmp_path): + path = tmp_path / "shared.sqlite3" + clock = Clock() + stores = [Store(path, clock=clock) for _ in range(8)] + with pool("192.0.2.1", "192.0.2.2") as routes: + def request(index): + try: + return routes.acquire(stores[index % len(stores)], "provider", 1, 3, 60, max_wait=0) + except Throttled: + return None + with ThreadPoolExecutor(max_workers=8) as threads: + admitted = list(threads.map(request, range(40))) + assert sum(isinstance(lease, IPLease) for lease in admitted) == 6 + assert sum(lease.public_ip == "192.0.2.1" for lease in admitted if lease) == 3 + assert sum(lease.public_ip == "192.0.2.2" for lease in admitted if lease) == 3 + for store in stores: + store.close() + + +@pytest.mark.parametrize("kwargs", [ + {"weight": True}, {"weight": 0}, {"weight": -1}, {"capacity": 0}, + {"period": 0}, {"period": math.inf}, {"max_wait": -1}, {"max_wait": math.nan}, +]) +def test_invalid_admission_configuration_has_no_side_effects(kwargs): + defaults = dict(weight=1, capacity=2, period=60, max_wait=0) + defaults.update(kwargs) + store = Store() + with pool("192.0.2.1") as routes: + with pytest.raises(ValueError): + routes.acquire(store, "provider", **defaults) + with store.transaction() as db: + assert db.execute("SELECT count(*) FROM pacing").fetchone()[0] == 0 + + +def test_excessive_weight_fails_without_wait_or_charge(): + with pool("192.0.2.1") as routes: + with pytest.raises(BudgetExceeded): + routes.acquire(Store(), "provider", 3, 2, 60, lambda _: pytest.fail("must not sleep"), 60) + + +def test_window_cannot_be_shortened_after_history_has_been_pruned(): + store = Store() + with pool("192.0.2.1") as routes: + acquire(routes, store) + with pytest.raises(ValueError, match="period"): + routes.acquire(store, "provider", 1, 2, 10, max_wait=0) + + +def test_budget_callback_runs_only_on_final_admission_and_rolls_back_failures(): + clock = Clock() + store = Store(clock=clock) + callbacks = [] + with pool("192.0.2.1") as routes: + routes.acquire(store, "provider", 1, 1, 10, max_wait=0) + + def deny(db, now): + callbacks.append(now) + db.execute("INSERT INTO budgets VALUES('provider:key','credits','day',1)") + raise BudgetExceeded("provider: day allowance exhausted") + + with pytest.raises(BudgetExceeded): + routes.acquire(store, "provider", 1, 1, 10, clock.sleep, 10, + pacing=(("provider:key", 2),), before_admit=deny) + assert callbacks == [1010] + assert clock.waits == [10] + with store.transaction() as db: + assert db.execute("SELECT count(*) FROM budgets").fetchone()[0] == 0 + assert db.execute("SELECT count(*) FROM pacing").fetchone()[0] == 0 + lease = routes.acquire(store, "provider", 1, 1, 10, max_wait=0, + before_admit=lambda db, now: callbacks.append(now)) + assert lease.public_ip == "192.0.2.1" + assert callbacks == [1010, 1010] diff --git a/tests/test_finance_cli.py b/tests/test_finance_cli.py new file mode 100644 index 0000000..c787dad --- /dev/null +++ b/tests/test_finance_cli.py @@ -0,0 +1,80 @@ +import json + +import httpx + +from dedomena.sources import Store +from dedomena.sources.core import Transport +from dedomena.sources import __main__ as cli + + +def test_missing_finance_credentials_are_structured(monkeypatch, capsys): + monkeypatch.delenv("FRED_API_KEY", raising=False) + assert cli.main(["observations", "fred", "GDP"]) == 2 + output = capsys.readouterr() + assert not output.out + error = json.loads(output.err) + assert error["error"] == "ValueError" and not error["complete"] + assert "Traceback" not in output.err + + +def test_fred_replay_does_not_require_current_credentials(tmp_path, monkeypatch, capsys): + monkeypatch.delenv("FRED_API_KEY", raising=False) + path = tmp_path / "sources.sqlite3" + store = Store(path) + with Transport("fred", "https://api.stlouisfed.org/fred", credential="a" * 32, + store=store, client=httpx.Client(transport=httpx.MockTransport( + lambda _: httpx.Response(200, json={"observations": []})))) as source: + response = source._request("GET", "/series/observations", + params={"series_id": "GDP"}, + private_params={"api_key": "a" * 32}) + store.close() + assert cli.main(["replay", "fred", response.provenance.snapshot_id, + "--store", str(path)]) == 0 + receipt = json.loads(capsys.readouterr().out) + assert receipt["provenance"]["source"] == "fred" + assert json.loads(receipt["body"]) == {"observations": []} + assert cli.main(["replay", "sec", response.provenance.snapshot_id, + "--store", str(path)]) == 2 + assert not json.loads(capsys.readouterr().err)["complete"] + + +def test_ecb_observation_flags_and_benchmark_receipt(monkeypatch, capsys): + from dedomena.sources import ECB + clock = [0.0] + monkeypatch.setattr(cli.time, "perf_counter", lambda: clock[0]) + def constructor(**kwargs): + def handler(request): + clock[0] += 2 + assert request.url.params["lastNObservations"] == "2" + assert request.url.params["startPeriod"] == "2025-01" + assert request.url.params["endPeriod"] == "2025-03" + return httpx.Response(200, text=( + "KEY,TIME_PERIOD,OBS_VALUE,UNIT,UNIT_MULT\n" + "EXR.M.USD.EUR.SP00.A,2025-03,1.08,USD,0\n")) + return ECB(store=Store(), client=httpx.Client(transport=httpx.MockTransport(handler))) + monkeypatch.setattr(cli, "ECB", constructor) + assert cli.main(["benchmark", "ecb", "EXR/M.USD.EUR.SP00.A", + "--operation", "observations", "--last-n", "2", + "--start-period", "2025-01", "--end-period", "2025-03"]) == 0 + receipt = json.loads(capsys.readouterr().out) + assert receipt["elapsed_seconds"] == 2.0 + assert receipt["operation"] == "observations" + assert receipt["complete"] and receipt["records"] == 1 + assert "records_seen" in receipt + assert "records" not in receipt.get("provenance", {}) + + +def test_release_benchmark_uses_native_bulk_endpoint(monkeypatch, capsys): + from dedomena.sources import FRED + def constructor(**kwargs): + def handler(request): + assert request.url.path == "/fred/v2/release/observations" + assert request.url.params["release_id"] == "53" + return httpx.Response(200, json={ + "release": {"release_id": 53}, "series": [], "has_more": False}) + return FRED("a" * 32, store=Store(), + client=httpx.Client(transport=httpx.MockTransport(handler))) + monkeypatch.setattr(cli, "FRED", constructor) + assert cli.main(["benchmark", "fred", "53", "--operation", "release"]) == 0 + receipt = json.loads(capsys.readouterr().out) + assert receipt["operation"] == "release" and receipt["complete"] diff --git a/tests/test_finance_ecb.py b/tests/test_finance_ecb.py new file mode 100644 index 0000000..764b436 --- /dev/null +++ b/tests/test_finance_ecb.py @@ -0,0 +1,233 @@ +import json + +import httpx +import pytest + +from dedomena.sources.core import InvalidResponse, Store, Transport +from dedomena.sources.ecb import ECB + + +CSV = ("KEY,FREQ,CURRENCY,CURRENCY_DENOM,TIME_PERIOD,OBS_VALUE,UNIT,UNIT_MULT,OBS_STATUS\r\n" + "EXR.M.USD.EUR.SP00.A,M,USD,EUR,2025-01,1.0353727272727,USD,0,A\r\n" + "EXR.M.GBP.EUR.SP00.A,M,GBP,EUR,2025-01,0.8390809090909,GBP,0,A\r\n") + + +def source(handler): + return ECB(client=httpx.Client(transport=httpx.MockTransport(handler)), + store=Store(), sleep=lambda _: None) + + +def test_fx_batch_preserves_precision_units_and_query_receipt(): + calls = [] + def handler(request): + calls.append(request) + assert request.url.path == "/service/data/EXR/M.USD+GBP.EUR.SP00.A" + assert request.url.params["startPeriod"] == "2025-01" + assert request.url.params["endPeriod"] == "2025-03" + assert request.url.params["detail"] == "full" + return httpx.Response(200, text=CSV, headers={"ETag": '"edition-1"'}) + with source(handler) as ecb: + page = ecb.fx(["USD", "GBP", "USD"], frequency="M", + start_period="2025-01", end_period="2025-03") + assert page.complete and page.total == 2 + assert page.records[0]["OBS_VALUE"] == "1.0353727272727" + assert page.records[0]["UNIT_MULT"] == "0" + assert ecb.store.replay(page.provenance.snapshot_id).body == CSV.encode() + cached = ecb.fx(["USD", "GBP"], frequency="M", + start_period="2025-01", end_period="2025-03") + assert cached.cache_hit and len(calls) == 1 + assert ecb.store.replay(page.provenance.snapshot_id).headers["etag"] == '"edition-1"' + + +def test_revision_delta_and_history_are_explicit(): + def handler(request): + assert request.url.params["includeHistory"] == "true" + assert request.url.params["updatedAfter"] == "2025-01-01T00:00:00Z" + assert request.url.params["lastNObservations"] == "2" + return httpx.Response(200, text=CSV) + with source(handler) as ecb: + page = ecb.observations("EXR", "M..EUR.SP00.A", include_history=True, + updated_after="2025-01-01T00:00:00Z", last_n=2) + assert len(page.warnings) == 4 + + +def test_discovery_retains_series_attributes_without_values(): + def handler(request): + assert request.url.params["detail"] == "nodata" + return httpx.Response(200, text="KEY,UNIT,UNIT_MULT\r\nEXR.M.USD.EUR.SP00.A,USD,0\r\n") + with source(handler) as ecb: + page = ecb.series("EXR", "M..EUR.SP00.A") + assert page.records[0]["UNIT"] == "USD" + assert "OBS_VALUE" not in page.records[0] + + +@pytest.mark.parametrize("body", [ + "KEY,TIME_PERIOD,OBS_VALUE\nEXR.M.USD.EUR.SP00.A,2025-01\n", + "KEY,TIME_PERIOD,OBS_VALUE\nEXR.M.USD.EUR.SP00.A,,1\n", + "KEY,TIME_PERIOD,OBS_VALUE\nEXR.M.USD.EUR.SP00.A,2025-01,1,extra\n", + "KEY,KEY,TIME_PERIOD,OBS_VALUE\nEXR.M.USD.EUR.SP00.A,EXR.M.USD.EUR.SP00.A,2025-01,1\n", + '{"error":"unexpected format"}', +], ids=["missing_value", "empty_period", "extra_column", "duplicate_header", "wrong_format"]) +def test_malformed_observations_fail_with_matching_series_identity(body): + with source(lambda _: httpx.Response(200, text=body)) as ecb: + with pytest.raises(InvalidResponse): + ecb.fetch("EXR/M.USD.EUR.SP00.A") + + +def test_foreign_dataset_fails_when_no_dimension_filter_can_mask_it(): + body = "KEY,TIME_PERIOD,OBS_VALUE\nOTHER.M.USD.EUR.SP00.A,2025-01,1\n" + with source(lambda _: httpx.Response(200, text=body)) as ecb: + with pytest.raises(InvalidResponse): + ecb.observations("EXR") + + +def test_valid_observation_control_for_malformed_csv_cases(): + body = "KEY,TIME_PERIOD,OBS_VALUE\nEXR.M.USD.EUR.SP00.A,2025-01,1\n" + with source(lambda _: httpx.Response(200, text=body)) as ecb: + page = ecb.fetch("EXR/M.USD.EUR.SP00.A") + assert page.complete and page.records[0]["OBS_VALUE"] == "1" + + +@pytest.mark.parametrize("kwargs", [ + {"flow": "../users", "key": ""}, + {"flow": "EXR", "key": ".."}, + {"flow": "EXR", "key": "D.USD/EUR"}, + {"flow": "EXR", "key": "", "updated_after": "2025-01-01"}, + {"flow": "EXR", "key": "", "start_period": "2025-02-30"}, + {"flow": "EXR", "key": "", "start_period": "2025-03", "end_period": "2025-01"}, + {"flow": "EXR", "key": "", "include_history": "false"}, +]) +def test_invalid_queries_fail_before_network(kwargs): + with source(lambda _: pytest.fail("unexpected network")) as ecb: + with pytest.raises(ValueError): + ecb.observations(**kwargs) + + +def test_metadata_xml_keeps_source_ids_names_and_raw_snapshot(): + xml = ('' + '' + 'Exchange rates' + '') + with source(lambda _: httpx.Response(200, text=xml)) as ecb: + page = ecb.dataflows() + assert page.records[0]["id"] == "EXR" + assert page.records[0]["names"][0]["text"] == "Exchange rates" + assert ecb.store.replay(page.provenance.snapshot_id).body == xml.encode() + + +def test_fred_query_key_is_sent_but_never_archived(): + secret = "a" * 32 + def handler(request): + assert request.url.params["api_key"] == secret + return httpx.Response(200, json={"observations": []}) + with Transport("fred", "https://api.stlouisfed.org/fred", credential=secret, + client=httpx.Client(transport=httpx.MockTransport(handler)), + store=Store(), sleep=lambda _: None) as fred: + response = fred._request("GET", "/series/observations", params={"series_id": "GDP"}, + private_params={"api_key": secret}) + assert secret not in json.dumps(response.provenance.to_dict()) + assert "api_key" not in response.provenance.parameters + assert secret.encode() not in fred.store.replay(response.provenance.snapshot_id).body + with pytest.raises(ValueError): + fred._request("GET", "/series/observations", private_params={"api_key": "b" * 32}) + + +def test_query_credentials_are_not_a_general_transport_escape_hatch(): + with source(lambda _: pytest.fail("unexpected network")) as ecb: + with pytest.raises(ValueError): + ecb._request("GET", "/data/EXR", private_params={"api_key": ""}) + with pytest.raises(ValueError): + ecb._request("GET", "/data/EXR/../../users") + + +def test_empty_ecb_interval_has_a_receipt_and_is_complete(): + with source(lambda _: httpx.Response(200, content=b"")) as ecb: + page = ecb.fx("USD", start_period="2099-01-01", end_period="2099-01-05") + assert page.complete and page.total == 0 and page.warnings + assert page.provenance.http_status == 200 + assert ecb.store.replay(page.provenance.snapshot_id).body == b"" + + +def test_exact_series_query_rejects_another_currency(): + with source(lambda _: httpx.Response(200, text=CSV)) as ecb: + with pytest.raises(InvalidResponse): + ecb.fx("EUR", frequency="M") + + +def test_fred_private_key_is_redacted_from_httpx_info_logs(caplog): + import logging + secret = "c" * 32 + caplog.set_level(logging.INFO, logger="httpx") + with Transport("fred", "https://api.stlouisfed.org/fred", credential=secret, + store=Store(), client=httpx.Client(transport=httpx.MockTransport( + lambda _: httpx.Response(200, json={"observations": []})))) as fred: + fred._request("GET", "/series/observations", + params={"series_id": "GDP"}, private_params={"api_key": secret}) + assert "[REDACTED]" in caplog.text + assert secret not in caplog.text + + +def test_fred_logging_redaction_survives_parallel_keys(caplog): + from concurrent.futures import ThreadPoolExecutor + import logging + caplog.set_level(logging.INFO, logger="httpx") + keys = [str(index).zfill(32) for index in range(20)] + def request(secret): + with Transport("fred", "https://api.stlouisfed.org/fred", credential=secret, + store=Store(), client=httpx.Client(transport=httpx.MockTransport( + lambda _: httpx.Response(200, json={"observations": []})))) as fred: + fred._request("GET", "/series/observations", private_params={"api_key": secret}) + with ThreadPoolExecutor(max_workers=10) as workers: + list(workers.map(request, keys)) + assert all(key not in caplog.text for key in keys) + assert caplog.text.count("[REDACTED]") == len(keys) + + +@pytest.mark.parametrize("secret", ["s" + "1" * 31, "1" * 15 + "s" + "1" * 16]) +def test_httpx_redaction_replaces_entire_parameter_and_keeps_url_shape(secret): + import logging + from dedomena.sources.core import _PrivateQueryLogFilter + record = logging.LogRecord("httpx", logging.INFO, "", 0, + 'HTTP Request: GET %s "HTTP/1.1 200 OK"', + ("https://api.stlouisfed.org/fred/series?api_key=" + + secret + "&series_id=GDP",), None) + assert _PrivateQueryLogFilter().filter(record) + assert record.getMessage() == ( + 'HTTP Request: GET https://api.stlouisfed.org/fred/series?api_key=' + '[REDACTED]&series_id=GDP "HTTP/1.1 200 OK"') + + +@pytest.mark.parametrize("start,end", [ + ("2025-Q1", "2025-12"), + ("2025-S2", "2025-Q4"), + ("2025-W01", "2025-01"), + ("2025-01", "2025-Q4"), + ("2025-01-01", "2025-12"), +]) +def test_mixed_period_formats_are_forwarded_verbatim(start, end): + def handler(request): + assert request.url.params["startPeriod"] == start + assert request.url.params["endPeriod"] == end + return httpx.Response(200, text=( + "KEY,TIME_PERIOD,OBS_VALUE\nEXR.M.USD.EUR.SP00.A,2025-12,1\n")) + with source(handler) as ecb: + page = ecb.observations("EXR", "M.USD.EUR.SP00.A", + start_period=start, end_period=end) + assert page.complete + assert page.provenance.parameters["startPeriod"] == start + assert page.provenance.parameters["endPeriod"] == end + + +@pytest.mark.parametrize("start,end", [ + ("2025", "2024"), + ("2025-S2", "2025-S1"), + ("2025-Q4", "2025-Q1"), + ("2025-12", "2025-01"), + ("2025-W52", "2025-W01"), + ("2025-12-01", "2025-01-01"), +]) +def test_reversed_periods_of_the_same_format_fail_before_network(start, end): + with source(lambda _: pytest.fail("unexpected network")) as ecb: + with pytest.raises(ValueError, match="start_period"): + ecb.observations("EXR", "M.USD.EUR.SP00.A", + start_period=start, end_period=end) diff --git a/tests/test_finance_macro.py b/tests/test_finance_macro.py new file mode 100644 index 0000000..cf14178 --- /dev/null +++ b/tests/test_finance_macro.py @@ -0,0 +1,301 @@ +"""Offline macro contracts: dated knowledge, decimals, identities and bulk limits.""" +from datetime import datetime, timezone +import json + +import httpx +import pytest + +from dedomena.sources.core import InvalidResponse, Store +from dedomena.sources.fred import FRED +from dedomena.sources.worldbank import WorldBank + + +class Clock: + def __init__(self): + self.now = datetime(2026, 10, 2, tzinfo=timezone.utc).timestamp() + self.waits = [] + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.waits.append(seconds) + self.now += seconds + + +@pytest.fixture +def clock(): + return Clock() + + +@pytest.fixture +def store(clock): + with_store = Store(clock=clock) + yield with_store + with_store.close() + + +def fred(handler, store, clock, **kwargs): + return FRED("0123456789abcdef0123456789abcdef", store=store, sleep=clock.sleep, + client=httpx.Client(transport=httpx.MockTransport(handler)), **kwargs) + + +def wb(handler, store, clock, **kwargs): + return WorldBank(store=store, sleep=clock.sleep, + client=httpx.Client(transport=httpx.MockTransport(handler)), **kwargs) + + +def v1(request, rows, key="observations", total=None): + return httpx.Response(200, json={"count": len(rows) if total is None else total, + "offset": int(request.url.params.get("offset", 0)), + "realtime_start": request.url.params.get("realtime_start"), + "realtime_end": request.url.params.get("realtime_end"), key: rows}) + + +def series(ident="GDP", value="100.010000000001", date="2024-01-01", updated="2026-09-01T00:00:00Z"): + return {"series_id": ident, "title": "Gross Domestic Product", "units": "Billions of Dollars", + "frequency": "Quarterly", "last_updated": updated, + "copyright_id": "public domain: citation requested", + "observations": [{"date": date, "value": value}]} + + +def release(rows, *, more=False, cursor=None, release_id=53): + body = {"release": {"release_id": release_id, "name": "GDP", "sources": []}, + "series": rows, "has_more": more} + if cursor is not None: + body["next_cursor"] = cursor + return httpx.Response(200, json=body) + + +def observation(indicator="NY.GDP.MKTP.CD", country="US", iso3="USA", value=1.5, date="2024"): + return {"indicator": {"id": indicator, "value": "GDP"}, + "country": {"id": country, "value": "United States"}, + "countryiso3code": iso3, "date": date, "value": value, + "unit": "", "obs_status": "", "decimal": 0} + + +def worldbank(request, rows, *, total=None, pages=None, updated="2026-07-13"): + total = len(rows) if total is None else total + size, page = int(request.url.params["per_page"]), int(request.url.params["page"]) + meta = {"page": page, "per_page": str(size) if request.url.path.startswith("/v2/indicator") else size, "total": total, + "pages": (total + size - 1) // size if pages is None else pages, + "lastupdated": updated} + return httpx.Response(200, json=[meta, rows]) + + +def test_fred_v1_asof_private_key_native_missing_and_replay(store, clock): + requests = [] + def handler(request): + requests.append(request) + assert request.url.params["api_key"] == "0123456789abcdef0123456789abcdef" + assert request.url.params["realtime_start"] == "2008-09-01" + assert request.url.params["realtime_end"] == "2008-09-01" + assert request.url.params["limit"] == "100000" + return v1(request, [{"date": "2008-01-01", "value": "100.010000000001"}, + {"date": "2008-02-01", "value": "."}]) + source = fred(handler, store, clock) + page = next(source.observations("GDP", as_of="2008-09-01")) + assert page.complete and page.records[0]["value"] == "100.010000000001" + assert page.records[1]["value"] == "." + assert "0123456789abcdef0123456789abcdef" not in json.dumps(page.to_dict()) + assert "api_key" not in page.provenance.parameters + assert store.replay(page.provenance.snapshot_id).body + assert next(source.observations("GDP", as_of="2008-09-01")).cache_hit + assert len(requests) == 1 + + +def test_fred_default_vintage_pinned_and_v1_pagination(store, clock): + def handler(request): + assert request.url.params["realtime_start"] == "2026-10-02" + assert request.url.params["realtime_end"] == "2026-10-02" + offset = int(request.url.params["offset"]) + return v1(request, [{"date": f"2024-01-0{offset+1}", "value": str(offset)}], total=2) + pages = list(fred(handler, store, clock).observations("GDP", page_size=1)) + assert [page.complete for page in pages] == [False, True] + assert pages[0].next_cursor == "1" and pages[-1].records_seen == 2 + assert sum(clock.waits) == 0.5 + + +def test_fred_search_preserves_text_and_metadata_identity(store, clock): + query = 'inflation "united states"' + def handler(request): + assert request.url.params["search_text"] == query + assert request.url.params["order_by"] == "series_id" + return v1(request, [{"id": "CPIAUCSL", "units": "Index 1982-1984=100"}], key="seriess") + result = next(fred(handler, store, clock).search(query)) + assert result.records[0]["id"] == "CPIAUCSL" + assert result.provenance.parameters["search_text"] == query + + +@pytest.mark.parametrize("change", ["wrong-asof", "changed-count", "empty-page", "wrong-offset"]) +def test_fred_v1_refuses_incomplete_or_wrong_vintage(store, clock, change): + def handler(request): + offset = int(request.url.params["offset"]) + response = v1(request, [{"date": "2024-01-01", "value": "1"}], total=2) + body = response.json() + if change == "wrong-asof": + body["realtime_end"] = "2026-10-03" + elif change == "wrong-offset": + body["offset"] = offset + 1 + elif offset: + if change == "changed-count": + body["count"] = 3 + if change == "empty-page": + body["observations"] = [] + return httpx.Response(200, json=body) + with pytest.raises(InvalidResponse): + list(fred(handler, store, clock).observations("GDP", page_size=1)) + + +def test_fred_metadata_and_vintage_identity(store, clock): + def handler(request): + if request.url.path.endswith("vintagedates"): + return v1(request, ["2008-09-01", "2008-10-01"], key="vintage_dates") + return v1(request, [{"id": "GDP", "units": "Billions of Dollars", "notes": "Source BEA"}], key="seriess") + source = fred(handler, store, clock) + assert source.fetch("GDP").records[0]["notes"] == "Source BEA" + assert next(source.vintages("GDP")).records[0] == {"series_id": "GDP", "vintage_date": "2008-09-01"} + with pytest.raises(InvalidResponse): + source.fetch("UNRATE") + + +def test_fred_bulk_bearer_limit_grouping_and_metrics(store, clock): + def handler(request): + assert request.headers["Authorization"] == "Bearer 0123456789abcdef0123456789abcdef" + assert "api_key" not in request.url.params + assert request.url.params["limit"] == "500000" + if "next_cursor" not in request.url.params: + return release([series(value=".")], more=True, cursor="GDP,2024-04-01") + return release([series(date="2024-04-01")]) + source = fred(handler, store, clock) + pages = list(source.release_observations(53)) + assert pages[0].records[0]["observations"][0]["value"] == "." + assert pages[-1].complete and pages[-1].records_seen == 2 + assert pages[-1].records[0]["units"] == "Billions of Dollars" + assert source.usage()["observations_delivered"] == 2 + assert source.quota()["max_release_observations"] == 500000 + assert "0123456789abcdef0123456789abcdef" not in json.dumps(pages[0].to_dict()) + + +@pytest.mark.parametrize("mode", ["wrong-release", "missing-cursor", "repeated-cursor", "revised", "oversize"]) +def test_fred_bulk_rejects_bad_identity_or_incomplete_results(store, clock, mode): + def handler(request): + subsequent = "next_cursor" in request.url.params + if mode == "wrong-release": + return release([series()], release_id=99) + if mode == "missing-cursor": + return release([series()], more=True) + if mode == "oversize": + return release([series(), series("UNRATE")]) + if not subsequent: + return release([series()], more=True, cursor="GDP,2024-04-01") + if mode == "revised": + return release([series(date="2024-04-01", updated="2026-10-02T00:00:00Z")]) + return release([series()], more=True, cursor="GDP,2024-04-01") + with pytest.raises(InvalidResponse): + list(fred(handler, store, clock).release_observations(53, page_size=1)) + + +def test_fred_validation_fails_before_network(store, clock): + source = fred(lambda r: pytest.fail("unexpected network"), store, clock) + for operation in [lambda: list(source.observations("GDP", as_of="2008-09-01", realtime_end="2020-01-01")), + lambda: list(source.observations("GDP", observation_start="2025-01-01", observation_end="2024-01-01")), + lambda: list(source.observations("GDP", as_of="2024-02-30")), + lambda: list(source.observations("GDP", units="dollars")), + lambda: list(source.release_observations(53, page_size=500001)), + lambda: list(source.release_observations(53, cursor="GDP,2024-01-01")), + lambda: list(source.search("GDP", offset=1))]: + with pytest.raises(ValueError): + operation() + for key in ("", "short", "A" * 32, "a" * 31, "a" * 33, "a" * 31 + "-"): + with pytest.raises(ValueError): + FRED(key) + with pytest.raises(ValueError): + FRED("a" * 32, requests_per_second=3) + + +def test_worldbank_batch_maximum_page_native_null_and_cache(store, clock): + calls = [] + def handler(request): + calls.append(request) + assert request.url.params["per_page"] == "32767" + assert request.url.params["source"] == "2" + assert request.url.params["date"] == "2020:2024" + assert request.url.path.endswith("/NY.GDP.MKTP.CD;FP.CPI.TOTL.ZG") + return worldbank(request, [observation(), observation("FP.CPI.TOTL.ZG", value=None)]) + source = wb(handler, store, clock) + result = next(source.search("NY.GDP.MKTP.CD;FP.CPI.TOTL.ZG", date="2020:2024")) + assert result.complete and result.records[1]["value"] is None + assert result.records[1]["decimal"] == 0 + assert next(source.search("NY.GDP.MKTP.CD;FP.CPI.TOTL.ZG", date="2020:2024")).cache_hit + assert len(calls) == 1 + assert json.loads(store.replay(result.provenance.snapshot_id).body)[1][1]["value"] is None + + +def test_worldbank_pagination_resume_country_and_footnotes(store, clock): + def handler(request): + assert request.url.params["footnote"] == "y" + return worldbank(request, [observation(date=str(2025-int(request.url.params["page"])))], total=2) + source = wb(handler, store, clock) + pages = list(source.search("NY.GDP.MKTP.CD", countries="USA", footnotes=True, page_size=1)) + assert pages[0].next_cursor == "2" and pages[-1].complete + resumed = next(source.search("NY.GDP.MKTP.CD", countries="USA", footnotes=True, + page_size=1, page=2, records_seen=1)) + assert resumed.cache_hit and resumed.records_seen == 2 + + +@pytest.mark.parametrize("mode", ["error", "foreign-indicator", "foreign-country", "short-page", "wrong-pages", "revised", "count-change"]) +def test_worldbank_rejects_wrong_or_incomplete_data(store, clock, mode): + def handler(request): + subsequent = request.url.params["page"] == "2" + if mode == "error": + return httpx.Response(200, json=[{"message": [{"id": "120", "value": "bad indicator"}]}]) + if mode == "foreign-indicator": + return worldbank(request, [observation("SP.POP.TOTL")]) + if mode == "foreign-country": + return worldbank(request, [observation(country="FR", iso3="FRA")]) + if mode == "short-page": + return worldbank(request, [], total=2) + if mode == "wrong-pages": + return worldbank(request, [observation()], total=2, pages=1) + return worldbank(request, [observation()], total=3 if subsequent and mode == "count-change" else 2, + updated="2026-10-02" if subsequent and mode == "revised" else "2026-07-13") + with pytest.raises(InvalidResponse): + list(wb(handler, store, clock).search("NY.GDP.MKTP.CD", countries="USA", page_size=1)) + + +def test_worldbank_empty_result_complete(store, clock): + source = wb(lambda request: worldbank(request, None, total=0), store, clock) + result = next(source.search("NY.GDP.MKTP.CD")) + assert result.records == () and result.complete and result.total == 0 + + +def test_worldbank_indicator_metadata_identity(store, clock): + def handler(request): + return worldbank(request, [{"id": "NY.GDP.MKTP.CD", "unit": "", "source": {"id": "2"}, + "sourceOrganization": "World Bank national accounts data"}]) + source = wb(handler, store, clock) + assert source.fetch("NY.GDP.MKTP.CD").records[0]["source"]["id"] == "2" + assert next(source.indicators()).records[0]["sourceOrganization"] + with pytest.raises(InvalidResponse): + source.fetch("FP.CPI.TOTL.ZG") + with pytest.raises(InvalidResponse): + source.fetch("NY.GDP.MKTP.CD", source=16) + with pytest.raises(InvalidResponse): + next(source.indicators(source=16)) + + +def test_worldbank_limits_before_network(store, clock): + source = wb(lambda r: pytest.fail("unexpected network"), store, clock) + for operation in [lambda: list(source.search([f"GDP{n}" for n in range(61)])), + lambda: list(source.search("../private")), + lambda: list(source.search("GDP", countries="all;US")), + lambda: list(source.search("GDP", countries="US", date="2025:2024")), + lambda: list(source.search("GDP", date="2024M13")), + lambda: list(source.search("GDP", date="2024:2025Q1")), + lambda: list(source.search("GDP", page_size=32768)), + lambda: list(source.search("GDP", page=2)), + lambda: list(source.search("GDP;GDP")), + lambda: list(source.search("A" * 1501))]: + with pytest.raises(ValueError): + operation() diff --git a/tests/test_finance_sec.py b/tests/test_finance_sec.py new file mode 100644 index 0000000..347dd46 --- /dev/null +++ b/tests/test_finance_sec.py @@ -0,0 +1,253 @@ +"""SEC source contracts: identity, native units and complete filing history.""" +from datetime import datetime, timezone +import json + +import httpx +import pytest + +from dedomena.sources.core import HTTPFailure, InvalidResponse, Store +from dedomena.sources.sec import SEC + + +class Clock: + def __init__(self): + self.now = datetime(2026, 10, 2, tzinfo=timezone.utc).timestamp() + self.waits = [] + + def __call__(self): + return self.now + + def sleep(self, amount): + self.waits.append(amount) + self.now += amount + + +@pytest.fixture +def clock(): + return Clock() + + +@pytest.fixture +def store(clock): + with_store = Store(clock=clock) + yield with_store + with_store.close() + + +def sec(handler, store, clock, **kwargs): + return SEC("Research institution researcher@example.org", + client=httpx.Client(transport=httpx.MockTransport(handler)), + store=store, sleep=clock.sleep, **kwargs) + + +def fact(accn="0000320193-26-000001", **kwargs): + return {"accn": accn, "end": "2026-06-30", "filed": "2026-07-15", + "form": "10-Q", "val": 1234, **kwargs} + + +def company_facts(cik=320193): + return {"cik": cik, "entityName": "Company", "facts": {"us-gaap": {"Assets": { + "label": "Assets", "description": "Total assets", + "units": {"USD": [fact(), fact("0000320193-26-000002", val=2345)], + "CAD": [fact(val=3456)]}}}}} + + +def columns(numbers=(1,)): + return {"accessionNumber": [f"0000320193-26-{n:06d}" for n in numbers], + "filingDate": ["2026-07-15"] * len(numbers), + "form": ["10-Q"] * len(numbers), + "primaryDocument": ["report.htm"] * len(numbers)} + + +def submissions(files=None): + return {"cik": "0000320193", "name": "Company", "tickers": ["CMP"], + "filings": {"recent": columns(), "files": files or []}} + + +def descriptor(number=1, count=1): + return {"name": f"CIK0000320193-submissions-{number:03d}.json", "filingCount": count, + "filingFrom": "2025-01-01", "filingTo": "2025-12-31"} + + +def test_companyfacts_one_call_keeps_units_amendments_identity_and_replay(store, clock): + calls = [] + body = company_facts() + def handler(request): + calls.append(request) + assert request.url.path == "/api/xbrl/companyfacts/CIK0000320193.json" + assert request.headers["User-Agent"] == "Research institution researcher@example.org" + assert request.headers.get("Authorization", "") == "" + return httpx.Response(200, json=body) + source = sec(handler, store, clock) + page = source.fetch("CIK320193") + assert page.records == (body,) + assert page.complete and page.total == 1 + units = page.records[0]["facts"]["us-gaap"]["Assets"]["units"] + assert list(units) == ["USD", "CAD"] + assert [row["val"] for row in units["USD"]] == [1234, 2345] + assert json.loads(store.replay(page.provenance.snapshot_id).body) == body + assert source.fetch(320193).cache_hit + assert len(calls) == 1 + + +@pytest.mark.parametrize("cik", [True, 0, -1, 10_000_000_000, "apple", "320193/..", "00000000000", "CIK", None]) +def test_invalid_cik_rejected_before_network(cik, store, clock): + source = sec(lambda request: pytest.fail("invalid CIK sent upstream"), store, clock) + with pytest.raises(ValueError): + source.fetch(cik) + + +@pytest.mark.parametrize("user_agent", ["", "python-httpx", "user@example.org", "Company no-email", "Company a@b.c\nInjected: x"]) +def test_user_agent_requires_organization_and_contact(user_agent): + with pytest.raises(ValueError): + SEC(user_agent) + + +def test_rate_limit_aggregate_across_user_agents(store, clock): + handler = lambda request: httpx.Response(200, json=company_facts()) + first = sec(handler, store, clock) + second = SEC("Another institution contact@example.org", store=store, sleep=clock.sleep, + client=httpx.Client(transport=httpx.MockTransport(handler))) + first.fetch(320193) + second.fetch(320193) + assert first.scope == second.scope + assert clock.waits == pytest.approx([0.1]) + with pytest.raises(ValueError): + SEC("Company contact@example.org", requests_per_second=11) + + +@pytest.mark.parametrize("body", [ + company_facts(cik=1), + {"cik": 320193, "facts": []}, + {"cik": 320193, "facts": {"us-gaap": {"Assets": {"units": {"USD": [fact(val=None)]}}}}}, + {"cik": 320193, "facts": {"us-gaap": {"Assets": {"units": {"USD": [fact(accn="missing")]}}}}}, + {"cik": 320193, "facts": {"us-gaap": {"Assets": {"units": {"USD": [fact(end="bad-date")]}}}}}, +]) +def test_companyfacts_rejects_foreign_identity_or_broken_units(body, store, clock): + source = sec(lambda request: httpx.Response(200, json=body), store, clock) + with pytest.raises(InvalidResponse): + source.fetch(320193) + + +def test_single_concept_and_frame_keep_semantic_envelopes(store, clock): + concept = {"cik": 320193, "taxonomy": "us-gaap", "tag": "Assets", + "units": {"USD": [fact()]}, "entityName": "Company"} + observation = fact(cik=320193, entityName="Company") + del observation["filed"] + frame = {"taxonomy": "us-gaap", "tag": "Assets", "uom": "USD", "ccp": "CY2026Q2I", + "pts": 1, "data": [observation]} + def handler(request): + if "companyconcept" in request.url.path: + assert request.url.path.endswith("/us-gaap/Assets.json") + return httpx.Response(200, json=concept) + assert request.url.path.endswith("/us-gaap/Assets/USD/CY2026Q2I.json") + return httpx.Response(200, json=frame) + source = sec(handler, store, clock) + assert source.company_concept(320193, "us-gaap", "Assets").records == (concept,) + page = source.frame("us-gaap", "Assets", "USD", "CY2026Q2I") + assert page.records == (frame,) and page.complete + assert "latest filing" in page.warnings[0] + + +@pytest.mark.parametrize("field,value", [("ccp", "CY2025Q2I"), ("uom", "CAD"), ("pts", 2)]) +def test_frame_identity_and_counts_must_match(field, value, store, clock): + body = {"taxonomy": "us-gaap", "tag": "Assets", "uom": "USD", "ccp": "CY2026Q2I", + "pts": 1, "data": [fact(cik=320193)], field: value} + source = sec(lambda request: httpx.Response(200, json=body), store, clock) + with pytest.raises(InvalidResponse): + source.frame("us-gaap", "Assets", "USD", "CY2026Q2I") + + +def test_submissions_enumerates_recent_and_every_historical_file(store, clock): + calls = [] + bodies = {"/submissions/CIK0000320193.json": submissions([descriptor(1), descriptor(2)]), + "/submissions/CIK0000320193-submissions-001.json": columns((2,)), + "/submissions/CIK0000320193-submissions-002.json": columns((3,))} + def handler(request): + calls.append(request.url.path) + return httpx.Response(200, json=bodies[request.url.path]) + source = sec(handler, store, clock) + pages = list(source.submissions(320193)) + assert len(calls) == 3 + assert [page.complete for page in pages] == [False, False, True] + assert [page.total for page in pages] == [3, 3, 3] + assert [page.records_seen for page in pages] == [1, 2, 3] + assert [page.records[0]["accessionNumber"] for page in pages] == [ + "0000320193-26-000001", "0000320193-26-000002", "0000320193-26-000003"] + assert "company metadata" in pages[0].warnings[0] + assert "tickers" in json.loads(store.replay(pages[0].provenance.snapshot_id).body) + resumed = list(source.submissions(320193, cursor=pages[0].next_cursor, + records_seen=pages[0].records_seen)) + assert [page.records for page in resumed] == [page.records for page in pages[1:]] + assert all(page.cache_hit for page in resumed) + + +def test_recent_only_is_explicitly_incomplete(store, clock): + body = submissions([descriptor()]) + source = sec(lambda request: httpx.Response(200, json=body), store, clock) + page = next(source.submissions(320193, include_history=False)) + assert not page.complete and page.total == 2 + assert page.next_cursor == "CIK0000320193-submissions-001.json" + assert "incomplete" in page.warnings[-1] + + +@pytest.mark.parametrize("historical", [columns(()), columns((1,)), {"accessionNumber": ["wrong"]}, + {"accessionNumber": ["0000320193-26-000002"], "form": []}]) +def test_history_cannot_silently_truncate_or_duplicate(historical, store, clock): + def handler(request): + body = submissions([descriptor()]) if request.url.path.endswith("CIK0000320193.json") else historical + return httpx.Response(200, json=body) + with pytest.raises(InvalidResponse): + list(sec(handler, store, clock).submissions(320193)) + + +@pytest.mark.parametrize("filename", ["../other.json", "CIK0000000001-submissions-001.json", "https://evil.test/file.json"]) +def test_manifest_cannot_redirect_company_identity_or_origin(filename, store, clock): + body = submissions([{**descriptor(), "name": filename}]) + source = sec(lambda request: httpx.Response(200, json=body), store, clock) + with pytest.raises(InvalidResponse): + list(source.submissions(320193)) + + +def test_changed_manifest_refuses_resume_and_size_limit_is_configurable(store, clock): + body = submissions([descriptor()]) + source = sec(lambda request: httpx.Response(200, json=body), store, clock) + with pytest.raises(ValueError, match="checkpoint"): + list(source.submissions(320193, cursor=descriptor()["name"], records_seen=2)) + limited = sec(lambda request: httpx.Response(200, json=company_facts()), store, clock, + max_response_bytes=30) + with pytest.raises(InvalidResponse, match="size limit"): + limited.fetch(320193) + + +def test_provider_no_facts_404_is_failure_not_empty_dataset(store, clock): + source = sec(lambda request: httpx.Response(404, json={"error": "No company facts"}), store, clock) + with pytest.raises(HTTPFailure, match="HTTP 404"): + source.fetch(320193) + + +def test_ratio_frame_uses_native_slash_unit_and_url_per_encoding(store, clock): + body = {"taxonomy": "us-gaap", "tag": "EarningsPerShareDiluted", "uom": "USD/shares", + "ccp": "CY2025", "pts": 1, "data": [fact(cik=320193)]} + def handler(request): + assert "/USD-per-shares/" in request.url.path + return httpx.Response(200, json=body) + page = sec(handler, store, clock).frame("us-gaap", "EarningsPerShareDiluted", "USD-per-shares", "CY2025") + assert page.records[0]["uom"] == "USD/shares" + + +def test_keyless_pacing_scope_cannot_be_overridden(): + with pytest.raises(ValueError, match="aggregate pacing"): + SEC("Company researcher@example.org", credential="separate-quota") + + +def test_parallel_company_reads_preserve_order_and_window_duplicates(store, clock): + called = [] + def handler(request): + cik = int(request.url.path.rsplit("/", 1)[-1].removeprefix("CIK").removesuffix(".json")) + called.append(cik) + return httpx.Response(200, json=company_facts(cik)) + source = sec(handler, store, clock) + pages = list(source.fetch_many([2, 1, "CIK2"], workers=3, refresh=True)) + assert [page.records[0]["cik"] for page in pages] == [2, 1, 2] + assert sorted(called) == [1, 2] diff --git a/tests/test_hyperliquid.py b/tests/test_hyperliquid.py new file mode 100644 index 0000000..05b4fe8 --- /dev/null +++ b/tests/test_hyperliquid.py @@ -0,0 +1,297 @@ +import json + +import httpx +import pytest + +from dedomena.sources.core import InvalidResponse, SearchLimitExceeded, Store, canonical +from dedomena.sources.hyperliquid import Hyperliquid, info_policy + + +BOOK = {"coin": "BTC", "time": 1000, + "levels": [[{"px": "113377.0123456789", "sz": "7.6699", "n": 17}], []]} +META = [{"universe": [{"name": "BTC", "szDecimals": 5}], + "marginTables": [[50, {"marginTiers": [{"lowerBound": "0.0", "maxLeverage": 50}]}]], + "collateralToken": 0}, + [{"markPx": "113377.0123456789", "premium": None, "impactPxs": None, "midPx": None}]] +CANDLE = {"s": "BTC", "i": "1m", "t": 1000, "T": 60999, "n": 1, + "o": "100.0123456789", "h": "100.3", "l": "99.9", "c": "100.1", "v": "0.00123"} + + +def source(handler, **kwargs): + return Hyperliquid(client=httpx.Client(transport=httpx.MockTransport(handler)), + store=kwargs.pop("store", Store()), sleep=lambda _: None, **kwargs) + + +def funding(time, rate="0.0000125"): + return {"coin": "BTC", "time": time, "fundingRate": rate, "premium": "-0.00031774"} + + +def test_book_preserves_native_precision_receipt_and_replay(): + calls = [] + def handler(request): + calls.append(request) + assert request.method == "POST" + assert request.url == "https://api.hyperliquid.xyz/info" + assert json.loads(request.content) == {"type": "l2Book", "coin": "BTC", "nSigFigs": 5, "mantissa": 2} + return httpx.Response(200, json=BOOK) + with source(handler, cache_ttl=86400) as liquid: + page = liquid.order_book("BTC", n_sig_figs=5, mantissa=2) + assert page.complete and page.total == 1 + assert page.records == (BOOK,) + assert page.records[0]["levels"][0][0]["px"] == "113377.0123456789" + assert json.loads(page.provenance.request_body)["mantissa"] == 2 + assert json.loads(liquid.store.replay(page.provenance.snapshot_id).body) == BOOK + again = liquid.order_book("BTC", n_sig_figs=5, mantissa=2) + assert again.cache_hit and len(calls) == 1 + + +def test_metadata_retains_margin_tables_nulls_and_context_positions(): + with source(lambda _: httpx.Response(200, json=META)) as liquid: + page = liquid.markets() + assert page.records == ({"meta": META[0], "assetCtxs": META[1]},) + assert page.records[0]["assetCtxs"][0]["premium"] is None + assert page.complete and page.warnings + + +def test_spot_contexts_are_not_zipped_or_truncated_to_pair_universe(): + data = [{"universe": [{"name": "PURR/USDC", "tokens": [1, 0], "index": 0}], + "tokens": [{"name": "USDC", "index": 0}, {"name": "PURR", "index": 1}]}, + [{"coin": "PURR/USDC", "midPx": "0.15"}, {"coin": "@999", "midPx": None}]] + def handler(request): + assert json.loads(request.content) == {"type": "spotMetaAndAssetCtxs"} + return httpx.Response(200, json=data) + with source(handler) as liquid: + page = liquid.markets(spot=True) + assert len(page.records[0]["assetCtxs"]) == 2 + assert page.records[0]["meta"]["tokens"] == data[0]["tokens"] + + +def test_all_mids_is_one_native_map_including_spot_and_prediction_asset_ids(): + data = {"BTC": "113377.0123456789", "@107": "36.456", "#14720": "0.385"} + with source(lambda _: httpx.Response(200, json=data)) as liquid: + page = liquid.all_mids() + assert page.records == (data,) + assert page.complete and page.records_seen == 1 + + +def test_candles_preserve_precision_and_never_claim_complete_history(): + def handler(request): + assert json.loads(request.content) == {"type": "candleSnapshot", "req": { + "coin": "BTC", "interval": "1m", "startTime": 1000, "endTime": 2000}} + return httpx.Response(200, json=[CANDLE]) + with source(handler) as liquid: + page = liquid.candles("BTC", "1m", 1000, 2000) + assert page.records == (CANDLE,) + assert not page.complete and page.next_cursor is None + assert any("5,000" in warning for warning in page.warnings) + + +def test_empty_candles_do_not_prove_historical_coverage(): + with source(lambda _: httpx.Response(200, json=[])) as liquid: + assert not liquid.candles("BTC", "1d", 0, 1000).complete + + +def test_funding_pagination_inclusive_overlap_and_count_are_exact(): + calls = [] + first = [funding(time) for time in range(1, 501)] + second = [funding(500), funding(501)] + def handler(request): + body = json.loads(request.content) + calls.append(body) + assert body["endTime"] == 1000 + return httpx.Response(200, json=first if body["startTime"] == 1 else second) + with source(handler) as liquid: + pages = list(liquid.funding_history("BTC", 1, 1000)) + assert len(pages) == 2 + assert len(pages[0].records) == 500 and not pages[0].complete + assert pages[1].records == (funding(501),) + assert pages[1].complete and pages[1].records_seen == 501 + assert pages[1].next_cursor is None + assert [body["startTime"] for body in calls] == [1, 500] + assert liquid.usage()["upstream_records"] == 502 + assert liquid.usage()["records_delivered"] == 501 + assert liquid.usage()["boundary_repeats"] == 1 + + +def test_funding_checkpoint_pins_default_end_and_deduplicates_resume(): + store = Store(clock=lambda: 1.0) + bodies = [] + def handler(request): + body = json.loads(request.content) + bodies.append(body) + return httpx.Response(200, json=[funding(time) for time in range(1, 501)] + if body["startTime"] == 1 else [funding(500), funding(501)]) + with source(handler, store=store) as liquid: + page = next(liquid.funding_history("BTC", 1)) + store.clock = lambda: 2.0 + resumed = list(liquid.funding_history("BTC", 1, cursor=page.next_cursor)) + assert resumed[0].records == (funding(501),) + assert resumed[0].records_seen == 501 + assert resumed[0].complete + assert bodies[-1]["endTime"] == 1000 + with pytest.raises(ValueError): + list(liquid.funding_history("ETH", 1, cursor=page.next_cursor)) + with pytest.raises(ValueError): + list(liquid.funding_history("BTC", 2, cursor=page.next_cursor)) + with pytest.raises(ValueError): + list(liquid.funding_history("BTC", 1, 2000, cursor=page.next_cursor)) + + +def test_full_same_timestamp_funding_fails_before_partial_page_is_yielded(): + rows = [funding(1, str(index)) for index in range(500)] + with source(lambda _: httpx.Response(200, json=rows)) as liquid: + with pytest.raises(SearchLimitExceeded): + next(liquid.funding_history("BTC", 1, 1000)) + assert liquid.usage().get("records_delivered", 0) == 0 + + +def test_multiple_distinct_boundary_rows_are_deduplicated_without_losing_new_row(): + first = [funding(time) for time in range(1, 499)] + [funding(499, "0.1"), funding(499, "0.2")] + second = [funding(499, "0.1"), funding(499, "0.2"), funding(499, "0.3"), funding(500)] + with source(lambda request: httpx.Response(200, json=first + if json.loads(request.content)["startTime"] == 1 else second)) as liquid: + pages = list(liquid.funding_history("BTC", 1, 1000)) + assert pages[-1].records == (funding(499, "0.3"), funding(500)) + assert pages[-1].records_seen == 502 + + +@pytest.mark.parametrize("method,args,kwargs", [ + ("order_book", ("../BTC",), {}), + ("order_book", ("BTC",), {"n_sig_figs": True}), + ("order_book", ("BTC",), {"mantissa": 2}), + ("order_book", ("BTC",), {"n_sig_figs": 5, "mantissa": 3}), + ("all_mids", (), {"dex": "../test"}), + ("markets", (), {"spot": "true"}), + ("markets", (), {"spot": True, "dex": "xyz"}), + ("candles", ("BTC", "2m", 1, 2), {}), + ("candles", ("BTC", "1m", True, 2), {}), + ("candles", ("BTC", "1m", 3, 2), {}), + ("funding_history", ("BTC", 1, 2), {"cursor": "{}"}), + ("funding_history", ("BTC", -1, 2), {}), + ("funding_history", ("BTC", 1, True), {}), +]) +def test_invalid_queries_fail_before_network(method, args, kwargs): + with source(lambda _: pytest.fail("unexpected network")) as liquid: + with pytest.raises(ValueError): + result = getattr(liquid, method)(*args, **kwargs) + if method == "funding_history": + list(result) + + +@pytest.mark.parametrize("method,body", [ + ("all_mids", {"BTC": 123.4}), + ("all_mids", {"BTC": "NaN"}), + ("order_book", {**BOOK, "coin": "ETH"}), + ("order_book", {**BOOK, "time": True}), + ("order_book", {**BOOK, "levels": [[{"px": "1", "sz": "-1", "n": 1}], []]}), + ("markets", [{"universe": [{"name": "BTC"}]}, []]), + ("markets", [{"universe": [{"name": "BTC"}, {"name": "BTC"}]}, [{}, {}]]), +]) +def test_malformed_snapshot_responses_fail_closed(method, body): + with source(lambda _: httpx.Response(200, json=body)) as liquid: + with pytest.raises(InvalidResponse): + getattr(liquid, method)("BTC") if method == "order_book" else getattr(liquid, method)() + + +@pytest.mark.parametrize("row", [ + {**CANDLE, "s": "ETH"}, {**CANDLE, "i": "1d"}, {**CANDLE, "t": True}, + {**CANDLE, "t": 3000}, {**CANDLE, "c": 100.1}, {**CANDLE, "v": "Infinity"}, +]) +def test_malformed_candle_identity_and_values_fail_closed(row): + with source(lambda _: httpx.Response(200, json=[row])) as liquid: + with pytest.raises(InvalidResponse): + liquid.candles("BTC", "1m", 1000, 2000) + + +@pytest.mark.parametrize("rows", [ + [funding(2), funding(1)], [funding(1), funding(1)], + [{**funding(1), "coin": "ETH"}], [funding(3000)], + [{**funding(1), "fundingRate": 0.1}], [{**funding(1), "time": True}], +]) +def test_malformed_funding_order_identity_and_values_fail_closed(rows): + with source(lambda _: httpx.Response(200, json=rows)) as liquid: + with pytest.raises(InvalidResponse): + list(liquid.funding_history("BTC", 1, 2000)) + + +@pytest.mark.parametrize("body", [ + {"type": "order", "coin": "BTC"}, {"type": "userFills", "user": "0xabc"}, + {"type": "recentTrades", "coin": "BTC"}, {"type": []}, + {"type": "allMids", "user": "0xabc"}, {"type": "l2Book", "coin": "BTC", "action": {}}, + {"type": "candleSnapshot", "req": {"coin": "BTC", "interval": "1m", "startTime": 0}}, + {"type": "fundingHistory", "coin": "BTC", "startTime": False, "endTime": 1000}, +]) +def test_transport_policy_rejects_account_actions_unknown_types_and_fields(body): + with pytest.raises(ValueError): + info_policy(canonical(body).encode()) + with source(lambda _: pytest.fail("unexpected network")) as liquid: + with pytest.raises(ValueError): + liquid._request("POST", "/info", content=canonical(body).encode()) + + +@pytest.mark.parametrize("path", ["/exchange", "/info/../exchange", "//elsewhere/info"]) +def test_transport_cannot_reach_exchange_even_directly(path): + with source(lambda _: pytest.fail("unexpected network")) as liquid: + with pytest.raises(ValueError): + liquid._request("POST", path, content=b'{"type":"allMids"}') + + +@pytest.mark.parametrize("body,maximum,actual_count,actual", [ + ({"type": "allMids"}, 2, None, 2), + ({"type": "metaAndAssetCtxs"}, 20, None, 20), + ({"type": "fundingHistory", "coin": "BTC", "startTime": 0, "endTime": 1000}, 45, 21, 22), + ({"type": "candleSnapshot", "req": {"coin": "BTC", "interval": "1m", "startTime": 0, "endTime": 1000}}, 104, 61, 22), +]) +def test_provider_policy_reserves_maximum_and_settles_by_count(body, maximum, actual_count, actual): + reserved, count = info_policy(canonical(body).encode()) + assert reserved == maximum + if actual_count is not None: + assert count(canonical([{}] * actual_count).encode()) == actual + with pytest.raises(ValueError): + count(b'{}') + else: + assert count is None + + +@pytest.mark.parametrize("limit", [(1201, 60), (1200, 59), None, (True, 60), (1200, float("nan"))]) +def test_provider_ip_policy_cannot_be_weakened(limit): + with pytest.raises(ValueError): + source(lambda _: pytest.fail("unexpected network"), ip_limit=limit) + + +def test_quota_and_fetch_alias_are_agent_accessible(): + with source(lambda _: httpx.Response(200, json=BOOK)) as liquid: + assert liquid.quota()["ip_weight_per_minute"] == 1200 + assert liquid.quota()["api_key_required"] is False + assert liquid.ip_throttle_only + assert liquid.fetch("BTC").records == (BOOK,) + + + +def test_direct_transport_cannot_undercharge_a_count_weighted_endpoint(): + body = {"type": "candleSnapshot", "req": {"coin": "BTC", "interval": "1m", + "startTime": 0, "endTime": 1000}} + with source(lambda _: httpx.Response(200, json=[])) as liquid: + liquid._request("POST", "/info", content=canonical(body).encode(), + rate_weight=1, actual_weight=lambda _: 1) + usage = liquid.usage() + assert usage["ip_weight_reserved"] == 104 + assert usage["ip_weight_used"] == 20 + + +@pytest.mark.parametrize("kwargs", [{"params": {"type": "allMids"}}, + {"form": {"type": "allMids"}}, + {"auth": ("user", "pass")}, + {"private_params": {"api_key": "secret"}}]) +def test_hyperliquid_transport_only_accepts_public_json_body(kwargs): + with source(lambda _: pytest.fail("unexpected network")) as liquid: + with pytest.raises(ValueError): + liquid._request("POST", "/info", content=b'{"type":"allMids"}', **kwargs) + + +def test_malformed_response_keeps_conservative_weight_reservation(): + with source(lambda _: httpx.Response(200, json={"unexpected": "schema"})) as liquid: + with pytest.raises(InvalidResponse): + liquid.candles("BTC", "1m", 0, 1000) + usage = liquid.usage() + assert usage["ip_weight_reserved"] == usage["ip_weight_used"] == 104 diff --git a/tests/test_hyperliquid_cli.py b/tests/test_hyperliquid_cli.py new file mode 100644 index 0000000..323b878 --- /dev/null +++ b/tests/test_hyperliquid_cli.py @@ -0,0 +1,125 @@ +"""Public market operations, route configuration and offline replay in JSON CLI.""" +import json + +import httpx +import pytest + +from dedomena.sources import Hyperliquid, IPPool, IPRoute, Store +from dedomena.sources import __main__ as cli + + +@pytest.fixture +def market_cli(monkeypatch): + store, requests = Store(), [] + def handler(request): + body = json.loads(request.content) + requests.append(body) + kind = body["type"] + if kind == "allMids": + data = {"BTC": "100.00000001"} + elif kind in ("metaAndAssetCtxs", "spotMetaAndAssetCtxs"): + data = [{"universe": [{"name": "BTC"}], "tokens": []}, [{"markPx": "100.1"}]] + elif kind == "l2Book": + data = {"coin": body["coin"], "time": 1234, "levels": [[], []]} + elif kind == "candleSnapshot": + data = [{"s": "BTC", "i": "1h", "t": 0, "T": 3599999, "n": 2, + "o": "100.01", "h": "101", "l": "99", "c": "100.02", "v": "1.1"}] + else: + assert kind == "fundingHistory" + data = [{"coin": "BTC", "time": 1234, "fundingRate": "0.0001", "premium": "0.001"}] + return httpx.Response(200, json=data) + client = httpx.Client(transport=httpx.MockTransport(handler)) + def constructor(**options): + options.pop("store", None) + if "ip_pool" not in options: + options["client"] = client + return Hyperliquid(store=store, **options) + monkeypatch.setattr(cli, "Hyperliquid", constructor) + yield requests, client, store + client.close() + store.close() + + +@pytest.mark.parametrize("args,kind,complete", [ + (["mids", "hyperliquid", "--dex", "xyz"], "allMids", True), + (["markets", "hyperliquid"], "metaAndAssetCtxs", True), + (["catalogue", "hyperliquid", "--spot"], "spotMetaAndAssetCtxs", True), + (["book", "hyperliquid", "BTC"], "l2Book", True), + (["candles", "hyperliquid", "BTC", "--interval", "1h", "--start-time", "0", + "--end-time", "3600000"], "candleSnapshot", False), + (["funding", "hyperliquid", "BTC", "--start-time", "0", "--end-time", "3600000"], + "fundingHistory", True), +]) +def test_market_operations_emit_native_page_and_truthful_receipt(market_cli, capsys, args, kind, complete): + assert cli.main(args) == 0 + requests, _, _ = market_cli + page, receipt = [json.loads(line) for line in capsys.readouterr().out.splitlines()] + assert requests[0]["type"] == kind + assert page["provenance"]["method"] == "POST" + assert json.loads(page["provenance"]["request_body"]) == requests[0] + assert page["complete"] == receipt["complete"] == complete + assert receipt["usage"]["ip_weight_used"] == (2 if kind in ("allMids", "l2Book") else + 21 if kind in ("candleSnapshot", "fundingHistory") else 20) + + +def test_fetch_alias_and_benchmark_mids(market_cli, capsys): + assert cli.main(["fetch", "hyperliquid", "BTC"]) == 0 + page = json.loads(capsys.readouterr().out) + assert page["records"][0]["coin"] == "BTC" + assert cli.main(["benchmark", "hyperliquid", "--operation", "mids"]) == 0 + receipt = json.loads(capsys.readouterr().out) + assert receipt["operation"] == "mids" and receipt["complete"] + assert receipt["records"] == 1 + + +def test_route_file_configures_shared_pool_and_receipt(tmp_path, monkeypatch, market_cli, capsys): + _, client, _ = market_cli + configured, pools = [], [] + def pool_factory(routes): + routes = list(routes) + configured.extend(routes) + pool = IPPool([IPRoute(route.public_ip, client=client) for route in routes]) + pools.append(pool) + return pool + monkeypatch.setattr(cli, "IPPool", pool_factory) + path = tmp_path / "routes.json" + path.write_text(json.dumps([{"public_ip": "203.0.113.1", "local_address": "10.0.0.1"}])) + assert cli.main(["mids", "hyperliquid", "--ip-routes", str(path)]) == 0 + page = json.loads(capsys.readouterr().out.splitlines()[0]) + assert configured[0].local_address == "10.0.0.1" + assert page["provenance"]["egress_ip_sha256"] is not None + assert pools[0]._closed and not client.is_closed + + +@pytest.mark.parametrize("contents", ["bad JSON", "{}", "[]", + '[{"public_ip":"203.0.113.1","client":"not a client"}]', + '[{"public_ip":"203.0.113.1","proxy":"http://user:secret@proxy.invalid/?bad"}]', +]) +def test_bad_route_file_is_structured_and_never_dispatches(tmp_path, market_cli, capsys, contents): + path = tmp_path / "routes.json" + path.write_text(contents) + assert cli.main(["mids", "hyperliquid", "--ip-routes", str(path)]) == 2 + output = capsys.readouterr() + assert not output.out and not market_cli[0] + assert json.loads(output.err)["error"] == "ValueError" + assert "secret" not in output.err + + +def test_missing_route_file_is_structured(tmp_path, market_cli, capsys): + assert cli.main(["mids", "hyperliquid", "--ip-routes", str(tmp_path / "absent")]) == 2 + assert not market_cli[0] + assert json.loads(capsys.readouterr().err)["error"] == "ValueError" + + +def test_hyperliquid_replay_needs_no_routes(tmp_path, capsys): + path = tmp_path / "saved.sqlite3" + store = Store(path) + client = httpx.Client(transport=httpx.MockTransport(lambda _: httpx.Response(200, json={"BTC": "100.00001"}))) + with Hyperliquid(store=store, client=client) as source: + page = source.all_mids() + store.close() + client.close() + assert cli.main(["replay", "hyperliquid", page.provenance.snapshot_id, + "--store", str(path), "--ip-routes", "not-needed.json"]) == 0 + replay = json.loads(capsys.readouterr().out) + assert json.loads(replay["body"]) == {"BTC": "100.00001"} diff --git a/tests/test_ip_transport.py b/tests/test_ip_transport.py new file mode 100644 index 0000000..73fbedb --- /dev/null +++ b/tests/test_ip_transport.py @@ -0,0 +1,214 @@ +"""Quota, cache, dispatch and retry contracts across routed source clients.""" +from datetime import datetime, timezone + +import httpx +import pytest + +from dedomena.sources import BudgetExceeded, IPPool, IPRoute, OpenAlex, Store, Throttled +from dedomena.sources.core import SourceError, Transport, digest, window + + +class Clock: + def __init__(self, now=1000.0): + self.now = now + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +def routed(*handlers): + return IPPool([IPRoute(f"203.0.113.{i + 1}", client=httpx.Client( + transport=httpx.MockTransport(handler))) for i, handler in enumerate(handlers)]) + + +def source(store, clock, pool, **kwargs): + kwargs.setdefault("requests_per_second", 1000) + return Transport("test", "https://example.org", store=store, ip_pool=pool, + sleep=clock.sleep, **kwargs) + + +def test_shared_cache_identity_and_replay_preserve_acquisition_route(): + clock, sent = Clock(), [] + def handler(label): + return lambda request: sent.append(label) or httpx.Response(200, json={"value": "1.00001"}) + with routed(handler("first"), handler("second")) as pool: + store = Store(clock=clock) + client = source(store, clock, pool, ip_limit=(10, 60)) + first = client._request("GET", "/same") + cached = client._request("GET", "/same") + refreshed = client._request("GET", "/same", refresh=True) + assert sent == ["first", "second"] + assert cached.cache_hit and cached.provenance == first.provenance + assert first.provenance.request_sha256 == refreshed.provenance.request_sha256 + assert first.provenance.egress_ip_sha256 == digest(b"203.0.113.1") + assert refreshed.provenance.egress_ip_sha256 == digest(b"203.0.113.2") + assert store.replay(first.provenance.snapshot_id).provenance == first.provenance + assert client.usage()["ip_weight_used"] == 2 + client.close() + assert not pool._closed + store.close() + + +def test_openalex_pool_cannot_multiply_key_budget_or_charge_unsent_requests(): + clock, sent = Clock(), [] + def handler(request): + sent.append(request.url.path) + if request.url.path == "/rate-limit": + return httpx.Response(200, json={"rate_limit": { + "credits_limit": 10000, "credits_remaining": 10000, + "credit_costs": {"singleton": 0, "list": 1, "search": 10}}}) + return httpx.Response(200, json={"results": [{"id": "https://openalex.org/W1"}], + "meta": {"count": 1, "next_cursor": None}}, + headers={"X-RateLimit-Credits-Used": "1"}) + store = Store(clock=clock) + with routed(handler, handler) as pool: + client = OpenAlex("fixture-secret", ip_pool=pool, store=store, + daily_credit_limit=1, sleep=clock.sleep) + assert next(client.search(filter="type:article")).complete + with pytest.raises(BudgetExceeded): + next(client.search(filter="type:article", refresh=True)) + assert sent == ["/rate-limit", "/works"] + with store.transaction() as db: + assert db.execute("SELECT sum(weight) FROM ip_events").fetchone()[0] == 2 + assert store.used(client.scope, "credits", "day") == 1 + store.close() + + +def test_verified_weight_releases_capacity_for_a_known_cheap_request(): + clock = Clock() + store = Store(clock=clock) + with routed(lambda _: httpx.Response(200, json={})) as pool: + client = source(store, clock, pool, ip_limit=(4, 60), max_wait=0) + client._request("GET", "/large-estimate", rate_weight=4, actual_weight=lambda _: 2) + clock.sleep(0.01) + client._request("GET", "/cheap", rate_weight=2) + clock.sleep(0.01) + with pytest.raises(Throttled): + client._request("GET", "/exhausted", rate_weight=1) + assert client.usage()["ip_weight_reserved"] == 6 + assert client.usage()["ip_weight_used"] == 4 + store.close() + + +def test_uncertain_send_retains_weight_and_error_does_not_expose_credentials(): + clock = Clock() + store = Store(clock=clock) + def handler(request): + raise httpx.ProxyError("http://user:private-password@example.net", request=request) + with routed(handler) as pool: + client = source(store, clock, pool, ip_limit=(4, 60), max_wait=0, max_attempts=1) + with pytest.raises(SourceError, match="transport failure") as error: + client._request("GET", "/uncertain", rate_weight=4) + assert "private-password" not in str(error.value) + clock.sleep(0.01) + with pytest.raises(Throttled): + client._request("GET", "/next") + assert client.usage()["ip_weight_used"] == 4 + store.close() + + +def test_budget_work_precedes_fresh_dispatch_lease(): + clock, sent = Clock(), [] + store = Store(clock=clock) + original = store._reserve_in + delayed = False + def reserve(*args, **kwargs): + nonlocal delayed + if not delayed: + delayed = True + clock.sleep(60) + return original(*args, **kwargs) + store._reserve_in = reserve + with routed(lambda _: sent.append(clock()) or httpx.Response(200, json={})) as pool: + client = source(store, clock, pool, ip_limit=(1, 60)) + client._request("GET", "/first") + client._request("GET", "/second") + assert sent == [1060.0, 1120.0] + store.close() + + +def test_slow_budget_admission_reanchors_both_calendar_windows(): + clock = Clock(datetime(2026, 10, 4, 23, 59, 59, tzinfo=timezone.utc).timestamp()) + store = Store(clock=clock) + old_day, old_week = window("day", clock()), window("week", clock()) + original = store._reserve_in + delayed = False + def reserve(*args, **kwargs): + nonlocal delayed + result = original(*args, **kwargs) + if not delayed: + delayed = True + clock.sleep(2) + return result + store._reserve_in = reserve + with routed(lambda _: httpx.Response(200, content=b"{}")) as pool: + client = source(store, clock, pool, ip_limit=(10, 60), max_response_bytes=10) + response = client._request("GET", "/data", estimated_credits=1, + credit_limit=100, byte_limit=100) + assert response.provenance.retrieved_at_utc.startswith("2026-10-05") + assert store.used(client.scope, "credits", "day") == 1 + assert store.used(client.scope, "bytes", "week") == 2 + with store.transaction() as db: + rows = db.execute("SELECT window FROM budgets UNION SELECT window FROM budget_reservations").fetchall() + assert all(key not in (old_day, old_week) for (key,) in rows) + store.close() + + +@pytest.mark.parametrize("ip_only", [False, True]) +def test_429_preserves_scope_while_an_independent_ip_can_continue(ip_only): + clock, sent = Clock(), [] + store = Store(clock=clock) + def first(request): + sent.append(("first", clock())) + return httpx.Response(429, headers={"Retry-After": "30"}) + def second(request): + sent.append(("second", clock())) + return httpx.Response(200, json={}) + with routed(first, second) as pool: + client = source(store, clock, pool, ip_limit=(10, 60), + ip_throttle_only=ip_only, max_attempts=1) + with pytest.raises(Throttled): + client._request("GET", "/first") + assert client._request("GET", "/second").provenance.http_status == 200 + assert [label for label, _ in sent] == ["first", "second"] + assert (sent[1][1] - sent[0][1] < 30) == ip_only + store.close() + + +@pytest.mark.parametrize("limit", [None, (101, 1), (100, 2), (True, 1)]) +def test_openalex_provider_ip_policy_cannot_be_disabled_or_expanded(limit): + with pytest.raises(ValueError, match="OpenAlex IP policy"): + OpenAlex("fixture-secret", ip_limit=limit) + + +@pytest.mark.parametrize("use_pool", [False, True]) +def test_positive_admission_deadline_includes_slow_budget_work(use_pool): + clock, sent = Clock(), [] + store = Store(clock=clock) + original = store._reserve_in + delayed = False + def reserve(*args, **kwargs): + nonlocal delayed + if not delayed: + delayed = True + clock.sleep(2) + return original(*args, **kwargs) + store._reserve_in = reserve + handler = lambda _: sent.append(clock()) or httpx.Response(200, json={}) + with routed(handler) as pool: + options = {"ip_pool": pool, "ip_limit": (10, 60)} if use_pool else { + "client": httpx.Client(transport=httpx.MockTransport(handler))} + client = Transport("test", "https://example.org", store=store, + sleep=clock.sleep, max_wait=1, **options) + with pytest.raises(Throttled): + client._request("GET", "/data", estimated_credits=1, credit_limit=100) + assert not sent + with store.transaction() as db: + assert db.execute("SELECT count(*) FROM budget_reservations").fetchone()[0] == 0 + assert db.execute("SELECT count(*) FROM pacing").fetchone()[0] == 0 + if not use_pool: + options["client"].close() + store.close() diff --git a/tests/test_review_finance_macro.py b/tests/test_review_finance_macro.py new file mode 100644 index 0000000..58f3ce6 --- /dev/null +++ b/tests/test_review_finance_macro.py @@ -0,0 +1,135 @@ +"""Regression contracts for complete FRED checkpoints and World Bank selection.""" +from datetime import datetime, timezone + +import httpx +import pytest + +from dedomena.sources.core import InvalidResponse, Store +from dedomena.sources.fred import FRED +from dedomena.sources.worldbank import WorldBank + + +class Clock: + def __init__(self): + self.now = datetime(2026, 10, 2, tzinfo=timezone.utc).timestamp() + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +@pytest.fixture +def resources(): + clock = Clock() + store = Store(clock=clock) + try: + yield store, clock + finally: + store.close() + + +def fred(handler, resources): + store, clock = resources + return FRED("0123456789abcdef0123456789abcdef", store=store, sleep=clock.sleep, + client=httpx.Client(transport=httpx.MockTransport(handler))) + + +def worldbank(handler, resources): + store, clock = resources + return WorldBank(store=store, sleep=clock.sleep, + client=httpx.Client(transport=httpx.MockTransport(handler))) + + +def release(date, *, more=False, cursor=None): + body = {"release": {"release_id": 53}, "has_more": more, + "series": [{"series_id": "GDP", "title": "Gross Domestic Product", + "frequency": "Quarterly", "units": "Billions of Dollars", + "last_updated": "2026-09-01T00:00:00Z", "copyright_id": "public domain", + "observations": [{"date": date, "value": "100.010000000001"}]}]} + if cursor is not None: + body["next_cursor"] = cursor + return httpx.Response(200, json=body) + + +def observation(country="US", iso3="USA"): + return {"indicator": {"id": "NY.GDP.MKTP.CD"}, "country": {"id": country}, + "countryiso3code": iso3, "date": "2024", "value": None} + + +def response(request, rows): + return httpx.Response(200, json=[{"page": 1, "pages": 1, "total": len(rows), + "per_page": int(request.url.params["per_page"])}, rows]) + + +@pytest.mark.parametrize("cursor,records_seen", [ + (None, 1), (None, 10), ("GDP,2024-04-01", 0), ("", 1), + ("GDP,2024-04-01", -1), (None, True), ("GDP,2024-04-01", True), + (None, 1.0), ("GDP,2024-04-01", "1"), +]) +def test_fred_release_requires_both_checkpoint_parts_before_network(resources, cursor, records_seen): + source = fred(lambda request: pytest.fail("invalid checkpoint reached network"), resources) + with pytest.raises(ValueError): + list(source.release_observations(53, cursor=cursor, records_seen=records_seen)) + + +def test_fred_release_two_page_resume_delivers_each_observation_once(resources): + cursors = [] + def handler(request): + cursor = request.url.params.get("next_cursor") + cursors.append(cursor) + if cursor is None: + return release("2024-01-01", more=True, cursor="GDP,2024-04-01") + assert cursor == "GDP,2024-04-01" + return release("2024-04-01") + source = fred(handler, resources) + first_iterator = source.release_observations(53, page_size=1) + first = next(first_iterator) + first_iterator.close() + remaining = list(source.release_observations(53, page_size=1, + cursor=first.next_cursor, + records_seen=first.records_seen)) + assert not first.complete and first.records_seen == 1 + assert len(remaining) == 1 and remaining[0].complete + assert remaining[0].records_seen == 2 and remaining[0].next_cursor is None + dates = [item["date"] for page in [first, *remaining] for record in page.records + for item in record["observations"]] + assert dates == ["2024-01-01", "2024-04-01"] + assert cursors == [None, "GDP,2024-04-01"] + assert source.usage()["observations_delivered"] == 2 + + +@pytest.mark.parametrize("countries", ["all", "ALL", "All", ["aLl"]]) +def test_worldbank_standalone_all_is_canonical_and_accepts_all_countries(resources, countries): + def handler(request): + assert request.url.path == "/v2/country/all/indicator/NY.GDP.MKTP.CD" + return response(request, [observation(), observation("FR", "FRA")]) + page = next(worldbank(handler, resources).search("NY.GDP.MKTP.CD", countries=countries)) + assert page.complete and len(page.records) == 2 + assert "/country/all/" in page.provenance.url + + +@pytest.mark.parametrize("countries", [ + "all;FIN", "ALL;FIN", "Fin;All", ["All", "USA"], + ["fin", "aLl"], ["ALL", "all"], +]) +def test_worldbank_mixed_all_rejected_before_network(resources, countries): + source = worldbank(lambda request: pytest.fail("mixed all selector reached network"), resources) + with pytest.raises(ValueError, match="all cannot be combined"): + list(source.search("NY.GDP.MKTP.CD", countries=countries)) + + +@pytest.mark.parametrize("countries", ["USA", "usa;fin", ["uSa", "fiN"], "US;FI"]) +def test_worldbank_explicit_countries_never_admit_foreign_country(resources, countries): + source = worldbank(lambda request: response(request, [observation("FR", "FRA")]), resources) + with pytest.raises(InvalidResponse, match="unrequested country"): + next(source.search("NY.GDP.MKTP.CD", countries=countries)) + + +def test_worldbank_explicit_country_selection_preserves_case_and_native_id_aliases(resources): + def handler(request): + assert request.url.path == "/v2/country/uSa;fiN/indicator/NY.GDP.MKTP.CD" + return response(request, [observation(), observation("FI", "FIN")]) + page = next(worldbank(handler, resources).search("NY.GDP.MKTP.CD", countries=["uSa", "fiN"])) + assert page.complete and len(page.records) == 2