diff --git a/eval/README.md b/eval/README.md index dcf1d08..570f143 100644 --- a/eval/README.md +++ b/eval/README.md @@ -234,6 +234,33 @@ Google SDK. `--api-key` overrides `CHEAPER_INFERENCE_API_KEY`, and other provide environment keys are never used. Do not combine this option with another provider flag or `--api-base`. +## API Route readers + +[API Route](https://api-route.com) provides an OpenAI-compatible endpoint. +Select it explicitly with `--api-route` and an exact chat-completions model ID +available to your account (listed by authenticated `GET +https://global.api-route.com/v1/models`). Existing reader defaults are unchanged. + +```bash +export API_ROUTE_API_KEY=your_api_route_key +export API_ROUTE_MODEL=your_chat_completions_model_id +python run_bench.py --task simpleqa --model "$API_ROUTE_MODEL" \ + --api-route --num-examples 1 --max-concurrent 1 --max-tokens 256 +``` + +Set `API_ROUTE_MODEL` to your chosen model ID before running this text-only, +no-retrieval example. For screenshot benchmarks, select a model whose image +input support you have verified; a model-list entry alone does not establish +that support. Model IDs are forwarded unchanged to +`https://global.api-route.com/v1` through the OpenAI-compatible client, including +IDs containing `gemini` (which do not select the native Google SDK). + +`--api-key` overrides `API_ROUTE_API_KEY`; other providers' environment keys are +never used. Do not combine `--api-route` with another provider flag, `--litellm`, +or `--api-base`. API Route requests are not automatically retried, including +timeouts, connection errors and rate limits, to avoid duplicating billable +generation requests. + ## MiniMax readers Set `MINIMAX_API_KEY` and select either registered model ID. The model context length diff --git a/eval/lib/model_config.py b/eval/lib/model_config.py index 0b89844..6979ba5 100644 --- a/eval/lib/model_config.py +++ b/eval/lib/model_config.py @@ -85,6 +85,19 @@ def get_cheaperinference_config( } +def get_api_route_config(model_name: str, api_key: str | None = None) -> Dict[str, Any]: + """Resolve an explicitly selected API Route reader with its own credentials.""" + if not api_key or api_key == "dummy": + api_key = os.getenv("API_ROUTE_API_KEY") + if not api_key or not api_key.strip() or api_key == "dummy": + raise ValueError("API Route requires --api-key or API_ROUTE_API_KEY.") + return { + "api_base": "https://global.api-route.com/v1", + "api_key": api_key, + "model": model_name, + } + + def get_model_config(model_name: str) -> Dict[str, Any]: """ Get model configuration based on model name. diff --git a/eval/run_bench.py b/eval/run_bench.py index ec3dd25..e90eaa0 100644 --- a/eval/run_bench.py +++ b/eval/run_bench.py @@ -73,6 +73,7 @@ ) from lib.model_config import ( ORCAROUTER_API_BASE, + get_api_route_config, get_atlascloud_config, get_cheaperinference_config, get_model_config, @@ -963,6 +964,8 @@ async def run_async(args): model_config = get_atlascloud_config(args.model, args.api_key) elif args.cheaperinference: model_config = get_cheaperinference_config(args.model, args.api_key) + elif args.api_route: + model_config = get_api_route_config(args.model, args.api_key) else: model_config = get_model_config(args.model) @@ -977,6 +980,11 @@ async def run_async(args): api_key = model_config["api_key"] model = model_config["model"] logger.info(f"Using Cheaper Inference API with model: {model}") + elif args.api_route: + api_base = model_config["api_base"] + api_key = model_config["api_key"] + model = model_config["model"] + logger.info(f"Using API Route API with model: {model}") elif args.open_router: api_base = "https://openrouter.ai/api/v1" if args.api_key and args.api_key != "dummy": @@ -1162,9 +1170,10 @@ async def run_async(args): or args.orcarouter or args.atlascloud or args.cheaperinference + or args.api_route ), use_litellm=args.litellm, - retry_requests=not args.atlascloud, + retry_requests=not (args.atlascloud or args.api_route), ) # 3b. Create pixel-compressed encoder for generation if requested @@ -1404,6 +1413,11 @@ def main(): action="store_true", help="Use Cheaper Inference (https://api.cheaperinference.com/v1) with a bare model ID. Requires --api-key or CHEAPER_INFERENCE_API_KEY.", ) + parser.add_argument( + "--api-route", + action="store_true", + help="Use API Route with an exact catalog model ID. Requires --api-key or API_ROUTE_API_KEY.", + ) parser.add_argument( "--litellm", action="store_true", @@ -1973,6 +1987,18 @@ def main(): if args.cheaperinference and args.api_base: parser.error("--cheaperinference uses its fixed endpoint; omit --api-base") + if args.api_route and ( + args.open_router + or args.commonstack + or args.orcarouter + or args.atlascloud + or args.cheaperinference + or args.litellm + ): + parser.error("--api-route cannot be combined with another provider flag") + if args.api_route and args.api_base: + parser.error("--api-route uses its fixed endpoint; omit --api-base") + # Validate mutually exclusive options mode_count = sum( [ diff --git a/tests/test_api_route_client.py b/tests/test_api_route_client.py new file mode 100644 index 0000000..62f1f82 --- /dev/null +++ b/tests/test_api_route_client.py @@ -0,0 +1,112 @@ +"""Offline coverage of the optional API Route evaluation reader.""" + +import asyncio +import sys +import types +from pathlib import Path +from runpy import run_path + +import pytest + +_LIB = Path(__file__).parents[1] / "eval" / "lib" +if "lib" not in sys.modules: + _pkg = types.ModuleType("lib") + _pkg.__path__ = [str(_LIB)] + sys.modules["lib"] = _pkg + +from lib.llm import LLMClient + +get_config = run_path(str(_LIB / "model_config.py"))["get_api_route_config"] + + +@pytest.mark.parametrize("model", ["gpt-5.5", "google/gemini-fixture"]) +def test_api_route_preserves_model_and_isolates_credentials(monkeypatch, model): + monkeypatch.setenv("API_ROUTE_API_KEY", "api-route-test-key") + monkeypatch.setenv("OPENAI_API_KEY", "other-key") + monkeypatch.setenv("API_KEY", "generic-key") + config = get_config(model) + assert config == { + "model": model, + "api_base": "https://global.api-route.com/v1", + "api_key": "api-route-test-key", + } + assert get_config(model, "explicit-key")["api_key"] == "explicit-key" + + +@pytest.mark.parametrize("key", [None, "", " ", "dummy"]) +def test_api_route_requires_own_key(monkeypatch, key): + monkeypatch.delenv("API_ROUTE_API_KEY", raising=False) + if key is not None: + monkeypatch.setenv("API_ROUTE_API_KEY", key) + monkeypatch.setenv("API_KEY", "generic-key") + monkeypatch.setenv("GOOGLE_API_KEY", "google-key") + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + with pytest.raises(ValueError, match="API_ROUTE_API_KEY"): + get_config("deepseek-ai/deepseek-v3.2") + + +@pytest.mark.parametrize( + "error", + [ + None, + asyncio.TimeoutError(), + RuntimeError("Connection error"), + RuntimeError("429"), + ], +) +def test_api_route_client_routes_once(monkeypatch, error): + calls = [] + options = {} + + async def create(**kwargs): + calls.append(kwargs) + if error is not None: + raise error + return types.SimpleNamespace( + choices=[types.SimpleNamespace(message=types.SimpleNamespace(content="4"))], + usage=types.SimpleNamespace( + prompt_tokens=3, completion_tokens=1, total_tokens=4 + ), + ) + + def openai_client(**kwargs): + options.update(kwargs) + return types.SimpleNamespace( + chat=types.SimpleNamespace(completions=types.SimpleNamespace(create=create)) + ) + + monkeypatch.setitem( + sys.modules, "openai", types.SimpleNamespace(AsyncOpenAI=openai_client) + ) + client = LLMClient( + **get_config("google/gemini-fixture", "api-route-test-key"), + force_openai_compat=True, + retry_requests=False, + max_tokens=64, + ) + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,AAAA"}, + }, + {"type": "text", "text": "2+2?"}, + ], + } + ] + if error is None: + text, usage = asyncio.run(client.generate(messages)) + assert text == "4" + assert usage["total_tokens"] == 4 + else: + with pytest.raises(type(error)): + asyncio.run(client.generate(messages)) + assert len(calls) == 1 + assert calls[0]["model"] == "google/gemini-fixture" + assert calls[0]["messages"] == messages + assert options["base_url"] == "https://global.api-route.com/v1" + assert options["api_key"] == "api-route-test-key" + assert options["max_retries"] == 0 + assert not client.is_gemini