From a9f04d01909483a9b98d542c95b424165f4eae7c Mon Sep 17 00:00:00 2001 From: aiapienthusiast <327629069+aiapienthusiast@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:18:32 +0400 Subject: [PATCH] feat(eval): add optional Cheaper Inference reader --- eval/README.md | 22 +++++++ eval/lib/model_config.py | 17 +++++ eval/run_bench.py | 38 +++++++++-- tests/test_cheaperinference_client.py | 95 +++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 tests/test_cheaperinference_client.py diff --git a/eval/README.md b/eval/README.md index bfede20..69ae8ac 100644 --- a/eval/README.md +++ b/eval/README.md @@ -194,6 +194,28 @@ with another provider flag or `--api-base`. Atlas requests are not automatically retried, including timeouts, connection errors and rate limits, to avoid duplicating billable generation requests. +## Cheaper Inference readers + +[Cheaper Inference](https://cheaperinference.com) is an OpenAI-compatible LLM gateway. +Each model costs 15–60% less than the list price of its lab. + +Select it explicitly with `--cheaperinference`; other readers are unchanged. +Set `CHEAPER_INFERENCE_API_KEY` and use a bare model ID from the +[model list](https://cheaperinference.com/#models): + +```bash +export CHEAPER_INFERENCE_API_KEY=ci_live_... +python run_bench.py --task simpleqa --model gpt-5.4-mini \ + --cheaperinference --num-examples 1 --max-concurrent 1 --max-tokens 256 +``` + +This text-only example does not use retrieval. For screenshot benchmarks, use +`gpt-5.4-mini` or `gpt-5.4`, which accept image input. Model IDs are forwarded +unchanged to `https://api.cheaperinference.com/v1` and never select the native +Google SDK. `--api-key` overrides `CHEAPER_INFERENCE_API_KEY`, and other providers' +environment keys are never used. Do not combine this option with another provider +flag or `--api-base`. + ## MiniMax readers Set `MINIMAX_API_KEY` and select either registered model ID. The model context length diff --git a/eval/lib/model_config.py b/eval/lib/model_config.py index 6fe6c3d..0b89844 100644 --- a/eval/lib/model_config.py +++ b/eval/lib/model_config.py @@ -68,6 +68,23 @@ def get_atlascloud_config( } +def get_cheaperinference_config( + model_name: str, api_key: str | None = None +) -> Dict[str, Any]: + """Resolve an explicitly selected Cheaper Inference reader without native routing.""" + if not api_key or api_key == "dummy": + api_key = os.getenv("CHEAPER_INFERENCE_API_KEY") + if not api_key or not api_key.strip() or api_key == "dummy": + raise ValueError( + "Cheaper Inference requires --api-key or CHEAPER_INFERENCE_API_KEY." + ) + return { + "api_base": "https://api.cheaperinference.com/v1", + "api_key": api_key, + "model": model_name, + } + + def get_model_config(model_name: str) -> Dict[str, Any]: """ Get model configuration based on model name. diff --git a/eval/run_bench.py b/eval/run_bench.py index 05361db..13fbdc1 100644 --- a/eval/run_bench.py +++ b/eval/run_bench.py @@ -74,6 +74,7 @@ from lib.model_config import ( ORCAROUTER_API_BASE, get_atlascloud_config, + get_cheaperinference_config, get_model_config, get_output_filename, ) @@ -958,11 +959,12 @@ async def run_async(args): ) # Get model configuration - model_config = ( - get_atlascloud_config(args.model, args.api_key) - if args.atlascloud - else get_model_config(args.model) - ) + if args.atlascloud: + model_config = get_atlascloud_config(args.model, args.api_key) + elif args.cheaperinference: + model_config = get_cheaperinference_config(args.model, args.api_key) + else: + model_config = get_model_config(args.model) # Handle OpenRouter API if args.atlascloud: @@ -970,6 +972,11 @@ async def run_async(args): api_key = model_config["api_key"] model = model_config["model"] logger.info(f"Using Atlas Cloud API with model: {model}") + elif args.cheaperinference: + api_base = model_config["api_base"] + api_key = model_config["api_key"] + model = model_config["model"] + logger.info(f"Using Cheaper Inference API with model: {model}") elif args.open_router: api_base = "https://openrouter.ai/api/v1" if args.api_key and args.api_key != "dummy": @@ -1144,7 +1151,11 @@ async def run_async(args): timeout=args.timeout, enable_thinking=(False if args.no_think else None), force_openai_compat=( - args.open_router or args.commonstack or args.orcarouter or args.atlascloud + args.open_router + or args.commonstack + or args.orcarouter + or args.atlascloud + or args.cheaperinference ), use_litellm=args.litellm, retry_requests=not args.atlascloud, @@ -1374,6 +1385,11 @@ def main(): action="store_true", help="Use Atlas Cloud with an exact catalog model ID. Requires --api-key or ATLASCLOUD_API_KEY.", ) + parser.add_argument( + "--cheaperinference", + action="store_true", + help="Use Cheaper Inference (https://api.cheaperinference.com/v1) with a bare model ID. Requires --api-key or CHEAPER_INFERENCE_API_KEY.", + ) parser.add_argument( "--litellm", action="store_true", @@ -1932,6 +1948,16 @@ def main(): parser.error("--atlascloud cannot be combined with another provider flag") if args.atlascloud and args.api_base: parser.error("--atlascloud uses its fixed endpoint; omit --api-base") + if args.cheaperinference and ( + args.open_router + or args.commonstack + or args.orcarouter + or args.atlascloud + or args.litellm + ): + parser.error("--cheaperinference cannot be combined with another provider flag") + if args.cheaperinference and args.api_base: + parser.error("--cheaperinference uses its fixed endpoint; omit --api-base") # Validate mutually exclusive options mode_count = sum( diff --git a/tests/test_cheaperinference_client.py b/tests/test_cheaperinference_client.py new file mode 100644 index 0000000..a101ad2 --- /dev/null +++ b/tests/test_cheaperinference_client.py @@ -0,0 +1,95 @@ +"""Offline coverage of the optional Cheaper Inference evaluation reader.""" + +import asyncio +import sys +import types +from pathlib import Path +from runpy import run_path + +import pytest + +_LIB = Path(__file__).parents[1] / "eval" / "lib" +if "lib" not in sys.modules: + _pkg = types.ModuleType("lib") + _pkg.__path__ = [str(_LIB)] + sys.modules["lib"] = _pkg + +from lib.llm import LLMClient + +get_config = run_path(str(_LIB / "model_config.py"))["get_cheaperinference_config"] + + +@pytest.mark.parametrize("model", ["gpt-5.4-mini", "gemini-3.1-pro"]) +def test_cheaperinference_preserves_model_and_isolates_credentials(monkeypatch, model): + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", "ci-test-key") + monkeypatch.setenv("OPENAI_API_KEY", "other-key") + monkeypatch.setenv("API_KEY", "generic-key") + config = get_config(model) + assert config == { + "model": model, + "api_base": "https://api.cheaperinference.com/v1", + "api_key": "ci-test-key", + } + assert get_config(model, "explicit-key")["api_key"] == "explicit-key" + + +@pytest.mark.parametrize("key", [None, "", " ", "dummy"]) +def test_cheaperinference_requires_own_key(monkeypatch, key): + monkeypatch.delenv("CHEAPER_INFERENCE_API_KEY", raising=False) + if key is not None: + monkeypatch.setenv("CHEAPER_INFERENCE_API_KEY", key) + monkeypatch.setenv("API_KEY", "generic-key") + monkeypatch.setenv("OPENAI_API_KEY", "openai-key") + with pytest.raises(ValueError, match="CHEAPER_INFERENCE_API_KEY"): + get_config("gpt-5.4-mini") + + +@pytest.mark.parametrize("model", ["gpt-5.4-mini", "gemini-3.1-pro"]) +def test_cheaperinference_client_forwards_image_messages(monkeypatch, model): + calls = [] + options = {} + + async def create(**kwargs): + calls.append(kwargs) + return types.SimpleNamespace( + choices=[types.SimpleNamespace(message=types.SimpleNamespace(content="4"))], + usage=types.SimpleNamespace( + prompt_tokens=3, completion_tokens=1, total_tokens=4 + ), + ) + + def openai_client(**kwargs): + options.update(kwargs) + return types.SimpleNamespace( + chat=types.SimpleNamespace(completions=types.SimpleNamespace(create=create)) + ) + + monkeypatch.setitem( + sys.modules, "openai", types.SimpleNamespace(AsyncOpenAI=openai_client) + ) + client = LLMClient( + **get_config(model, "ci-test-key"), + force_openai_compat=True, + max_tokens=64, + ) + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,AAAA"}, + }, + {"type": "text", "text": "2+2?"}, + ], + } + ] + text, usage = asyncio.run(client.generate(messages)) + assert text == "4" + assert usage["total_tokens"] == 4 + assert len(calls) == 1 + assert calls[0]["model"] == model + assert calls[0]["messages"] == messages + assert options["base_url"] == "https://api.cheaperinference.com/v1" + assert options["api_key"] == "ci-test-key" + assert not client.is_gemini