diff --git a/docs/en/user_guides/models.md b/docs/en/user_guides/models.md index b2f120e97..fc6084448 100644 --- a/docs/en/user_guides/models.md +++ b/docs/en/user_guides/models.md @@ -71,6 +71,7 @@ Currently, OpenCompass supports API-based model inference for the following: - OpenAI (`opencompass.models.OpenAI`) - LiteLLM - unified gateway to 100+ providers (`opencompass.models.LiteLLMAPI`) +- OrcaRouter - OpenAI-compatible AI gateway for models and agents (`opencompass.models.OrcaRouterAPI`) - ChatGLM (`opencompass.models.ZhiPuAI`) - ABAB-Chat from MiniMax (`opencompass.models.MiniMax`) - XunFei from XunFei (`opencompass.models.XunFei`) diff --git a/examples/eval_orcarouter.py b/examples/eval_orcarouter.py new file mode 100644 index 000000000..59b7cb58f --- /dev/null +++ b/examples/eval_orcarouter.py @@ -0,0 +1,35 @@ +# Evaluate models through the OrcaRouter AI gateway. +# +# OrcaRouter is an OpenAI-compatible AI gateway for models and agents. It +# exposes a provider/model namespace (e.g. `orcarouter/free`, +# `orcarouter/fusion`) alongside many third-party models, with adaptive +# routing, automatic failover, zero-markup inference, observability, +# guardrails, and agent-tool governance on the same endpoint. +# +# Before running, set your key: +# export ORCAROUTER_API_KEY=your_key +# +# The full model list is available at: +# https://api.orcarouter.ai/v1/models +# and at https://www.orcarouter.ai + +from opencompass.models import OrcaRouterAPI + +api_meta_template = dict(round=[ + dict(role='HUMAN', api_role='HUMAN'), + dict(role='BOT', api_role='BOT', generate=True), +], ) + +models = [ + dict( + abbr='OrcaRouter-Free', + type=OrcaRouterAPI, + path='orcarouter/free', # Any model id exposed by the gateway + key='ENV', # Read from $ORCAROUTER_API_KEY + meta_template=api_meta_template, + query_per_second=1, + max_out_len=512, + max_seq_len=16384, + batch_size=1, + ), +] diff --git a/opencompass/models/__init__.py b/opencompass/models/__init__.py index 5d7348955..6d81c3c8d 100644 --- a/opencompass/models/__init__.py +++ b/opencompass/models/__init__.py @@ -39,6 +39,8 @@ from .openai_api import OpenAISDKRollout # noqa: F401 from .openai_response import OpenAISDKResponse # noqa: F401 from .openai_streaming import OpenAISDKStreaming # noqa: F401 +from .orcarouter_api import OrcaRouterAPI # noqa: F401 +from .orcarouter_api import OrcaRouterAPIStreaming # noqa: F401 from .pangu_api import PanGu # noqa: F401 from .qwen_api import Qwen # noqa: F401 from .rendu_api import Rendu # noqa: F401 diff --git a/opencompass/models/orcarouter_api.py b/opencompass/models/orcarouter_api.py new file mode 100644 index 000000000..4377df14d --- /dev/null +++ b/opencompass/models/orcarouter_api.py @@ -0,0 +1,187 @@ +"""OrcaRouter AI gateway backend for OpenCompass. + +OrcaRouter is an OpenAI-compatible AI gateway built for both models and +agents. Like OpenRouter, it exposes a provider/model namespace across many +models, but it also combines adaptive routing, automatic failover, +zero-markup inference, observability, guardrails, and agent-tool governance +behind the same endpoint. It runs gateway-level, zero-trust security for AI +agents on the same endpoint, screening every prompt/response and governing +every tool call on a default-deny basis, with no application code changes. + +This wrapper makes ``orcarouter/*`` and third-party models available through +the OrcaRouter gateway. Target models use the OrcaRouter namespace convention, +e.g. ``orcarouter/fusion``, ``orcarouter/free``, or any model id exposed by +``GET https://api.orcarouter.ai/v1/models``. + +See https://www.orcarouter.ai for the full model list and API reference. +""" + +import os +from typing import Dict, List, Optional, Union + +from opencompass.registry import MODELS + +from .openai_api import OpenAISDK +from .openai_streaming import OpenAISDKStreaming + +#: Default OrcaRouter gateway base URL (OpenAI-compatible ``/v1`` endpoint). +ORCAROUTER_API_BASE = os.environ.get('ORCAROUTER_API_BASE', + 'https://api.orcarouter.ai/v1/') + + +@MODELS.register_module() +class OrcaRouterAPI(OpenAISDK): + """Model wrapper around the OrcaRouter AI gateway. + + ``OrcaRouterAPI`` mirrors the existing ``OpenAISDK`` integration and + points it at the OrcaRouter gateway, so all ``OpenAISDK`` arguments + (``mode``, ``meta_template``, ``extra_body``, ``openai_extra_kwargs``, + ...) apply unchanged. + + Args: + path (str): Model id exposed by the OrcaRouter gateway, e.g. + ``orcarouter/fusion``. Defaults to ``orcarouter/free``. + key (str): OrcaRouter API key. When ``'ENV'`` (default), the key is + read from the ``ORCAROUTER_API_KEY`` environment variable. + openai_api_base (str | List[str]): The gateway base URL. Defaults to + ``ORCAROUTER_API_BASE``. + All other args are inherited from ``OpenAISDK``. + """ + + def __init__( + self, + path: str = 'orcarouter/free', + max_seq_len: int = 16384, + query_per_second: int = 1, + rpm_verbose: bool = False, + retry: int = 2, + key: str = 'ENV', + org: Union[str, List[str], None] = None, + meta_template: Optional[Dict] = None, + openai_api_base: Union[str, List[str]] = ORCAROUTER_API_BASE, + openai_proxy_url: Optional[str] = None, + mode: str = 'none', + logprobs: Optional[bool] = False, + top_logprobs: Optional[int] = None, + temperature: Optional[float] = None, + tokenizer_path: Optional[str] = None, + extra_body: Optional[Dict] = None, + verbose: bool = False, + http_client_cfg: dict = {}, + status_code_mappings: dict = {}, + think_tag: str = '', + max_workers: Optional[int] = None, + openai_extra_kwargs: Optional[Dict] = None, + timeout: int = 3600, + image_format: Optional[str] = None, + image_min_edge: Optional[int] = None, + ): + if key == 'ENV': + if 'ORCAROUTER_API_KEY' not in os.environ: + raise ValueError('OrcaRouter API key is not set.') + key = os.environ.get('ORCAROUTER_API_KEY') + super().__init__( + path=path, + max_seq_len=max_seq_len, + query_per_second=query_per_second, + rpm_verbose=rpm_verbose, + retry=retry, + key=key, + org=org, + meta_template=meta_template, + openai_api_base=openai_api_base, + openai_proxy_url=openai_proxy_url, + mode=mode, + logprobs=logprobs, + top_logprobs=top_logprobs, + temperature=temperature, + tokenizer_path=tokenizer_path, + extra_body=extra_body, + verbose=verbose, + http_client_cfg=http_client_cfg, + status_code_mappings=status_code_mappings, + think_tag=think_tag, + max_workers=max_workers, + openai_extra_kwargs=openai_extra_kwargs, + timeout=timeout, + image_format=image_format, + image_min_edge=image_min_edge, + ) + + +@MODELS.register_module() +class OrcaRouterAPIStreaming(OpenAISDKStreaming): + """Streaming variant of :class:`OrcaRouterAPI`. + + Wraps :class:`OpenAISDKStreaming` against the OrcaRouter gateway. All + arguments are inherited from :class:`OpenAISDKStreaming`, except + ``path`` (defaults to ``orcarouter/free``), ``key`` (defaults to + ``ORCAROUTER_API_KEY`` env var), and ``openai_api_base`` (defaults to + ``ORCAROUTER_API_BASE``). + """ + + def __init__( + self, + path: str = 'orcarouter/free', + max_seq_len: int = 16384, + query_per_second: int = 1, + rpm_verbose: bool = False, + retry: int = 2, + key: str = 'ENV', + org: Union[str, List[str], None] = None, + meta_template: Optional[Dict] = None, + openai_api_base: Union[str, List[str]] = ORCAROUTER_API_BASE, + openai_proxy_url: Optional[str] = None, + mode: str = 'none', + logprobs: Optional[bool] = False, + top_logprobs: Optional[int] = None, + temperature: Optional[float] = None, + tokenizer_path: Optional[str] = None, + extra_body: Optional[Dict] = None, + verbose: bool = False, + http_client_cfg: dict = {}, + status_code_mappings: dict = {}, + think_tag: str = '', + openai_extra_kwargs: Optional[Dict] = None, + stream: bool = True, + stream_chunk_size: int = 1, + timeout: int = 3600, + finish_reason_confirm: bool = True, + max_workers: Optional[int] = None, + image_format: Optional[str] = None, + image_min_edge: Optional[int] = None, + ): + if key == 'ENV': + if 'ORCAROUTER_API_KEY' not in os.environ: + raise ValueError('OrcaRouter API key is not set.') + key = os.environ.get('ORCAROUTER_API_KEY') + super().__init__( + path=path, + max_seq_len=max_seq_len, + query_per_second=query_per_second, + rpm_verbose=rpm_verbose, + retry=retry, + key=key, + org=org, + meta_template=meta_template, + openai_api_base=openai_api_base, + openai_proxy_url=openai_proxy_url, + mode=mode, + logprobs=logprobs, + top_logprobs=top_logprobs, + temperature=temperature, + tokenizer_path=tokenizer_path, + extra_body=extra_body, + verbose=verbose, + http_client_cfg=http_client_cfg, + status_code_mappings=status_code_mappings, + think_tag=think_tag, + openai_extra_kwargs=openai_extra_kwargs, + stream=stream, + stream_chunk_size=stream_chunk_size, + timeout=timeout, + finish_reason_confirm=finish_reason_confirm, + max_workers=max_workers, + image_format=image_format, + image_min_edge=image_min_edge, + ) diff --git a/tests/models/test_orcarouter_api.py b/tests/models/test_orcarouter_api.py new file mode 100644 index 000000000..84eb76d14 --- /dev/null +++ b/tests/models/test_orcarouter_api.py @@ -0,0 +1,226 @@ +"""Unit tests for OrcaRouterAPI and OrcaRouterAPIStreaming.""" + +import unittest +from unittest.mock import MagicMock, patch + +from opencompass.models.orcarouter_api import (ORCAROUTER_API_BASE, + OrcaRouterAPI, + OrcaRouterAPIStreaming) + + +def _setup_tiktoken(mock_tiktoken): + """Install a tiktoken stub that resolves any requested encoding.""" + mock_enc = MagicMock() + mock_enc.encode = MagicMock(return_value=[1, 2, 3]) + mock_tiktoken.encoding_for_model = MagicMock(return_value=mock_enc) + mock_tiktoken.model = MagicMock() + mock_tiktoken.model.MODEL_TO_ENCODING = {} + return mock_enc + + +class TestOrcaRouterAPI(unittest.TestCase): + """Initialization and key/base handling.""" + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {'ORCAROUTER_API_KEY': 'sk-orca'}) + def test_registers_in_models_registry(self, mock_httpx_client, + mock_openai_class, mock_tiktoken): + from opencompass.registry import MODELS + + _setup_tiktoken(mock_tiktoken) + mock_openai_class.return_value = MagicMock() + mock_httpx_client.return_value = MagicMock() + self.assertIs(MODELS.get('OrcaRouterAPI'), OrcaRouterAPI) + self.assertIs(MODELS.get('OrcaRouterAPIStreaming'), + OrcaRouterAPIStreaming) + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {'ORCAROUTER_API_KEY': 'sk-orca'}) + def test_default_init_uses_orcarouter_defaults(self, mock_httpx_client, + mock_openai_class, + mock_tiktoken): + _setup_tiktoken(mock_tiktoken) + mock_openai_class.return_value = MagicMock() + mock_httpx_client.return_value = MagicMock() + + model = OrcaRouterAPI() + + self.assertEqual(model.path, 'orcarouter/free') + self.assertEqual(model.openai_api_base, ORCAROUTER_API_BASE) + self.assertEqual(model.keys, ['sk-orca']) + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {'ORCAROUTER_API_KEY': 'sk-orca'}) + def test_env_key_reads_orcarouter_var(self, mock_httpx_client, + mock_openai_class, mock_tiktoken): + _setup_tiktoken(mock_tiktoken) + mock_openai_class.return_value = MagicMock() + mock_httpx_client.return_value = MagicMock() + + model = OrcaRouterAPI(key='ENV') + + self.assertEqual(model.keys, ['sk-orca']) + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {}, clear=True) + def test_env_key_raises_when_missing(self, mock_httpx_client, + mock_openai_class, mock_tiktoken): + _setup_tiktoken(mock_tiktoken) + mock_openai_class.return_value = MagicMock() + mock_httpx_client.return_value = MagicMock() + + with self.assertRaisesRegex(ValueError, 'OrcaRouter API key'): + OrcaRouterAPI(key='ENV') + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {}, clear=True) + def test_explicit_key_skips_env(self, mock_httpx_client, mock_openai_class, + mock_tiktoken): + _setup_tiktoken(mock_tiktoken) + mock_openai_class.return_value = MagicMock() + mock_httpx_client.return_value = MagicMock() + + model = OrcaRouterAPI(key='sk-explicit') + + self.assertEqual(model.keys, ['sk-explicit']) + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {'ORCAROUTER_API_KEY': 'sk-orca'}) + def test_custom_api_base_is_respected(self, mock_httpx_client, + mock_openai_class, mock_tiktoken): + _setup_tiktoken(mock_tiktoken) + mock_openai_class.return_value = MagicMock() + mock_httpx_client.return_value = MagicMock() + + model = OrcaRouterAPI(openai_api_base='https://self-hosted/v1/') + + self.assertEqual(model.openai_api_base, 'https://self-hosted/v1/') + + +class TestOrcaRouterAPIGenerate(unittest.TestCase): + """End-to-end generate with a mocked OpenAI client.""" + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {'ORCAROUTER_API_KEY': 'sk-orca'}) + def test_generate_single(self, mock_httpx_client, mock_openai_class, + mock_tiktoken): + _setup_tiktoken(mock_tiktoken) + + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[0].message.content = 'pong' + mock_response.choices[0].message.reasoning_content = None + mock_response.choices[0].finish_reason = 'stop' + mock_client.chat.completions.create.return_value = mock_response + mock_openai_class.return_value = mock_client + mock_httpx_client.return_value = MagicMock() + + model = OrcaRouterAPI(path='orcarouter/free') + results = model.generate(['ping'], max_out_len=16) + + self.assertEqual(results, ['pong']) + call_kwargs = mock_client.chat.completions.create.call_args[1] + self.assertEqual(call_kwargs['model'], 'orcarouter/free') + + # The OpenAI SDK client is constructed with the gateway base URL + # and the resolved key. + client_kwargs = mock_openai_class.call_args[1] + self.assertEqual(client_kwargs['base_url'], ORCAROUTER_API_BASE) + self.assertEqual(client_kwargs['api_key'], 'sk-orca') + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {'ORCAROUTER_API_KEY': 'sk-orca'}) + def test_generate_merges_reasoning_content(self, mock_httpx_client, + mock_openai_class, + mock_tiktoken): + _setup_tiktoken(mock_tiktoken) + + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[0].message.content = 'Final' + mock_response.choices[0].message.reasoning_content = 'Thinking' + mock_response.choices[0].finish_reason = 'stop' + mock_client.chat.completions.create.return_value = mock_response + mock_openai_class.return_value = mock_client + mock_httpx_client.return_value = MagicMock() + + model = OrcaRouterAPI(think_tag='') + results = model.generate(['q'], max_out_len=16) + + self.assertEqual(results, ['ThinkingFinal']) + + +class TestOrcaRouterAPIStreaming(unittest.TestCase): + """Streaming variant parses chunks back into a full response.""" + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {'ORCAROUTER_API_KEY': 'sk-orca'}) + def test_stream_defaults(self, mock_httpx_client, mock_openai_class, + mock_tiktoken): + _setup_tiktoken(mock_tiktoken) + mock_openai_class.return_value = MagicMock() + mock_httpx_client.return_value = MagicMock() + + model = OrcaRouterAPIStreaming() + + self.assertEqual(model.path, 'orcarouter/free') + self.assertEqual(model.openai_api_base, ORCAROUTER_API_BASE) + self.assertTrue(model.stream) + self.assertEqual(model.stream_chunk_size, 1) + + @patch('opencompass.models.openai_api.tiktoken', create=True) + @patch('openai.OpenAI') + @patch('httpx.Client') + @patch.dict('os.environ', {'ORCAROUTER_API_KEY': 'sk-orca'}) + def test_stream_generate(self, mock_httpx_client, mock_openai_class, + mock_tiktoken): + _setup_tiktoken(mock_tiktoken) + + chunk1 = MagicMock() + chunk1.choices = [MagicMock()] + chunk1.choices[0].delta.content = 'Hel' + chunk1.choices[0].delta.reasoning_content = None + chunk1.choices[0].finish_reason = None + chunk2 = MagicMock() + chunk2.choices = [MagicMock()] + chunk2.choices[0].delta.content = 'lo' + chunk2.choices[0].delta.reasoning_content = None + chunk2.choices[0].finish_reason = 'stop' + + mock_client = MagicMock() + mock_client.chat.completions.create.return_value = iter( + [chunk1, chunk2]) + mock_openai_class.return_value = mock_client + mock_httpx_client.return_value = MagicMock() + + model = OrcaRouterAPIStreaming() + results = model.generate(['hi'], max_out_len=16) + + self.assertEqual(results, ['Hello']) + call_kwargs = mock_client.chat.completions.create.call_args[1] + self.assertTrue(call_kwargs['stream']) + self.assertEqual(call_kwargs['model'], 'orcarouter/free') + + +if __name__ == '__main__': + unittest.main()