diff --git a/backend/.env.example b/backend/.env.example index b772481cb..c2efebc17 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -27,7 +27,7 @@ GOOGLE_APPLICATION_CREDENTIALS=keys/credentials.json # (e.g. us-central1) — these models are not published in the global publisher catalog. VERTEX_API_EMBEDDINGS_REGION=us-central1 # Region for generative-AI calls (Gemini etc.). Can be a regional location or "global". -VERTEX_API_GEN_AI_REGION=us-central1 +VERTEX_API_GEN_AI_REGION=global EMBEDDINGS_SERVICE_NAME=GOOGLE-VERTEX-AI # See https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings#supported-models e.g.: # text-embedding-005 @@ -104,11 +104,13 @@ LLM_PROVIDER=gemini # LLM_REASONING_MODEL — classifiers, routing and extraction that need good reasoning # LLM_DEEP_REASONING_MODEL — the strongest reasoning, e.g. evaluation judges # When a tier is unset: -# gemini: uses its own default (gemini-2.5-flash-lite / gemini-2.5-flash / gemini-2.5-pro) +# gemini: uses its own default (gemini-3.5-flash-lite / gemini-3.8-flash / gemini-3.1-pro-preview) # anthropic: falls back to the next tier down that is set # (LLM_DEEP_REASONING_MODEL -> LLM_REASONING_MODEL -> LLM_DEFAULT_MODEL), then to claude-sonnet-4-6 # ollama: same as anthropic, then to qwen2.5:7b # To use one model for everything with a non-Gemini provider, set only LLM_DEFAULT_MODEL. +# Gemini models are called through the google-genai Interactions API, which only supports +# gemini-3.5-flash-lite, gemini-3.8-flash and gemini-3.1-pro-preview (the 2.5 models are rejected). # LLM_DEFAULT_MODEL= # LLM_REASONING_MODEL= # LLM_DEEP_REASONING_MODEL= diff --git a/backend/README.md b/backend/README.md index df0cc6dee..0604a9444 100644 --- a/backend/README.md +++ b/backend/README.md @@ -567,13 +567,14 @@ For more details, see [app/agent/preference_elicitation_agent/README.md](app/age ### AI & Embeddings - `VERTEX_API_EMBEDDINGS_REGION` - Vertex AI region for embedding models (must be a regional location, e.g. `us-central1`) -- `VERTEX_API_GEN_AI_REGION` - Vertex AI region for generative-AI calls (regional or `global`, default: `us-central1`) +- `VERTEX_API_GEN_AI_REGION` - Vertex AI region for generative-AI calls (regional or `global`, default: `global`). The default and reasoning Gemini 3.x models are only available in `global` - `EMBEDDINGS_SERVICE_NAME` - Embeddings service provider - `EMBEDDINGS_MODEL_NAME` - Model for generating embeddings - `LLM_PROVIDER` - LLM backend for all agents: `gemini` (default), `anthropic` or `ollama` -- `LLM_DEFAULT_MODEL` - Model for the default tier (gemini default: `gemini-2.5-flash-lite`) -- `LLM_REASONING_MODEL` - Model for the reasoning tier (gemini default: `gemini-2.5-flash`) -- `LLM_DEEP_REASONING_MODEL` - Model for the deep reasoning tier (gemini default: `gemini-2.5-pro`) +- `LLM_DEFAULT_MODEL` - Model for the default tier (gemini default: `gemini-3.5-flash-lite`) +- `LLM_REASONING_MODEL` - Model for the reasoning tier (gemini default: `gemini-3.8-flash`) +- `LLM_DEEP_REASONING_MODEL` - Model for the deep reasoning tier (gemini default: `gemini-3.1-pro-preview`) + - Gemini models are called through the google-genai Interactions API, which only supports `gemini-3.5-flash-lite`, `gemini-3.8-flash` and `gemini-3.1-pro-preview` (the 2.5 models are rejected) - For `anthropic` and `ollama`, an unset tier falls back to the next tier down that is set (`LLM_DEEP_REASONING_MODEL` → `LLM_REASONING_MODEL` → `LLM_DEFAULT_MODEL`), then to the provider default (`claude-sonnet-4-6` / `qwen2.5:7b`) - `ANTHROPIC_API_KEY` - Anthropic API key, required when `LLM_PROVIDER=anthropic` - `OLLAMA_BASE_URL` - Ollama server URL when `LLM_PROVIDER=ollama` (default: `http://localhost:11434`) diff --git a/backend/app/agent/config.py b/backend/app/agent/config.py index 19141dc9e..f147b29de 100644 --- a/backend/app/agent/config.py +++ b/backend/app/agent/config.py @@ -2,9 +2,9 @@ from typing import Literal, Final Model = Literal[ - "gemini-2.5-flash-lite", - "gemini-2.5-flash", - "gemini-2.5-pro", + "gemini-3.5-flash-lite", + "gemini-3.8-flash", + "gemini-3.1-pro-preview", ] @@ -39,10 +39,12 @@ class ModelTier(StrEnum): GEMINI_DEFAULT_MODELS: Final[dict[ModelTier, Model]] = { - ModelTier.DEFAULT: "gemini-2.5-flash-lite", - ModelTier.REASONING: "gemini-2.5-flash", - ModelTier.DEEP_REASONING: "gemini-2.5-pro", + ModelTier.DEFAULT: "gemini-3.5-flash-lite", + ModelTier.REASONING: "gemini-3.8-flash", + ModelTier.DEEP_REASONING: "gemini-3.1-pro-preview", } """ The Gemini model used for each tier when the tier's environment variable is not set. +The Gemini 3.x models are only served from the Vertex AI `global` location, see `VERTEX_API_GEN_AI_REGION`. +These are models the google-genai Interactions API supports, it rejects the Gemini 2.5 models. """ diff --git a/backend/app/app_config.py b/backend/app/app_config.py index ddf545fcb..a9d3c79e0 100644 --- a/backend/app/app_config.py +++ b/backend/app/app_config.py @@ -143,7 +143,7 @@ class ApplicationConfig(BaseModel): llm_default_model: Optional[str] = None """ The model for the default tier (ModelTier.DEFAULT) of the selected provider. - When unset, each provider uses its own default (gemini: gemini-2.5-flash-lite, + When unset, each provider uses its own default (gemini: gemini-3.5-flash-lite, anthropic: claude-sonnet-4-6, ollama: qwen2.5:7b). Set via LLM_DEFAULT_MODEL environment variable. """ @@ -151,14 +151,14 @@ class ApplicationConfig(BaseModel): llm_reasoning_model: Optional[str] = None """ The model for the reasoning tier (ModelTier.REASONING) of the selected provider. - When unset, gemini uses gemini-2.5-flash; other providers fall back to llm_default_model. + When unset, gemini uses gemini-3.8-flash; other providers fall back to llm_default_model. Set via LLM_REASONING_MODEL environment variable. """ llm_deep_reasoning_model: Optional[str] = None """ The model for the deep reasoning tier (ModelTier.DEEP_REASONING) of the selected provider. - When unset, gemini uses gemini-2.5-pro; other providers fall back to llm_reasoning_model, then llm_default_model. + When unset, gemini uses gemini-3.1-pro-preview; other providers fall back to llm_reasoning_model, then llm_default_model. Set via LLM_DEEP_REASONING_MODEL environment variable. """ diff --git a/backend/common_libs/llm/models_utils.py b/backend/common_libs/llm/models_utils.py index 6fffa0185..de21182f1 100644 --- a/backend/common_libs/llm/models_utils.py +++ b/backend/common_libs/llm/models_utils.py @@ -24,8 +24,9 @@ DEFAULT_VERTEX_API_GEN_AI_REGION = os.getenv("VERTEX_API_GEN_AI_REGION") if not DEFAULT_VERTEX_API_GEN_AI_REGION: - logging.warning("VERTEX_API_GEN_AI_REGION is not set. Using 'us-central1' as the default region.") - DEFAULT_VERTEX_API_GEN_AI_REGION = "us-central1" + # The Gemini 3.x models are only served from the global location, regional locations return a 404 for them. + logging.warning("VERTEX_API_GEN_AI_REGION is not set. Using 'global' as the default region.") + DEFAULT_VERTEX_API_GEN_AI_REGION = "global" else: logging.info("Default Vertex AI gen-AI region is %s", DEFAULT_VERTEX_API_GEN_AI_REGION) @@ -167,7 +168,7 @@ def resolve_model_name(*, tier: ModelTier, provider: "LLMProvider") -> str: ModelTier.REASONING: app_config.llm_reasoning_model, ModelTier.DEEP_REASONING: app_config.llm_deep_reasoning_model, } - # Gemini falls back to its own 2.5 default for the tier rather than to another tier's model. + # Gemini falls back to its own default for the tier rather than to another tier's model. tiers_to_try = [tier] if provider == "gemini" else [tier, *_NON_GEMINI_FALLBACK_TIERS[tier]] for candidate_tier in tiers_to_try: if configured_models[candidate_tier]: diff --git a/backend/common_libs/llm/test_model_resolution.py b/backend/common_libs/llm/test_model_resolution.py index c4ebf7d64..ac741db3d 100644 --- a/backend/common_libs/llm/test_model_resolution.py +++ b/backend/common_libs/llm/test_model_resolution.py @@ -27,11 +27,11 @@ def _set_llm_config(config: ApplicationConfig, *, class TestResolveModelName: @pytest.mark.parametrize("given_tier, expected_model", [ - (ModelTier.DEFAULT, "gemini-2.5-flash-lite"), - (ModelTier.REASONING, "gemini-2.5-flash"), - (ModelTier.DEEP_REASONING, "gemini-2.5-pro"), + (ModelTier.DEFAULT, "gemini-3.5-flash-lite"), + (ModelTier.REASONING, "gemini-3.8-flash"), + (ModelTier.DEEP_REASONING, "gemini-3.1-pro-preview"), ]) - def test_gemini_uses_gemini_2_5_defaults_when_no_model_is_set(self, setup_application_config, + def test_gemini_uses_gemini_defaults_when_no_model_is_set(self, setup_application_config, given_tier: ModelTier, expected_model: str): # GIVEN the gemini provider is configured without any tier models _set_llm_config(setup_application_config, provider="gemini") @@ -39,7 +39,7 @@ def test_gemini_uses_gemini_2_5_defaults_when_no_model_is_set(self, setup_applic # WHEN the model name for the given tier is resolved actual_model = resolve_model_name(tier=given_tier, provider="gemini") - # THEN expect the gemini 2.5 default for the tier + # THEN expect the gemini default for the tier assert actual_model == expected_model @pytest.mark.parametrize("given_tier, expected_model", [ @@ -63,9 +63,9 @@ def test_uses_the_tier_model_when_it_is_set(self, setup_application_config, assert actual_model == expected_model @pytest.mark.parametrize("given_reasoning_model, given_tier, expected_model", [ - (None, ModelTier.REASONING, "gemini-2.5-flash"), - (None, ModelTier.DEEP_REASONING, "gemini-2.5-pro"), - ("given-reasoning", ModelTier.DEEP_REASONING, "gemini-2.5-pro"), + (None, ModelTier.REASONING, "gemini-3.8-flash"), + (None, ModelTier.DEEP_REASONING, "gemini-3.1-pro-preview"), + ("given-reasoning", ModelTier.DEEP_REASONING, "gemini-3.1-pro-preview"), ]) def test_gemini_does_not_fall_back_to_another_tier(self, setup_application_config, given_reasoning_model: Optional[str], @@ -78,7 +78,7 @@ def test_gemini_does_not_fall_back_to_another_tier(self, setup_application_confi # WHEN the model name for a tier without a model is resolved actual_model = resolve_model_name(tier=given_tier, provider="gemini") - # THEN expect the gemini 2.5 default for the tier, not another tier's model + # THEN expect the gemini default for the tier, not another tier's model assert actual_model == expected_model @pytest.mark.parametrize("given_tier", list(ModelTier)) @@ -144,9 +144,9 @@ def test_non_gemini_default_tier_does_not_fall_back_to_a_higher_tier(self, setup assert actual_model == expected_model @pytest.mark.parametrize("given_tier, expected_model", [ - (ModelTier.DEFAULT, "gemini-2.5-flash-lite"), - (ModelTier.REASONING, "gemini-2.5-flash"), - (ModelTier.DEEP_REASONING, "gemini-2.5-pro"), + (ModelTier.DEFAULT, "gemini-3.5-flash-lite"), + (ModelTier.REASONING, "gemini-3.8-flash"), + (ModelTier.DEEP_REASONING, "gemini-3.1-pro-preview"), ]) def test_ignores_the_models_of_another_provider(self, setup_application_config, given_tier: ModelTier, expected_model: str): @@ -159,13 +159,13 @@ def test_ignores_the_models_of_another_provider(self, setup_application_config, # WHEN the model name for the given tier is resolved for gemini (e.g. a gemini-only call site) actual_model = resolve_model_name(tier=given_tier, provider="gemini") - # THEN expect the gemini 2.5 default for the tier + # THEN expect the gemini default for the tier assert actual_model == expected_model @pytest.mark.parametrize("given_tier, expected_model", [ - (ModelTier.DEFAULT, "gemini-2.5-flash-lite"), - (ModelTier.REASONING, "gemini-2.5-flash"), - (ModelTier.DEEP_REASONING, "gemini-2.5-pro"), + (ModelTier.DEFAULT, "gemini-3.5-flash-lite"), + (ModelTier.REASONING, "gemini-3.8-flash"), + (ModelTier.DEEP_REASONING, "gemini-3.1-pro-preview"), ]) def test_uses_the_defaults_when_the_application_config_is_not_set(self, given_tier: ModelTier, expected_model: str): @@ -175,7 +175,7 @@ def test_uses_the_defaults_when_the_application_config_is_not_set(self, given_ti # WHEN the model name for the given tier is resolved actual_model = resolve_model_name(tier=given_tier, provider="gemini") - # THEN expect the gemini 2.5 default for the tier + # THEN expect the gemini default for the tier assert actual_model == expected_model @@ -214,7 +214,7 @@ def test_explicit_model_name_is_used_as_is(self, setup_application_config): class TestGeminiGenerativeLLM: @pytest.mark.parametrize("given_config, expected_model", [ - (LLMConfig(), "gemini-2.5-flash-lite"), + (LLMConfig(), "gemini-3.5-flash-lite"), (LLMConfig(model_tier=ModelTier.DEEP_REASONING), "given-deep-reasoning"), (LLMConfig(model_tier=ModelTier.DEEP_REASONING, language_model_name="given-explicit-model"), "given-explicit-model"), diff --git a/backend/evaluation_tests/agent_director/llm_agent_director_scripted_user_test.py b/backend/evaluation_tests/agent_director/llm_agent_director_scripted_user_test.py index 371862fa0..acf3547d8 100644 --- a/backend/evaluation_tests/agent_director/llm_agent_director_scripted_user_test.py +++ b/backend/evaluation_tests/agent_director/llm_agent_director_scripted_user_test.py @@ -104,7 +104,7 @@ async def agent_director_exec(caplog: LogCaptureFixture, test_case: ScriptedUser @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) async def test_user_says_all_the_time_yes(caplog: LogCaptureFixture, setup_agent_director: Awaitable[tuple[ConversationMemoryManager, Callable[ @@ -142,7 +142,7 @@ async def test_user_says_all_the_time_yes(caplog: LogCaptureFixture, @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) async def test_user_talks_about_occupations(caplog: LogCaptureFixture, setup_agent_director: Awaitable[tuple[ConversationMemoryManager, Callable[ @@ -206,7 +206,7 @@ class AgentState: @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) async def test_argentina_counseling_flow(caplog: LogCaptureFixture, setup_agent_director: Awaitable[tuple[ConversationMemoryManager, Callable[ diff --git a/backend/evaluation_tests/career_explorer_agent/career_explorer_agent_scripted_user_test.py b/backend/evaluation_tests/career_explorer_agent/career_explorer_agent_scripted_user_test.py index 0c3937760..b0b489b26 100644 --- a/backend/evaluation_tests/career_explorer_agent/career_explorer_agent_scripted_user_test.py +++ b/backend/evaluation_tests/career_explorer_agent/career_explorer_agent_scripted_user_test.py @@ -183,7 +183,7 @@ def career_explorer_config_with_sectors(setup_multi_locale_app_config): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize("test_case", TEST_CASES, ids=[tc.name for tc in TEST_CASES]) async def test_career_explorer_sector_content( evals_setup, setup_multi_locale_app_config, career_explorer_config_with_sectors, test_case: SectorContentTestCase @@ -239,7 +239,7 @@ async def test_career_explorer_sector_content( @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize( "test_case", TEST_CASES_GENERAL_KNOWLEDGE, ids=[tc.name for tc in TEST_CASES_GENERAL_KNOWLEDGE] ) diff --git a/backend/evaluation_tests/career_explorer_agent/non_priority_sector_explorer_eval_test.py b/backend/evaluation_tests/career_explorer_agent/non_priority_sector_explorer_eval_test.py index 4b0779019..2cfeecb35 100644 --- a/backend/evaluation_tests/career_explorer_agent/non_priority_sector_explorer_eval_test.py +++ b/backend/evaluation_tests/career_explorer_agent/non_priority_sector_explorer_eval_test.py @@ -54,7 +54,7 @@ def career_explorer_config_with_sectors(setup_multi_locale_app_config): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize("test_case", NON_PRIORITY_EXPLORER_TEST_CASES, ids=[tc.name for tc in NON_PRIORITY_EXPLORER_TEST_CASES]) async def test_non_priority_sector_explorer_web_search( evals_setup, diff --git a/backend/evaluation_tests/career_explorer_agent/priority_sector_explorer_eval_test.py b/backend/evaluation_tests/career_explorer_agent/priority_sector_explorer_eval_test.py index 18186bff9..a3df1c11a 100644 --- a/backend/evaluation_tests/career_explorer_agent/priority_sector_explorer_eval_test.py +++ b/backend/evaluation_tests/career_explorer_agent/priority_sector_explorer_eval_test.py @@ -104,7 +104,7 @@ def career_explorer_config_with_sectors(setup_multi_locale_app_config): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize("test_case", PRIORITY_EXPLORER_TEST_CASES, ids=[tc.name for tc in PRIORITY_EXPLORER_TEST_CASES]) async def test_priority_sector_explorer_rag_content( evals_setup, diff --git a/backend/evaluation_tests/career_explorer_agent/sector_relevance_classifier_eval_test.py b/backend/evaluation_tests/career_explorer_agent/sector_relevance_classifier_eval_test.py index c4b0bc296..ec6a30312 100644 --- a/backend/evaluation_tests/career_explorer_agent/sector_relevance_classifier_eval_test.py +++ b/backend/evaluation_tests/career_explorer_agent/sector_relevance_classifier_eval_test.py @@ -70,7 +70,7 @@ def career_explorer_config_with_sectors(setup_multi_locale_app_config): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize("test_case", CLASSIFIER_TEST_CASES, ids=[tc.name for tc in CLASSIFIER_TEST_CASES]) async def test_sector_relevance_classifier( evals_setup, @@ -116,7 +116,7 @@ async def test_sector_relevance_classifier( @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") async def test_classifier_extracts_multiple_sectors( evals_setup, career_explorer_config_with_sectors, diff --git a/backend/evaluation_tests/career_readiness_agent/career_readiness_agent_scripted_user_test.py b/backend/evaluation_tests/career_readiness_agent/career_readiness_agent_scripted_user_test.py index b2a274bd9..b575e7ab4 100644 --- a/backend/evaluation_tests/career_readiness_agent/career_readiness_agent_scripted_user_test.py +++ b/backend/evaluation_tests/career_readiness_agent/career_readiness_agent_scripted_user_test.py @@ -246,7 +246,7 @@ def _save_conversation_output( @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize("test_case", TEST_CASES, ids=[tc.name for tc in TEST_CASES]) async def test_career_readiness_scripted(evals_setup, setup_multi_locale_app_config, test_case: CareerReadinessTestCase): @@ -285,7 +285,7 @@ async def test_career_readiness_scripted(evals_setup, setup_multi_locale_app_con @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize("test_case", EDGE_CASE_TEST_CASES, ids=[tc.name for tc in EDGE_CASE_TEST_CASES]) async def test_career_readiness_edge_cases(evals_setup, setup_multi_locale_app_config, test_case: CareerReadinessEdgeCaseTestCase): diff --git a/backend/evaluation_tests/collect_experiences_agent/_data_extraction_llm_test.py b/backend/evaluation_tests/collect_experiences_agent/_data_extraction_llm_test.py index f2c0e98aa..a3a276b8b 100644 --- a/backend/evaluation_tests/collect_experiences_agent/_data_extraction_llm_test.py +++ b/backend/evaluation_tests/collect_experiences_agent/_data_extraction_llm_test.py @@ -999,7 +999,7 @@ class _TestCaseDataExtraction(CompassTestCase): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases_data_extraction), ids=[case.name for case in get_test_cases_to_run(test_cases_data_extraction)]) diff --git a/backend/evaluation_tests/collect_experiences_agent/collect_experiences_agent_scripted_user_test.py b/backend/evaluation_tests/collect_experiences_agent/collect_experiences_agent_scripted_user_test.py index 781006732..05e5e327f 100644 --- a/backend/evaluation_tests/collect_experiences_agent/collect_experiences_agent_scripted_user_test.py +++ b/backend/evaluation_tests/collect_experiences_agent/collect_experiences_agent_scripted_user_test.py @@ -67,7 +67,7 @@ async def collect_experiences_exec(caplog: LogCaptureFixture, test_case: Scripte @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) async def test_kenyan_user_multiple_experiences_transition_issues(caplog: LogCaptureFixture, setup_collect_experiences_agent: Awaitable[tuple[ @@ -201,7 +201,7 @@ async def test_stuck_after_recap_confirmation(caplog: LogCaptureFixture, @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) async def test_quick_reply_options_appear_at_yes_no_questions(caplog: LogCaptureFixture, setup_collect_experiences_agent: Awaitable[tuple[ diff --git a/backend/evaluation_tests/collect_experiences_agent/collect_experiences_agent_simulated_user_test.py b/backend/evaluation_tests/collect_experiences_agent/collect_experiences_agent_simulated_user_test.py index a2d7252da..36387738d 100644 --- a/backend/evaluation_tests/collect_experiences_agent/collect_experiences_agent_simulated_user_test.py +++ b/backend/evaluation_tests/collect_experiences_agent/collect_experiences_agent_simulated_user_test.py @@ -21,7 +21,7 @@ @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases), ids=[case.name for case in get_test_cases_to_run(test_cases)]) diff --git a/backend/evaluation_tests/collect_experiences_agent/data_extraction/entity_extraction_tool_test.py b/backend/evaluation_tests/collect_experiences_agent/data_extraction/entity_extraction_tool_test.py index 9926a12f3..a27eba243 100644 --- a/backend/evaluation_tests/collect_experiences_agent/data_extraction/entity_extraction_tool_test.py +++ b/backend/evaluation_tests/collect_experiences_agent/data_extraction/entity_extraction_tool_test.py @@ -385,7 +385,7 @@ class EntityExtractionToolTestCase(CompassTestCase): @pytest.mark.asyncio @pytest.mark.repeat(3) -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases), ids=[case.name for case in get_test_cases_to_run(test_cases)]) async def test_entity_extraction_tool(test_case: EntityExtractionToolTestCase, caplog): diff --git a/backend/evaluation_tests/collect_experiences_agent/data_extraction/intent_analyzer_llm_test.py b/backend/evaluation_tests/collect_experiences_agent/data_extraction/intent_analyzer_llm_test.py index 0600011b7..94cc6f179 100644 --- a/backend/evaluation_tests/collect_experiences_agent/data_extraction/intent_analyzer_llm_test.py +++ b/backend/evaluation_tests/collect_experiences_agent/data_extraction/intent_analyzer_llm_test.py @@ -264,7 +264,7 @@ class IntentAnalyzerToolTestCase(CompassTestCase): @pytest.mark.asyncio @pytest.mark.repeat(3) -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases), ids=[case.name for case in get_test_cases_to_run(test_cases)]) async def test_intent_analyzer_tool(test_case: IntentAnalyzerToolTestCase, caplog): diff --git a/backend/evaluation_tests/collect_experiences_agent/data_extraction/temporal_classifier_test.py b/backend/evaluation_tests/collect_experiences_agent/data_extraction/temporal_classifier_test.py index 6ab84054f..1dc11b54e 100644 --- a/backend/evaluation_tests/collect_experiences_agent/data_extraction/temporal_classifier_test.py +++ b/backend/evaluation_tests/collect_experiences_agent/data_extraction/temporal_classifier_test.py @@ -298,7 +298,7 @@ class TemporalAndWorkTypeClassifierToolTestCase(CompassTestCase): @pytest.mark.asyncio @pytest.mark.repeat(3) -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases), ids=[case.name for case in get_test_cases_to_run(test_cases)]) async def test_temporal_and_work_type_classification(test_case: TemporalAndWorkTypeClassifierToolTestCase, caplog, diff --git a/backend/evaluation_tests/collect_experiences_agent/data_extraction/transition_decision_tool_test.py b/backend/evaluation_tests/collect_experiences_agent/data_extraction/transition_decision_tool_test.py index 29a817a4e..aae06095a 100644 --- a/backend/evaluation_tests/collect_experiences_agent/data_extraction/transition_decision_tool_test.py +++ b/backend/evaluation_tests/collect_experiences_agent/data_extraction/transition_decision_tool_test.py @@ -448,7 +448,7 @@ class TransitionDecisionToolTestCase(CompassTestCase): @pytest.mark.asyncio @pytest.mark.repeat(3) -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases), ids=[case.name for case in get_test_cases_to_run(test_cases)]) async def test_transition_decision_tool(test_case: TransitionDecisionToolTestCase, caplog): diff --git a/backend/evaluation_tests/collect_experiences_agent/memory_hallucination_baseline_test.py b/backend/evaluation_tests/collect_experiences_agent/memory_hallucination_baseline_test.py index bcab82a09..ad0821300 100644 --- a/backend/evaluation_tests/collect_experiences_agent/memory_hallucination_baseline_test.py +++ b/backend/evaluation_tests/collect_experiences_agent/memory_hallucination_baseline_test.py @@ -108,7 +108,7 @@ async def collect_experiences_exec(caplog: LogCaptureFixture, test_case: Scripte @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(5) async def test_memory_recall_accuracy_after_summarization( caplog: LogCaptureFixture, diff --git a/backend/evaluation_tests/collect_experiences_agent/recap_field_accuracy_test.py b/backend/evaluation_tests/collect_experiences_agent/recap_field_accuracy_test.py index 12634241c..bcec6e644 100644 --- a/backend/evaluation_tests/collect_experiences_agent/recap_field_accuracy_test.py +++ b/backend/evaluation_tests/collect_experiences_agent/recap_field_accuracy_test.py @@ -104,7 +104,7 @@ async def collect_experiences_exec(caplog: LogCaptureFixture, test_case: Scripte @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) async def test_recap_contains_only_structured_fields( caplog: LogCaptureFixture, diff --git a/backend/evaluation_tests/cv_parser/test_cv_parser.py b/backend/evaluation_tests/cv_parser/test_cv_parser.py index c651c75b1..b039c04eb 100644 --- a/backend/evaluation_tests/cv_parser/test_cv_parser.py +++ b/backend/evaluation_tests/cv_parser/test_cv_parser.py @@ -35,7 +35,7 @@ def _to_markdown(self) -> str: @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize("case", test_cases_to_run, ids=[c.name for c in test_cases_to_run]) async def test_cv_parser(case: CVParserTestCase, common_folder_path: str): diff --git a/backend/evaluation_tests/linking_and_ranking_pipeline/experience_pipeline_test.py b/backend/evaluation_tests/linking_and_ranking_pipeline/experience_pipeline_test.py index 9a6bb8b04..4ec4bed42 100644 --- a/backend/evaluation_tests/linking_and_ranking_pipeline/experience_pipeline_test.py +++ b/backend/evaluation_tests/linking_and_ranking_pipeline/experience_pipeline_test.py @@ -275,7 +275,7 @@ class ExperiencePipelineTestCase(CompassTestCase): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize("test_case", get_test_cases_to_run(test_cases), ids=[test_case.name for test_case in get_test_cases_to_run(test_cases)]) async def test_experience_pipeline(test_case: ExperiencePipelineTestCase, setup_search_services: Awaitable[SearchServices], caplog: pytest.LogCaptureFixture): diff --git a/backend/evaluation_tests/linking_and_ranking_pipeline/infer_occupation_tool/_contextualization_llm_test.py b/backend/evaluation_tests/linking_and_ranking_pipeline/infer_occupation_tool/_contextualization_llm_test.py index 5dca4fd8f..526cd5a27 100644 --- a/backend/evaluation_tests/linking_and_ranking_pipeline/infer_occupation_tool/_contextualization_llm_test.py +++ b/backend/evaluation_tests/linking_and_ranking_pipeline/infer_occupation_tool/_contextualization_llm_test.py @@ -94,7 +94,7 @@ class ContextualizationTestCase(CompassTestCase): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.parametrize("test_case", get_test_cases_to_run(test_cases), ids=[test_case.name for test_case in get_test_cases_to_run(test_cases)]) async def test_relevant_occupations_classifier_llm(test_case: ContextualizationTestCase, caplog: LogCaptureFixture): logger = logging.getLogger(__name__) diff --git a/backend/evaluation_tests/linking_and_ranking_pipeline/pick_top_skills_tool_test.py b/backend/evaluation_tests/linking_and_ranking_pipeline/pick_top_skills_tool_test.py index e4a4b2770..669804cb9 100644 --- a/backend/evaluation_tests/linking_and_ranking_pipeline/pick_top_skills_tool_test.py +++ b/backend/evaluation_tests/linking_and_ranking_pipeline/pick_top_skills_tool_test.py @@ -133,7 +133,7 @@ class PickTopSkillToolTestCase(CompassTestCase): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize("test_case", get_test_cases_to_run(test_cases), ids=[test_case.name for test_case in get_test_cases_to_run(test_cases)]) async def test_pick_top_skill_tool(test_case: PickTopSkillToolTestCase, caplog): diff --git a/backend/evaluation_tests/skill_explorer_agent/_conversation_llm_test.py b/backend/evaluation_tests/skill_explorer_agent/_conversation_llm_test.py index 1924c7968..30592e06a 100644 --- a/backend/evaluation_tests/skill_explorer_agent/_conversation_llm_test.py +++ b/backend/evaluation_tests/skill_explorer_agent/_conversation_llm_test.py @@ -192,7 +192,7 @@ class _TestCaseConversation(CompassTestCase): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases), ids=[case.name for case in get_test_cases_to_run(test_cases)]) diff --git a/backend/evaluation_tests/skill_explorer_agent/_responsibilities_extraction_tool_test.py b/backend/evaluation_tests/skill_explorer_agent/_responsibilities_extraction_tool_test.py index 4fcea4cf8..9e9059452 100644 --- a/backend/evaluation_tests/skill_explorer_agent/_responsibilities_extraction_tool_test.py +++ b/backend/evaluation_tests/skill_explorer_agent/_responsibilities_extraction_tool_test.py @@ -145,7 +145,7 @@ class _TestCaseDataExtraction(CompassTestCase): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases_data_extraction), ids=[case.name for case in get_test_cases_to_run(test_cases_data_extraction)]) diff --git a/backend/evaluation_tests/skill_explorer_agent/_sentence_decomposition_llm_test.py b/backend/evaluation_tests/skill_explorer_agent/_sentence_decomposition_llm_test.py index 09057cd97..7d6cb11c2 100644 --- a/backend/evaluation_tests/skill_explorer_agent/_sentence_decomposition_llm_test.py +++ b/backend/evaluation_tests/skill_explorer_agent/_sentence_decomposition_llm_test.py @@ -129,7 +129,7 @@ class _TestCaseSentenceDecomposition(CompassTestCase): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases_sentence_decomposition), ids=[case.name for case in get_test_cases_to_run(test_cases_sentence_decomposition)]) diff --git a/backend/evaluation_tests/skill_explorer_agent/loop_detection_scripted_user_test.py b/backend/evaluation_tests/skill_explorer_agent/loop_detection_scripted_user_test.py index fc96994b7..9f5f6700d 100644 --- a/backend/evaluation_tests/skill_explorer_agent/loop_detection_scripted_user_test.py +++ b/backend/evaluation_tests/skill_explorer_agent/loop_detection_scripted_user_test.py @@ -168,7 +168,7 @@ class ScriptedLoopDetectionTestCase(ScriptedUserEvaluationTestCase): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize( "test_case", diff --git a/backend/evaluation_tests/skill_explorer_agent/loop_detection_test.py b/backend/evaluation_tests/skill_explorer_agent/loop_detection_test.py index d3285b84b..b6e0bb340 100644 --- a/backend/evaluation_tests/skill_explorer_agent/loop_detection_test.py +++ b/backend/evaluation_tests/skill_explorer_agent/loop_detection_test.py @@ -230,7 +230,7 @@ class LoopDetectionTestCase(EvaluationTestCase): ] @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize( "test_case", diff --git a/backend/evaluation_tests/skill_explorer_agent/skills_explorer_agent_simulated_user_test.py b/backend/evaluation_tests/skill_explorer_agent/skills_explorer_agent_simulated_user_test.py index 91b3f5dec..06a675e35 100644 --- a/backend/evaluation_tests/skill_explorer_agent/skills_explorer_agent_simulated_user_test.py +++ b/backend/evaluation_tests/skill_explorer_agent/skills_explorer_agent_simulated_user_test.py @@ -22,7 +22,7 @@ @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases), ids=[case.name for case in get_test_cases_to_run(test_cases)]) diff --git a/backend/evaluation_tests/summarizer/summarizer_test.py b/backend/evaluation_tests/summarizer/summarizer_test.py index 689f30676..f40ce0eea 100644 --- a/backend/evaluation_tests/summarizer/summarizer_test.py +++ b/backend/evaluation_tests/summarizer/summarizer_test.py @@ -17,7 +17,7 @@ async def _evaluate_with_llm(prompt: str) -> str: @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) async def test_summarizer_without_existing_summary(fake_conversation_context: FakeConversationContext, common_folder_path: str): @@ -70,7 +70,7 @@ async def test_summarizer_without_existing_summary(fake_conversation_context: Fa @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) async def test_summarizer_with_existing_summary(fake_conversation_context: FakeConversationContext, common_folder_path: str): diff --git a/backend/evaluation_tests/translation_tool_test.py b/backend/evaluation_tests/translation_tool_test.py index f30735372..13ffb74b1 100644 --- a/backend/evaluation_tests/translation_tool_test.py +++ b/backend/evaluation_tests/translation_tool_test.py @@ -42,7 +42,7 @@ class _TestCase(CompassTestCase): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize('current_test_case', get_test_cases_to_run(test_cases), ids=[case.test_id for case in get_test_cases_to_run(test_cases)]) diff --git a/backend/evaluation_tests/welcome_agent/welcome_agent_no_redundant_questions_test.py b/backend/evaluation_tests/welcome_agent/welcome_agent_no_redundant_questions_test.py index 91096ee63..139a74f8e 100644 --- a/backend/evaluation_tests/welcome_agent/welcome_agent_no_redundant_questions_test.py +++ b/backend/evaluation_tests/welcome_agent/welcome_agent_no_redundant_questions_test.py @@ -89,7 +89,7 @@ def _assert_no_redundant_questions(response_text: str): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(5) async def test_welcome_agent_does_not_ask_about_known_personal_data(evals_setup, caplog): """ diff --git a/backend/evaluation_tests/welcome_agent/welcome_agent_scripted_user_test.py b/backend/evaluation_tests/welcome_agent/welcome_agent_scripted_user_test.py index 5af42c975..b36041bfd 100644 --- a/backend/evaluation_tests/welcome_agent/welcome_agent_scripted_user_test.py +++ b/backend/evaluation_tests/welcome_agent/welcome_agent_scripted_user_test.py @@ -167,7 +167,7 @@ def event_loop(): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases), ids=[case.name for case in get_test_cases_to_run(test_cases)]) diff --git a/backend/evaluation_tests/welcome_agent/welcome_agent_simulated_user_test.py b/backend/evaluation_tests/welcome_agent/welcome_agent_simulated_user_test.py index c07909bc5..e25f01727 100644 --- a/backend/evaluation_tests/welcome_agent/welcome_agent_simulated_user_test.py +++ b/backend/evaluation_tests/welcome_agent/welcome_agent_simulated_user_test.py @@ -32,7 +32,7 @@ def event_loop(): @pytest.mark.asyncio -@pytest.mark.evaluation_test("gemini-2.5-flash-lite/") +@pytest.mark.evaluation_test("gemini-3.5-flash-lite/") @pytest.mark.repeat(3) @pytest.mark.parametrize('test_case', get_test_cases_to_run(test_cases), ids=[case.name for case in get_test_cases_to_run(test_cases)]) diff --git a/iac/backend/deploy_backend.py b/iac/backend/deploy_backend.py index 7b13b70ec..901049b77 100644 --- a/iac/backend/deploy_backend.py +++ b/iac/backend/deploy_backend.py @@ -54,7 +54,7 @@ class BackendServiceConfig: langfuse_public_key: Optional[str] langfuse_secret_key: Optional[str | pulumi.Output[str]] tracing_config: Optional[str] - # LLM provider and per-tier models. All optional; unset deploys on the Gemini 2.5 defaults. + # LLM provider and per-tier models. All optional; unset deploys on the Gemini defaults. # The Anthropic API key must be provisioned as a secret, never as a plain environment value. llm_provider: Optional[str] llm_default_model: Optional[str] diff --git a/iac/templates/env.template b/iac/templates/env.template index 32a2fc146..5802db5da 100644 --- a/iac/templates/env.template +++ b/iac/templates/env.template @@ -53,8 +53,8 @@ BACKEND_LANGFUSE_PUBLIC_KEY= BACKEND_LANGFUSE_SECRET_KEY= BACKEND_TRACING_CONFIG= -# LLM provider and per-tier model settings - all optional, leave empty to deploy on the Gemini 2.5 defaults -# (gemini-2.5-flash-lite / gemini-2.5-flash / gemini-2.5-pro). +# LLM provider and per-tier model settings - all optional, leave empty to deploy on the Gemini defaults +# (gemini-3.5-flash-lite / gemini-3.8-flash / gemini-3.1-pro-preview). The Gemini 3.x models require VERTEX_API_GEN_AI_REGION=global. # For a non-Gemini provider, an empty tier falls back to the next tier down that is set # (deep reasoning -> reasoning -> default). See backend/.env.example for details. LLM_PROVIDER=/.*/