test(evaluation): update the evaluation tests for the current agents CORE-803

backend/.gitignore

@@ -15,3 +15,4 @@ offline_output/
 exports/
 evaluation_tests/test_output/
 evaluation_tests/golden_transcripts/output/
+

backend/app/agent/agent_director/_llm_router.py

@@ -279,6 +279,8 @@ def _get_system_instructions(self, *, user_input: str, context: ConversationCont
                     - Each agent is responsible for clarifying the questions and doubts the user may have regarding its tasks.
                     - When the User Input contains a keyword or phrase that the agent asked the user to say in the Conversation History,
                       select the model responsible for the task the keyword was asked for, even if the User Input also contains other information.
+                    - When the User Input asks to explain, clarify or repeat a question that was asked earlier in the Conversation History,
+                      it is not a question about the process or the tool: select the model responsible for the task the conversation is currently focused on.
 
                 {examples}
 

backend/app/agent/career_explorer_agent/sector_relevance_classifier.py

@@ -37,13 +37,18 @@ class Config:
 
 
 class SectorRelevanceClassification(BaseModel):
+    # Every field is required (no defaults): a field with a default is optional in the response schema, and the model
+    # then tends to leave it out (e.g. is_priority silently falling back to False). The reasoning comes first so the
+    # model explains itself before committing to the classification.
+    reasoning: str = Field(description=f"A short explanation of the classification, under {MAX_REASONING_LENGTH} characters.")
     relevance: SectorRelevance = Field(description="Whether the PRIMARY sector of the user's message is a priority sector (PRIORITY_SECTOR) "
                                                    "or not (NON_PRIORITY_SECTOR).")
-    sector_name: str | None = Field(default=None, description="The name of the PRIMARY sector, see the sector_name rules. null if the message isn't about any sector.")
-    is_priority: bool = Field(default=False, description="True only if the PRIMARY sector is a priority sector.")
-    all_sectors: list[SectorMention] = Field(default_factory=list,
-                                             description="EVERY distinct sector the user mentions or expresses interest in during this message.")
-    reasoning: str = Field(default="", description=f"A short explanation of the classification, under {MAX_REASONING_LENGTH} characters.")
+    sector_name: str | None = Field(description="The name of the PRIMARY sector, following the sector_name rules: the exact configured sector name "
+                                                "for a priority sector, a broad high-level industry category otherwise. "
+                                                "Just the name, nothing else. null if the message isn't about any sector.")
+    is_priority: bool = Field(description="True only if the PRIMARY sector is a priority sector, i.e. it is consistent with relevance being PRIORITY_SECTOR.")
+    all_sectors: list[SectorMention] = Field(description="EVERY distinct sector the user mentions or expresses interest in during this message. "
+                                                         "Empty if the message isn't about any sector.")
 
     class Config:
         extra = "forbid"

backend/app/agent/collect_experiences_agent/_conversation_llm.py

@@ -132,8 +132,8 @@ def _translate_field(field_key: str) -> str:
 class _ConversationLLMResponse(BaseModel):
     """Structured JSON response from the conversation LLM."""
     message: str = Field(description="The conversational message for the user")
+    # No default: a field with a default is not in the schema's `required` list, and the model then skips it.
     quick_reply_options: list[LLMQuickReplyOption] | None = Field(
-        default=None,
         description="The quick-reply options for the message, see the '#Quick Reply Options' instructions. null when there are none.")
     model_config = ConfigDict(extra="forbid")
 

backend/app/agent/collect_experiences_agent/_transition_decision_tool.py

@@ -60,13 +60,14 @@ class TransitionReasoning(BaseModel):
 
 
 class _TransitionDecisionOutput(BaseModel):
+    # The reasoning comes first, so that the model reasons before it answers the two questions.
+    reasoning: str = Field(
+        description="A brief 1-2 sentence explanation of the decision.")
     continue_current_type: bool = Field(
         description="Whether to continue asking about the current work type, see the '#Decision Logic'.")
     done_with_collection: bool = Field(
         description="Whether we are completely done collecting all work experiences (only true if continue_current_type is false), "
                     "see the '#Decision Logic'.")
-    reasoning: str = Field(
-        description="A brief 1-2 sentence explanation of the decision.")
 
     class Config:
         extra = "forbid"
@@ -292,6 +293,11 @@ async def _internal_execute(self,
 #Constraints
 - Use semantic understanding, not keyword matching
 - When the user clearly indicates no more experiences of this type (e.g. "no", "nope"), return continue_current_type=false
+- A "no" followed by a closing phrase (e.g. "No, that's cool", "No, it's fine", "No, that's ok") means the user has nothing
+  more to add or change, even when the agent's last question was a confirmation like "Does that sound right?".
+  If the experiences of this type are complete, return continue_current_type=false.
+  A plain confirmation of the details (e.g. "Yes, that's correct") is different: it does not say there are no more experiences,
+  so return continue_current_type=true to let the agent ask about other experiences of this type.
 - If unexplored_types is not empty, done_with_collection must be false
 - This is for collecting experience data excluding tasks, skills and responsibilties.
   If the agent itself keeps asking the user about tasks, skills or responsibilities, return continue_current_type=false.

backend/app/agent/collect_experiences_agent/data_extraction_llm/_entity_extraction_tool.py

@@ -36,31 +36,45 @@ class DataExtractionReferences(BaseModel):
     The keys are typed (rather than a free-form dict) because an untyped object in the response schema makes
     the Gemini 3.x models intermittently return null for every extracted field.
     """
-    experience_title_references: Optional[str] = Field(default=None, description=_REFERENCES_DESCRIPTION.format(field="experience_title"))
-    company_references: Optional[str] = Field(default=None, description=_REFERENCES_DESCRIPTION.format(field="company"))
-    location_references: Optional[str] = Field(default=None, description=_REFERENCES_DESCRIPTION.format(field="location"))
+    # No defaults: a field with a default is not in the schema's `required` list, and the model then skips it.
+    experience_title_references: str = Field(description=_REFERENCES_DESCRIPTION.format(field="experience_title"))
+    company_references: str = Field(description=_REFERENCES_DESCRIPTION.format(field="company"))
+    location_references: str = Field(description=_REFERENCES_DESCRIPTION.format(field="location"))
 
 
 class ExtractedData(BaseModel):
     """
     Extract data per an experience from the EntityExtractionTool.
     """
 
-    # References and Reasoning.
-    data_extraction_references: Optional[DataExtractionReferences] = Field(
-        default=None,
-        description="Where you found the information for the experience details, see the '#Extract data instructions'.")
-
     # Experience Details
-    experience_title: Optional[str] = Field(default=None, description="A title for the experience, see the \"##'experience_title' instructions\".")
-    company: Optional[str] = Field(default=None, description="The name of the company or its type, see the \"##'company' instructions\".")
-    location: Optional[str] = Field(default=None, description="The location in which the job was performed, see the \"##'location' instructions\".")
+    experience_title: Optional[str] = None
+    company: Optional[str] = None
+    location: Optional[str] = None
 
 
 # Empty/Default extracted data.
 _EMPTY_EXTRACTED_DATA = ExtractedData()
 
 
+class _LLMExtractedData(BaseModel):
+    """
+    The experience details in the LLM output.
+    Every field is required (no defaults, `null` stays a valid value): a field with a default is left out of the
+    schema's `required` list, and the model then skips it so it silently takes the default value.
+    The references come first, so that the model explains where it found the values before giving them.
+    """
+
+    # References and Reasoning.
+    data_extraction_references: DataExtractionReferences = Field(
+        description="Where you found the information for the experience details, see the '#Extract data instructions'.")
+
+    # Experience Details
+    experience_title: Optional[str] = Field(description="A title for the experience, see the \"##'experience_title' instructions\".")
+    company: Optional[str] = Field(description="The name of the company or its type, see the \"##'company' instructions\".")
+    location: Optional[str] = Field(description="The location in which the job was performed, see the \"##'location' instructions\".")
+
+
 class _LLMOutput(BaseModel):
     """
     LLM Model Output.
@@ -74,7 +88,7 @@ class _LLMOutput(BaseModel):
                     "Once you reach the Previously Extracted Experience Data, do not follow the associations anymore. "
                     "e.g. \"user(<answer>) -> model(<question>) -> ... -> user(<answer>) -> model(<question>) -> Previously Extracted Experience Data(...)\". "
                     "Each step is a summarized version of the actual user or model turn, and the chain has at most 10 steps to avoid circular references.")
-    experience_details: ExtractedData = Field(
+    experience_details: _LLMExtractedData = Field(
         description="The experience details you extracted from the user's statement and the conversation history.")
 
 
@@ -105,8 +119,10 @@ def _get_llm(temperature_config: Optional[dict] = None) -> LLM:
             system_instructions=_SYSTEM_INSTRUCTIONS.format(language_style=get_language_style()),
             config=LLMConfig(
                 generation_config=ZERO_TEMPERATURE_GENERATION_CONFIG | JSON_GENERATION_CONFIG | {
-                    "max_output_tokens": 3000
-                    # Limit the output to 3000 tokens to avoid the "reasoning recursion issues"
+                    "max_output_tokens": 8192
+                    # Limit the output to avoid the "reasoning recursion issues".
+                    # The limit includes the thinking tokens of the Gemini 3.x models (often 1000-1500, more on long
+                    # conversations): with a lower limit, the thinking used it up and the JSON answer got cut off.
                 } | temperature_config | with_response_schema(_LLMOutput)
             ))
 
@@ -203,7 +219,11 @@ async def _internal_execute(self,
         Extract the title of the experience from the '<User's Last Input>'.
         For unpaid work, use the kind of work done (e.g. "Helping Neighbors", "Volunteering" etc).
         Make sure that the user is actually referring to an experience they have have.
-        When summarizing a user-stated action (e.g., "I sell tomatoes"), convert it directly into a gerund-phrase experience title (e.g., "Selling Tomatoes"). 
+        When summarizing a user-stated action (e.g., "I sell tomatoes"), convert it directly into a gerund-phrase experience title (e.g., "Selling Tomatoes").
+        Follow the wording the user used, do not switch between a role and an activity:
+        - a stated job title or role stays as stated (e.g. "I was a Project Manager" is "Project Manager", not "Managing Projects"),
+        - a stated activity stays an activity (e.g. "I started teaching graphic design" is "Teaching Graphic Design",
+          "my consulting work" is "Consulting", not "Consultant").
         Return a string value containing the title of the experience.
         Use `null`: If the user has not mentioned their `experience title` and has not yet been asked to provide it.
         Use "": If the user explicitly declines to provide their `experience title` when explicitly asked, or requests that previously stored `experience title` data be deleted.
@@ -212,8 +232,11 @@ async def _internal_execute(self,
         What the company does or name of the company depending on the context.
         Use specific company names (eg: Acme inc) not generic ones (eg: 'company', 'online, 'organization', 'freelance' or 'self') in the output.
         For unpaid work, use the receiver of the work (e.g. "My Family", "My Community", etc) but not the generic name.
+        If the user does not name the company but says the kind of business or place they work at
+        (e.g. "at the market", "at a hotel", "at my father's shop"), use it in the user's own words, keeping what qualifies it
+        (e.g. "Market", "Hotel", "My father's shop", "Local market"), in the language of the user.
         Return a string value containing the type, or name of the company, or the receiver of the work.
-        Use `null`: If the user has not mentioned their `company name` and has not yet been asked to provide it.
+        Use `null`: If the user has mentioned neither the `company name` nor the kind of business or place, and has not yet been asked to provide it.
         Use "": If the user explicitly declines to provide their `company name` when explicitly asked, or requests that previously stored `company name` data be deleted.
 
      ##'location' instructions 

backend/app/agent/collect_experiences_agent/data_extraction_llm/_intent_analyzer_tool.py

@@ -33,13 +33,13 @@
 
 
 class Operation(BaseModel):
-    index: Optional[int] = Field(
-        default=None,
-        description="The unique identifier of the experience. If the experience already exists in <Previously Extracted Experience Data>, "
-                    "the existing index, otherwise `null`.")
-
+    """
+    An operation in the LLM output.
+    Every field is required (no defaults, `null` stays a valid value where it is allowed): a field with a default is left out
+    of the schema's `required` list, and the model then skips it so it silently takes the default value.
+    The associations and the reasoning come before the answer fields, so that the model reasons before answering.
+    """
     associations: Optional[str] = Field(
-        default="",
         description="A linear chain of associations in the form of ...-> ...->... that starts from the User's Last Input "
                     "and follows the relevant entries it refers to in the Conversation History, until it terminates at the "
                     "Previously Extracted Experience Data, if relevant. Skip unrelated or tangential turns to preserve a coherent "
@@ -49,18 +49,22 @@ class Operation(BaseModel):
                     "Each step is a summarized version of the actual user or model turn, and the chain has at most 10 steps to avoid circular references.")
 
     data_operation_reasoning: Optional[str] = Field(
-        default="",
         description="A detailed, step-by-step explanation in prose of what data operation should be performed. "
                     "Consider the conversation context carefully: if the user is answering a question about existing work, "
                     "this should be an UPDATE. If they are describing completely new work, this should be an ADD.")
     data_operation: str = Field(
         description="The operation that should be performed to the experience data, one of: 'ADD', 'UPDATE', 'DELETE', 'NOOP'. "
                     "'NOOP' means that no operation should be performed.")
+    index: Optional[int] = Field(
+        description="The unique identifier of the experience. If the experience already exists in <Previously Extracted Experience Data>, "
+                    "the existing index, otherwise `null`.")
 
     potential_new_experience_title: Optional[str] = Field(
-        default="",
         description="The potential experience title from the details described by the user in the '<User's Last Input>', "
-                    "like the title to be put on a CV when describing the experience. If the experience already exists in "
+                    "like the title to be put on a CV when describing the experience. "
+                    "When the user did not state a role (e.g. helping out at home or for the family), keep where or for whom "
+                    "the work was done in the title (e.g. 'Helping at my mother's house'), not a generic title (e.g. 'General helper'). "
+                    "If the experience already exists in "
                     "<Previously Extracted Experience Data> and the user is not updating it, the existing title. "
                     "An empty string if you can't figure it out.")
     users_statement: str = Field(
@@ -73,6 +77,9 @@ def is_empty(self):
 
 
 class _LLMOutput(BaseModel):
+    # ignored_experiences and users_statements are deliberately optional (not in the schema's `required` list):
+    # their values are not used, and when the model is forced to write ignored_experiences first, it talks itself into
+    # ignoring requests it should act on (e.g. "figure out which one should be deleted" ends up with no DELETE).
     ignored_experiences: Optional[str] = Field(
         default=None,
         description="A detailed, step-by-step explanation in prose of the experiences referenced by the user that will not be added "
@@ -114,8 +121,10 @@ def _get_llm(previously_extracted_data: str, temperature_config: Optional[dict]
                                                             language_style=get_language_style()),
             config=LLMConfig(
                 generation_config=ZERO_TEMPERATURE_GENERATION_CONFIG | JSON_GENERATION_CONFIG | {
-                    "max_output_tokens": 3000
-                    # Limit the output to 3000 tokens to avoid the "reasoning recursion issues"
+                    "max_output_tokens": 8192
+                    # Limit the output to avoid the "reasoning recursion issues".
+                    # The limit includes the thinking tokens of the Gemini 3.x models (often 1000-1500, more on long
+                    # conversations): with a lower limit, the thinking used it up and the JSON answer got cut off.
                 } | temperature_config | with_response_schema(_LLMOutput)
             ))
 

backend/app/agent/collect_experiences_agent/data_extraction_llm/_operations_processor.py

@@ -203,6 +203,8 @@ def _find_same_experience_for_add(self, add_payload: Operation, work_type_str: s
         Only merges when both work_type AND title match exactly (case-insensitive).
         Substring matching is intentionally avoided to prevent merging distinct experiences
         that share a common title prefix (e.g. two different 'Retail Sales Assistant' roles).
+        Experiences with the same title whose company, location or dates conflict are distinct
+        (e.g. a 'Research Assistant' job at two different labs), so they are not merged either.
         """
         new_title = (add_payload.experience_title or "").strip().lower()
         if not new_title:
@@ -213,10 +215,20 @@ def _find_same_experience_for_add(self, add_payload: Operation, work_type_str: s
             existing_title = (existing.experience_title or "").strip().lower()
             if not existing_title:
                 continue
-            if new_title == existing_title:
+            if new_title == existing_title and not self._has_conflicting_details(add_payload, existing):
                 return i
         return -1
 
+    @staticmethod
+    def _has_conflicting_details(add_payload: Operation, existing: CollectedData) -> bool:
+        """True if the ADD and the existing experience have different non-empty company, location or dates."""
+        for field in ("company", "location", "start_date", "end_date"):
+            new_value = (getattr(add_payload, field) or "").strip().lower()
+            existing_value = (getattr(existing, field) or "").strip().lower()
+            if new_value and existing_value and new_value != existing_value:
+                return True
+        return False
+
     @staticmethod
     def _merge_add_into_existing(add_payload: Operation, work_type_str: str | None,
                                  existing: CollectedData, _current_turn_index: int):