fix(rag): keep sources when the LLM omits the citation tag

CLAUDE.md

@@ -133,10 +133,10 @@ The RAG pipeline filters out false-positive sources by having the LLM self-repor
 1. `format_context()` (`openrag/core/prompts/chat_prompt_builder.py`) numbers each source (`[Source 1]`, `[Source 2]`, ...) in the context and returns `(formatted_text, included_indices)` — the indices track which docs fit within the token budget
 2. Prompt templates (`openrag/prompts/templates/*.txt`) instruct the LLM to append `[Sources: 1, 3, 5]` at the end of its response
 3. `extract_and_strip_sources_block()` (`openrag/core/utils/source_filtering.py`) strips this tag from the response before sending to the client
-4. `filter_sources_by_citations()` (`openrag/core/utils/source_filtering.py`) filters the source metadata to only include cited sources (falls back to all sources if none match)
+4. `filter_sources_by_citations()` (`openrag/core/utils/source_filtering.py`) filters the source metadata to only include cited sources; if no `[Sources: ...]` tag is found at all, every retrieved source is kept instead (a missing tag means the model didn't report citations, not that it used none)
 5. For streaming, the OpenAI router buffers the last 100 chars to catch the sources tag before it reaches the client
 
-The `extra` field in API responses is a JSON string: `{"sources": [filtered_source_list]}`.
+The `extra` field in API responses is a JSON string: `{"sources": [filtered_source_list], "all_retrieved_sources": [all_retrieved_sources]}`. `all_retrieved_sources` always carries every source retrieval produced, unfiltered by citation — useful for debugging and RAG evaluation, regardless of what the model cited.
 
 ### API Routers (`openrag/api/routers/`)
 

openrag/core/utils/source_filtering.py

@@ -79,15 +79,15 @@ def extract_and_strip_sources_block(
     return cleaned, set()
 
 
-def filter_sources_by_citations(
-    sources: list,
-    citations: set[int] | None,
-    *,
-    allow_uncited: bool = False,
-) -> list:
-    """Keep only sources whose 1-based index was cited."""
+def filter_sources_by_citations(sources: list, citations: set[int] | None) -> list:
+    """Keep only sources whose 1-based index was cited.
+
+    No tag at all (``citations is None``) means the model didn't report which
+    sources it used, not that it used none — the answer may still be grounded
+    in them, so keep everything rather than silently dropping real sources.
+    """
     if citations is None:
-        return sources if allow_uncited else []
+        return sources
     if not citations:
         return []
     return [source for i, source in enumerate(sources, start=1) if i in citations]
@@ -112,7 +112,6 @@ async def stream_with_source_filtering(
     model_name: str,
     buffer_size: int | None = None,
     *,
-    allow_uncited_sources: bool = False,
     citation_protocol_active: bool = True,
 ):
     """Process an LLM SSE stream and, when active, strip source tags.
@@ -263,8 +262,8 @@ async def stream_with_source_filtering(
     else:
         final_clean, citations = pending, None
 
-    filtered = filter_sources_by_citations(sources, citations, allow_uncited=allow_uncited_sources)
-    extra_payload = {"sources": filtered}
+    filtered = filter_sources_by_citations(sources, citations)
+    extra_payload = {"sources": filtered, "all_retrieved_sources": sources}
     if not saw_done:
         extra_payload["truncated"] = True
         logger.warning(

openrag/services/orchestrators/query_service.py

@@ -655,7 +655,7 @@ async def chat(
         else:
             payload, docs, web_results, citation_protocol_active = await self._prepare_chat(partitions, payload, llm)
         sources = prepare_sources(docs, web_results)
-        structured_output = _allows_uncited_sources(payload)
+        structured_output = _is_structured_output(payload)
 
         payload["messages"] = self._sanitize_messages(payload["messages"])
         chunk = await llm.chat(payload["messages"], **_sampling(payload))
@@ -670,13 +670,7 @@ async def chat(
             clean, citations = content, None
         chunk["choices"][0]["message"]["content"] = clean
         chunk["extra"] = json.dumps(
-            {
-                "sources": filter_sources_by_citations(
-                    sources,
-                    citations,
-                    allow_uncited=structured_output,
-                )
-            }
+            {"sources": filter_sources_by_citations(sources, citations), "all_retrieved_sources": sources}
         )
         return chunk
 
@@ -697,15 +691,14 @@ async def chat_stream(
         else:
             payload, docs, web_results, citation_protocol_active = await self._prepare_chat(partitions, payload, llm)
         sources = prepare_sources(docs, web_results)
-        structured_output = _allows_uncited_sources(payload)
+        structured_output = _is_structured_output(payload)
 
         payload["messages"] = self._sanitize_messages(payload["messages"])
         llm_stream = llm.stream_chat(payload["messages"], **_sampling(payload))
         async for sse_line in stream_with_source_filtering(
             llm_stream,
             sources,
             model_name,
-            allow_uncited_sources=structured_output,
             citation_protocol_active=citation_protocol_active and not structured_output,
         ):
             yield sse_line
@@ -725,7 +718,7 @@ async def complete(
         else:
             payload, docs = await self._prepare_completions(partitions, payload, llm)
         sources = prepare_sources(docs, [])
-        structured_output = _allows_uncited_sources(payload)
+        structured_output = _is_structured_output(payload)
 
         resp = await llm.generate(payload["prompt"], **_sampling(payload, key="prompt"))
         text = resp.get("choices", [{}])[0].get("text", "") or ""
@@ -738,13 +731,7 @@ async def complete(
             clean, citations = text, None
         resp["choices"][0]["text"] = clean
         resp["extra"] = json.dumps(
-            {
-                "sources": filter_sources_by_citations(
-                    sources,
-                    citations,
-                    allow_uncited=structured_output,
-                )
-            }
+            {"sources": filter_sources_by_citations(sources, citations), "all_retrieved_sources": sources}
         )
         return resp
 
@@ -789,7 +776,7 @@ def _sampling(payload: dict, key: str = "messages") -> dict:
     return {k: v for k, v in payload.items() if k not in drop}
 
 
-def _allows_uncited_sources(payload: dict) -> bool:
+def _is_structured_output(payload: dict) -> bool:
     """Structured output cannot carry the plain-text citation marker."""
     response_format = payload.get("response_format")
     return isinstance(response_format, dict) and response_format.get("type") in {"json_object", "json_schema"}

tests/unit/core/utils/test_source_filtering.py

@@ -160,14 +160,10 @@ def test_basic_filtering(self):
         result = filter_sources_by_citations(sources, {1, 3, 5})
         assert result == ["a", "c", "e"]
 
-    def test_none_citations_returns_empty(self):
+    def test_none_citations_returns_all_sources(self):
+        """No tag at all means the model didn't report citations, not that it used none."""
         sources = ["a", "b", "c"]
         result = filter_sources_by_citations(sources, None)
-        assert result == []
-
-    def test_none_citations_can_be_allowed_for_structured_output(self):
-        sources = ["a", "b", "c"]
-        result = filter_sources_by_citations(sources, None, allow_uncited=True)
         assert result == ["a", "b", "c"]
 
     def test_empty_citations_returns_empty(self):
@@ -298,6 +294,18 @@ async def test_case1_llm_cites_specific_sources(self):
         assert _collect_content(result) == "Here is the answer."
         assert _parse_finish_sources(result) == [{"file": "a.pdf"}, {"file": "c.pdf"}]
 
+    @pytest.mark.asyncio
+    async def test_all_retrieved_sources_includes_uncited_ones(self):
+        """`all_retrieved_sources` always carries every candidate, unfiltered by citation."""
+        lines = [
+            _make_chunk("Here is the answer."),
+            _make_chunk("\n[Sources: 1, 3]"),
+            _make_finish(),
+            DONE_LINE,
+        ]
+        result = await _collect(stream_with_source_filtering(_fake_stream(lines), self.SOURCES, "test-model"))
+        assert _parse_finish_extra(result)["all_retrieved_sources"] == self.SOURCES
+
     @pytest.mark.asyncio
     async def test_content_and_finish_reason_in_same_chunk_keeps_last_token(self):
         """A provider that packs the final token and finish_reason into one chunk
@@ -393,32 +401,15 @@ async def test_case2_llm_says_sources_none(self):
         assert _parse_finish_sources(result) == []
 
     @pytest.mark.asyncio
-    async def test_case3_llm_no_tag_returns_no_sources(self):
-        """Case 3: LLM omits tag entirely → no source is attributed."""
+    async def test_case3_llm_no_tag_returns_all_sources(self):
+        """Case 3: LLM omits tag entirely → treated as unreported, not uncited; all sources kept."""
         lines = [
             _make_chunk("Answer without any sources tag."),
             _make_finish(),
             DONE_LINE,
         ]
         result = await _collect(stream_with_source_filtering(_fake_stream(lines), self.SOURCES, "test-model"))
         assert _collect_content(result) == "Answer without any sources tag."
-        assert _parse_finish_sources(result) == []
-
-    @pytest.mark.asyncio
-    async def test_no_tag_keeps_sources_when_uncited_output_is_allowed(self):
-        lines = [
-            _make_chunk('{"answer": "structured"}'),
-            _make_finish(),
-            DONE_LINE,
-        ]
-        result = await _collect(
-            stream_with_source_filtering(
-                _fake_stream(lines),
-                self.SOURCES,
-                "test-model",
-                allow_uncited_sources=True,
-            )
-        )
         assert _parse_finish_sources(result) == self.SOURCES
 
     @pytest.mark.asyncio
@@ -434,7 +425,6 @@ async def test_structured_output_preserves_source_like_json_values(self):
                 _fake_stream(lines),
                 self.SOURCES,
                 "test-model",
-                allow_uncited_sources=True,
                 citation_protocol_active=False,
             )
         )
@@ -514,8 +504,8 @@ async def test_inline_prose_tag_preserved_in_stream(self):
         result = await _collect(stream_with_source_filtering(_fake_stream(lines), self.SOURCES, "test-model"))
         content = _collect_content(result)
         assert content == "Use the format [Sources: 1, 3] at the very end of your response."
-        # No line-terminal tag means no source was actually cited.
-        assert _parse_finish_sources(result) == []
+        # No line-terminal tag means the model didn't report citations — kept, not dropped.
+        assert _parse_finish_sources(result) == self.SOURCES
 
     @pytest.mark.asyncio
     async def test_mid_response_tag_stripped_plus_trailing_tag(self):

tests/unit/services/orchestrators/test_query_service.py

@@ -407,7 +407,7 @@ async def _spy(**kwargs):
     out = await svc.chat(
         partitions=None,
         payload={"messages": [{"role": "user", "content": "hi"}], "metadata": {}},
-        prepare_sources=lambda d, w: [{"source_type": "document"}],
+        prepare_sources=lambda d, w: [{"source_type": "document"}] if d or w else [],
         model_name="m1",
     )
     assert called["n"] == 0  # no retrieval in direct mode
@@ -458,8 +458,9 @@ async def test_chat_with_partition_retrieves_and_filters_sources():
         prepare_sources=lambda d, w: sources,
         model_name="m",
     )
-    filtered = json.loads(out["extra"])["sources"]
-    assert filtered == [{"source_type": "document", "n": 1}]  # only cited source 1
+    extra = json.loads(out["extra"])
+    assert extra["sources"] == [{"source_type": "document", "n": 1}]  # only cited source 1
+    assert extra["all_retrieved_sources"] == sources  # unfiltered, everything retrieved
 
 
 @pytest.mark.asyncio
@@ -487,7 +488,7 @@ async def test_chat_conversational_request_skips_partition_retrieval():
     out = await svc.chat(
         partitions=["p"],
         payload={"messages": [{"role": "user", "content": "How can you help me?"}], "metadata": {}},
-        prepare_sources=lambda d, w: [{"source_type": "document"}],
+        prepare_sources=lambda d, w: [{"source_type": "document"}] if d or w else [],
         model_name="m",
     )
 
@@ -569,7 +570,8 @@ async def test_chat_inconsistent_classifier_result_prefers_supplied_query():
 
 
 @pytest.mark.asyncio
-async def test_chat_without_citation_does_not_attribute_retrieved_sources():
+async def test_chat_without_citation_keeps_retrieved_sources():
+    """No tag at all means the model didn't report citations, not that the answer is unsourced."""
     svc = _svc(llm=FakeLLM(chat_responses=["A general answer with no citation marker."]))
     sources = [{"source_type": "document", "filename": "unrelated.pdf"}]
 
@@ -580,7 +582,7 @@ async def test_chat_without_citation_does_not_attribute_retrieved_sources():
         model_name="m",
     )
 
-    assert json.loads(out["extra"])["sources"] == []
+    assert json.loads(out["extra"])["sources"] == sources
 
 
 @pytest.mark.asyncio
@@ -966,7 +968,7 @@ async def test_complete_direct_mode_preserves_literal_source_marker():
     out = await svc.complete(
         partitions=None,
         payload={"prompt": "do x"},
-        prepare_sources=lambda d, w: [{"x": 1}],
+        prepare_sources=lambda d, w: [{"x": 1}] if d or w else [],
     )
     assert out["choices"][0]["text"] == answer
     assert json.loads(out["extra"])["sources"] == []