From ee5c593f7838f8f70188de9945eaad575a933553 Mon Sep 17 00:00:00 2001 From: "LongZhenren(Zhibo Zhang)" Date: Fri, 2 Oct 2026 23:06:44 +0800 Subject: [PATCH] fix(eval): strip query strings when extracting Wikipedia titles from URLs Mobile share links commonly carry query parameters like ?wprov=, which ended up inside the extracted title and made retrieve() look up a non-existent article. Strip both ?query and #fragment suffixes. --- eval/lib/retrieval.py | 2 +- tests/test_wiki_title_extraction.py | 35 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/test_wiki_title_extraction.py diff --git a/eval/lib/retrieval.py b/eval/lib/retrieval.py index 75a4ec7..747ef2f 100644 --- a/eval/lib/retrieval.py +++ b/eval/lib/retrieval.py @@ -900,7 +900,7 @@ def _extract_wiki_title(self, url: str) -> str | None: # Match patterns like: # https://en.wikipedia.org/wiki/Python_(programming_language) # https://zh.wikipedia.org/wiki/Artificial_intelligence - pattern = r"https?://[a-z]{2,3}\.wikipedia\.org/wiki/(.+?)(?:#.*)?$" + pattern = r"https?://[a-z]{2,3}\.wikipedia\.org/wiki/(.+?)(?:[?#].*)?$" match = re.match(pattern, url) if match: title = unquote(match.group(1)) diff --git a/tests/test_wiki_title_extraction.py b/tests/test_wiki_title_extraction.py new file mode 100644 index 0000000..4ac5ec2 --- /dev/null +++ b/tests/test_wiki_title_extraction.py @@ -0,0 +1,35 @@ +"""Regression tests for Wikipedia title extraction from URLs.""" + +from pathlib import Path +from runpy import run_path + +RETRIEVAL = run_path(str(Path(__file__).parents[1] / "eval" / "lib" / "retrieval.py")) +extract = RETRIEVAL["WikipediaAPIRetriever"]._extract_wiki_title + + +def test_plain_url_title_unchanged(): + assert extract(None, "https://en.wikipedia.org/wiki/Albert_Einstein") == ( + "Albert Einstein" + ) + + +def test_query_string_is_stripped(): + assert extract( + None, "https://en.wikipedia.org/wiki/Albert_Einstein?wprov=rarw1" + ) == ("Albert Einstein") + + +def test_fragment_is_stripped(): + assert extract( + None, "https://en.wikipedia.org/wiki/Albert_Einstein#Early_life" + ) == ("Albert Einstein") + + +def test_percent_encoded_title_is_decoded(): + assert extract( + None, "https://en.wikipedia.org/wiki/Python_%28programming_language%29" + ) == ("Python (programming language)") + + +def test_non_wiki_url_returns_none(): + assert extract(None, "https://example.com/wiki/Albert_Einstein") is None