diff --git a/eval/lib/retrieval.py b/eval/lib/retrieval.py index 75a4ec7..747ef2f 100644 --- a/eval/lib/retrieval.py +++ b/eval/lib/retrieval.py @@ -900,7 +900,7 @@ def _extract_wiki_title(self, url: str) -> str | None: # Match patterns like: # https://en.wikipedia.org/wiki/Python_(programming_language) # https://zh.wikipedia.org/wiki/Artificial_intelligence - pattern = r"https?://[a-z]{2,3}\.wikipedia\.org/wiki/(.+?)(?:#.*)?$" + pattern = r"https?://[a-z]{2,3}\.wikipedia\.org/wiki/(.+?)(?:[?#].*)?$" match = re.match(pattern, url) if match: title = unquote(match.group(1)) diff --git a/tests/test_wiki_title_extraction.py b/tests/test_wiki_title_extraction.py new file mode 100644 index 0000000..4ac5ec2 --- /dev/null +++ b/tests/test_wiki_title_extraction.py @@ -0,0 +1,35 @@ +"""Regression tests for Wikipedia title extraction from URLs.""" + +from pathlib import Path +from runpy import run_path + +RETRIEVAL = run_path(str(Path(__file__).parents[1] / "eval" / "lib" / "retrieval.py")) +extract = RETRIEVAL["WikipediaAPIRetriever"]._extract_wiki_title + + +def test_plain_url_title_unchanged(): + assert extract(None, "https://en.wikipedia.org/wiki/Albert_Einstein") == ( + "Albert Einstein" + ) + + +def test_query_string_is_stripped(): + assert extract( + None, "https://en.wikipedia.org/wiki/Albert_Einstein?wprov=rarw1" + ) == ("Albert Einstein") + + +def test_fragment_is_stripped(): + assert extract( + None, "https://en.wikipedia.org/wiki/Albert_Einstein#Early_life" + ) == ("Albert Einstein") + + +def test_percent_encoded_title_is_decoded(): + assert extract( + None, "https://en.wikipedia.org/wiki/Python_%28programming_language%29" + ) == ("Python (programming language)") + + +def test_non_wiki_url_returns_none(): + assert extract(None, "https://example.com/wiki/Albert_Einstein") is None