diff --git a/docs/spiders/advanced.md b/docs/spiders/advanced.md index f1814e830..e6ad6c497 100644 --- a/docs/spiders/advanced.md +++ b/docs/spiders/advanced.md @@ -216,6 +216,37 @@ There's no automatic expiration. To force a fresh crawl, delete the cache direct Development mode is meant for development, not production. Cached responses never expire, and replay bypasses rate limiting and blocked-request retries. Don't ship a spider with `development_mode = True`. +## Conditional Requests + +Spiders that run on a schedule, like one that checks a set of feeds every few minutes, mostly download pages that haven't changed since the last run. With `conditional_requests = True`, the spider saves the `ETag` and `Last-Modified` headers of each page it processed, and the next run sends them back as `If-None-Match` / `If-Modified-Since`. A website that supports them answers `304 Not Modified` with an empty body instead of the whole page. + +```python +class FeedMonitor(Spider): + name = "feed_monitor" + start_urls = ["https://example.com/feed.xml"] + conditional_requests = True + + async def parse(self, response: Response): + for item in response.css("item"): + yield {"title": item.css("title::text").get("")} + + async def on_not_modified(self, request, response): + self.logger.info(f"No changes in {request.url}") +``` + +When the answer is `304`, the request's callback is skipped, since there's nothing new for it to process, and the `on_not_modified()` hook is called instead. The `not_modified_count` stat counts these responses, and AutoThrottle treats them as healthy ones. + +### How It Works + +1. **Which requests**: GET and HEAD requests made with HTTP sessions (`FetcherSession`). Browser sessions fetch normally, and requests that already set their own `If-None-Match` / `If-Modified-Since` headers are left as they are. +2. **When validators are saved**: Only after the callback of a `2xx` response finished without raising. A page whose callback failed, or that was blocked, is downloaded in full on the next run. +3. **Following links**: The conditional headers are sent for that single fetch only, so requests created with `response.follow()` don't inherit them. +4. **Storage**: One small JSON file per request fingerprint in `.scrapling_validators/{spider.name}/`, or in the directory set with `conditional_requests_dir`. Delete it to make the next run download everything again. + +!!! note + + Conditional requests are disabled while `development_mode` is on, since the response cache already replays pages without fetching them, and a `304` would leave it nothing to cache. + ## Streaming For long-running spiders or applications that need real-time access to scraped items, use the `stream()` method instead of `start()`: @@ -357,6 +388,7 @@ print(f"Offsite filtered: {stats.offsite_requests_count}") print(f"Robots.txt disallowed: {stats.robots_disallowed_count}") print(f"Cache hits: {stats.cache_hits}") print(f"Cache misses: {stats.cache_misses}") +print(f"Not modified: {stats.not_modified_count}") print(f"Items scraped: {stats.items_scraped}") print(f"Items dropped: {stats.items_dropped}") print(f"Response bytes: {stats.response_bytes}") diff --git a/scrapling/spiders/conditional.py b/scrapling/spiders/conditional.py new file mode 100644 index 000000000..15b1bab64 --- /dev/null +++ b/scrapling/spiders/conditional.py @@ -0,0 +1,83 @@ +from pathlib import Path + +import orjson +import anyio +from anyio import Path as AsyncPath + +from scrapling.core.utils import log +from scrapling.core._types import Any, Dict, Mapping, Optional +from scrapling.engines.toolbelt.custom import Response + +# Headers that make a request conditional; a request that already sends one is left to its author +CONDITIONAL_HEADERS = frozenset({"if-none-match", "if-modified-since", "if-match", "if-unmodified-since", "if-range"}) + + +def _get_header(headers: Mapping[str, Any], name: str) -> Optional[str]: + """Case-insensitive header lookup, since engines don't agree on the case of header names.""" + for key, value in headers.items(): + if key.lower() == name: + return str(value) + return None + + +class ConditionalRequestManager: + """Saves the validators (`ETag` / `Last-Modified`) of processed responses to disk, so the next run can ask the + website whether each page changed instead of downloading it again.""" + + def __init__(self, validators_dir: str | Path): + self._validators_dir = AsyncPath(validators_dir) + + def _validators_path(self, fingerprint: bytes) -> AsyncPath: + return self._validators_dir / f"{fingerprint.hex()}.json" + + async def get_headers(self, fingerprint: bytes) -> Dict[str, str]: + """Return the `If-None-Match` / `If-Modified-Since` headers for a request, empty if nothing was saved for it.""" + path = self._validators_path(fingerprint) + if not await path.exists(): + return {} + + try: + async with await anyio.open_file(path, "rb") as f: + data: Dict[str, Any] = orjson.loads(await f.read()) + except Exception as e: + log.warning(f"Failed to read saved validators for {fingerprint.hex()}: {e}") + return {} + + headers = {} + if data.get("etag"): + headers["If-None-Match"] = data["etag"] + if data.get("last_modified"): + headers["If-Modified-Since"] = data["last_modified"] + return headers + + async def put(self, fingerprint: bytes, response: Response) -> None: + """Save the response's validators, or drop the saved ones when the website stopped sending any.""" + path = self._validators_path(fingerprint) + headers = response.headers or {} + etag = _get_header(headers, "etag") + last_modified = _get_header(headers, "last-modified") + if not etag and not last_modified: + if await path.exists(): + await path.unlink() + return + + await self._validators_dir.mkdir(parents=True, exist_ok=True) + temp_path = path.with_suffix(".tmp") + try: + serialized = orjson.dumps({"url": response.url, "etag": etag, "last_modified": last_modified}) + async with await anyio.open_file(temp_path, "wb") as f: + await f.write(serialized) + + await temp_path.replace(path) + except Exception as e: + if await temp_path.exists(): + await temp_path.unlink() + log.warning(f"Failed to save validators for {fingerprint.hex()}: {e}") + + async def clear(self) -> None: + if not await self._validators_dir.exists(): + return + async for entry in self._validators_dir.iterdir(): + if entry.suffix == ".json": + await entry.unlink() + log.info(f"Cleared saved validators at {self._validators_dir}") diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py index b9eade19b..817d36058 100644 --- a/scrapling/spiders/engine.py +++ b/scrapling/spiders/engine.py @@ -15,6 +15,7 @@ from scrapling.spiders.result import CrawlStats, ItemList from scrapling.spiders.throttle import AutoThrottle, parse_retry_after from scrapling.spiders.cache import ResponseCacheManager +from scrapling.spiders.conditional import ConditionalRequestManager, CONDITIONAL_HEADERS from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData from scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator @@ -62,6 +63,15 @@ async def _fetch_robots(url: str, sid: str) -> Response: else: self._cache_manager = None + self._conditional_manager: Optional[ConditionalRequestManager] = None + if getattr(self.spider, "conditional_requests", False): + if self._cache_manager: + # A cache miss needs the full body to cache it, so a 304 would leave nothing to replay + log.warning("Conditional requests are disabled in development mode, the response cache is used instead") + else: + validators_dir = self.spider.conditional_requests_dir or f".scrapling_validators/{self.spider.name}" + self._conditional_manager = ConditionalRequestManager(validators_dir) + if self.spider.autothrottle_enabled: self._autothrottle: Optional[AutoThrottle] = AutoThrottle( start_delay=self.spider.autothrottle_start_delay, @@ -152,8 +162,19 @@ def _normalize_request(self, request: Request) -> None: if not request.sid: request.sid = self.session_manager.default_session_id - async def _run_callbacks(self, request: Request, response: Response) -> None: - """Dispatch response to the request's callback and process yielded items/requests.""" + @staticmethod + def _uses_validators(request: Request) -> bool: + """Whether conditional requests apply to this request: only GET/HEAD ones that don't set their own validators.""" + if str(request._session_kwargs.get("method", "GET")).upper() not in ("GET", "HEAD"): + return False + headers = request._session_kwargs.get("headers") or {} + return not any(key.lower() in CONDITIONAL_HEADERS for key in headers) + + async def _run_callbacks(self, request: Request, response: Response) -> bool: + """Dispatch response to the request's callback and process yielded items/requests. + + Returns whether the callback finished without raising. + """ callback = request.callback if request.callback else self.spider.parse try: async for result in callback(response): @@ -182,6 +203,8 @@ async def _run_callbacks(self, request: Request, response: Response) -> None: msg = f"Spider error processing {request}:\n {e}" log.error(msg, exc_info=e) await self.spider.on_error(request, e) + return False + return True async def _process_request(self, request: Request) -> None: """Download and process a single request.""" @@ -211,6 +234,10 @@ async def _process_request(self, request: Request) -> None: await self._run_callbacks(request, cached) return + conditional_headers: Dict[str, str] = {} + if self._conditional_manager and request._fp is not None and self._uses_validators(request): + conditional_headers = await self._conditional_manager.get_headers(request._fp) + async with self._rate_limiter(request.domain): if self._autothrottle: delay = self._autothrottle.delay_for(request.domain, floor) @@ -224,7 +251,7 @@ async def _process_request(self, request: Request) -> None: self.stats.proxies.append(dict(request._session_kwargs["proxies"])) try: started_at = anyio.current_time() - response = await self.session_manager.fetch(request) + response = await self.session_manager.fetch(request, headers=conditional_headers) latency = anyio.current_time() - started_at self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id) self.stats.increment_response_bytes(request.domain, len(response.body)) @@ -238,6 +265,15 @@ async def _process_request(self, request: Request) -> None: if self._cache_manager and request._fp is not None: self.stats.cache_misses += 1 + if conditional_headers and response.status == 304: + # Nothing changed since a previous run processed this page, so the callback has nothing new to work on + self.stats.not_modified_count += 1 + if self._autothrottle: + self._autothrottle.record(request.domain, latency, True, floor) + log.debug(f"Not modified since the last run: {request.url}") + await self.spider.on_not_modified(request, response) + return + blocked = await self.spider.is_blocked(response) if self._autothrottle: ok = 200 <= response.status < 300 and not blocked @@ -267,7 +303,16 @@ async def _process_request(self, request: Request) -> None: if self._cache_manager and request._fp is not None: await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET")) - await self._run_callbacks(request, response) + processed = await self._run_callbacks(request, response) + # Validators are saved only once the callback succeeded, so a page that failed is downloaded in full next time + if ( + processed + and self._conditional_manager + and request._fp is not None + and 200 <= response.status < 300 + and self._uses_validators(request) + ): + await self._conditional_manager.put(request._fp, response) async def _task_wrapper(self, request: Request) -> None: """Wrapper to track active task count.""" diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py index 29bda89e3..43afcbfe7 100644 --- a/scrapling/spiders/result.py +++ b/scrapling/spiders/result.py @@ -130,6 +130,7 @@ class CrawlStats: robots_disallowed_count: int = 0 cache_hits: int = 0 cache_misses: int = 0 + not_modified_count: int = 0 response_bytes: int = 0 items_scraped: int = 0 items_dropped: int = 0 @@ -185,6 +186,7 @@ def to_dict(self) -> dict[str, Any]: "robots_disallowed_count": self.robots_disallowed_count, "cache_hits": self.cache_hits, "cache_misses": self.cache_misses, + "not_modified_count": self.not_modified_count, "blocked_requests_count": self.blocked_requests_count, "response_status_count": self.response_status_count, "response_bytes": self.response_bytes, diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py index 89e008b68..7ea088373 100644 --- a/scrapling/spiders/session.py +++ b/scrapling/spiders/session.py @@ -3,7 +3,7 @@ from scrapling.spiders.request import Request from scrapling.engines.static import _ASyncSessionLogic from scrapling.engines.toolbelt.convertor import Response -from scrapling.core._types import Set, cast, SUPPORTED_HTTP_METHODS +from scrapling.core._types import Dict, Optional, Set, cast, SUPPORTED_HTTP_METHODS from scrapling.fetchers import AsyncStealthySession, FetcherSession Session = FetcherSession | AsyncStealthySession @@ -100,7 +100,13 @@ async def close(self) -> None: self._started = False - async def fetch(self, request: Request) -> Response: + async def fetch(self, request: Request, headers: Optional[Dict[str, str]] = None) -> Response: + """Fetch a request with the session its `sid` points to. + + :param request: The request to fetch. + :param headers: Headers to add for this fetch only. They aren't saved on the request, so requests followed + from the response don't inherit them. Browser sessions read `extra_headers` instead, so they ignore these. + """ sid = request.sid if request.sid else self.default_session_id session = self.get(sid) @@ -110,11 +116,15 @@ async def fetch(self, request: Request) -> Response: if not session._is_alive: await session.__aenter__() + session_kwargs = request._session_kwargs + if headers: + session_kwargs = {**session_kwargs, "headers": {**(session_kwargs.get("headers") or {}), **headers}} + if isinstance(session, FetcherSession): client = session._client if isinstance(client, _ASyncSessionLogic): - kwargs = request._session_kwargs.copy() + kwargs = session_kwargs.copy() method = cast(SUPPORTED_HTTP_METHODS, kwargs.pop("method", "GET")) if "stealthy_headers" in kwargs: # Same mapping as the public session methods (`get`, `post`, ...), so it doesn't reach curl_cffi @@ -128,7 +138,7 @@ async def fetch(self, request: Request) -> Response: # Sync session or other types - shouldn't happen in async context raise TypeError(f"Session type {type(client)} not supported for async fetch") else: - response = await session.fetch(url=request.url, **request._session_kwargs) + response = await session.fetch(url=request.url, **session_kwargs) response.request = request # Merge request meta into response meta (response meta takes priority) diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py index 48750772e..3f4ff5d62 100644 --- a/scrapling/spiders/spider.py +++ b/scrapling/spiders/spider.py @@ -79,6 +79,10 @@ class Spider(ABC): development_mode: bool = False development_cache_dir: Optional[str] = None + # Conditional requests (ETag / Last-Modified validators saved between runs) + conditional_requests: bool = False + conditional_requests_dir: Optional[str] = None + # Concurrency settings concurrent_requests: int = 4 concurrent_requests_per_domain: int = 0 @@ -197,6 +201,14 @@ async def on_error(self, request: Request, error: Exception) -> None: """ pass + async def on_not_modified(self, request: Request, response: "Response") -> None: + """Called instead of the request's callback when `conditional_requests` is enabled and the website answered + `304 Not Modified`, meaning the page didn't change since a previous run processed it. + + Override to act on unchanged pages. + """ + pass + async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None: """A hook to be overridden by users to do some processing on scraped items, return `None` to drop the item silently.""" return item diff --git a/tests/spiders/test_conditional.py b/tests/spiders/test_conditional.py new file mode 100644 index 000000000..7788afb89 --- /dev/null +++ b/tests/spiders/test_conditional.py @@ -0,0 +1,418 @@ +"""Tests for conditional requests (`ETag` / `Last-Modified` validators saved between runs).""" + +from pathlib import Path + +import pytest + +from scrapling.spiders.conditional import ConditionalRequestManager +from scrapling.spiders.engine import CrawlerEngine +from scrapling.spiders.request import Request +from scrapling.spiders.session import SessionManager +from scrapling.spiders.spider import Spider +from scrapling.spiders.throttle import AutoThrottle +from scrapling.engines.toolbelt.custom import Response +from scrapling.core._types import Any, Dict, List, Optional, AsyncGenerator + + +def _make_response( + url: str = "https://example.com/feed", + status: int = 200, + body: bytes = b"", + headers: Optional[Dict[str, str]] = None, +) -> Response: + return Response( + url=url, + content=body, + status=status, + reason="", + cookies={}, + headers=headers or {}, + request_headers={}, + _log=False, + ) + + +class FakeSite: + """A website that answers conditional requests like a real server would. + + `pages` maps each URL to its `(body, etag, last_modified)`; either validator can be `None`. + """ + + def __init__(self, pages: Dict[str, tuple]): + self.pages = dict(pages) + self.sent_headers: List[Dict[str, str]] = [] + + def respond(self, url: str, headers: Dict[str, str]) -> Response: + self.sent_headers.append(dict(headers)) + body, etag, last_modified = self.pages[url] + sent = {key.lower(): value for key, value in headers.items()} + response_headers = {} + if etag: + response_headers["ETag"] = etag + if last_modified: + response_headers["Last-Modified"] = last_modified + + # If-None-Match wins over If-Modified-Since when both are sent (RFC 9110, section 13.2.2) + if "if-none-match" in sent: + unchanged = sent["if-none-match"] == etag + else: + unchanged = "if-modified-since" in sent and sent["if-modified-since"] == last_modified + + if unchanged: + return _make_response(url, status=304, body=b"", headers=response_headers) + return _make_response(url, body=body, headers=response_headers) + + +class FakeSession: + def __init__(self, site: FakeSite): + self.site = site + self._is_alive = False + + async def __aenter__(self): + self._is_alive = True + return self + + async def __aexit__(self, *args): + self._is_alive = False + + async def fetch(self, url: str, **kwargs: Any) -> Response: + return self.site.respond(url, kwargs.get("headers") or {}) + + +class MonitorSpider(Spider): + name = "monitor" + start_urls = ["https://example.com/feed"] + conditional_requests = True + + def __init__(self, site: FakeSite, validators_dir: Path, **kwargs: Any): + self.site = site + self.conditional_requests_dir = str(validators_dir) + self.not_modified: List[str] = [] + self.parsed: List[str] = [] + super().__init__(**kwargs) + + def configure_sessions(self, manager: SessionManager) -> None: + manager.add("default", FakeSession(self.site)) + + async def parse(self, response: Response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]: + self.parsed.append(response.url) + yield {"url": response.url, "body": response.body} + + async def on_not_modified(self, request: Request, response: Response) -> None: + self.not_modified.append(request.url) + + +FEED = {"https://example.com/feed": (b"v1", '"v1"', "Sat, 24 Oct 2026 10:00:00 GMT")} + + +class TestConditionalRequestManager: + @pytest.mark.asyncio + async def test_saved_validators_become_conditional_headers(self, tmp_path: Path): + manager = ConditionalRequestManager(tmp_path) + fp = b"\x01" * 20 + response = _make_response(headers={"ETag": 'W/"abc"', "Last-Modified": "Sat, 24 Oct 2026 10:00:00 GMT"}) + + await manager.put(fp, response) + assert await manager.get_headers(fp) == { + "If-None-Match": 'W/"abc"', + "If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT", + } + + @pytest.mark.asyncio + async def test_header_names_are_matched_in_any_case(self, tmp_path: Path): + manager = ConditionalRequestManager(tmp_path) + fp = b"\x02" * 20 + + await manager.put(fp, _make_response(headers={"etag": '"lower"'})) + assert await manager.get_headers(fp) == {"If-None-Match": '"lower"'} + + @pytest.mark.asyncio + async def test_only_last_modified(self, tmp_path: Path): + manager = ConditionalRequestManager(tmp_path) + fp = b"\x03" * 20 + + await manager.put(fp, _make_response(headers={"last-modified": "Sat, 24 Oct 2026 10:00:00 GMT"})) + assert await manager.get_headers(fp) == {"If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT"} + + @pytest.mark.asyncio + async def test_response_without_validators_drops_the_saved_ones(self, tmp_path: Path): + manager = ConditionalRequestManager(tmp_path) + fp = b"\x04" * 20 + + await manager.put(fp, _make_response(headers={"ETag": '"old"'})) + await manager.put(fp, _make_response(headers={})) + assert await manager.get_headers(fp) == {} + assert not list(tmp_path.glob("*.json")) + + @pytest.mark.asyncio + async def test_response_without_validators_writes_nothing(self, tmp_path: Path): + validators_dir = tmp_path / "validators" + manager = ConditionalRequestManager(validators_dir) + + await manager.put(b"\x05" * 20, _make_response(headers={"content-type": "text/html"})) + assert not validators_dir.exists() + + @pytest.mark.asyncio + async def test_put_overwrites(self, tmp_path: Path): + manager = ConditionalRequestManager(tmp_path) + fp = b"\x06" * 20 + + await manager.put(fp, _make_response(headers={"ETag": '"first"'})) + await manager.put(fp, _make_response(headers={"ETag": '"second"'})) + assert await manager.get_headers(fp) == {"If-None-Match": '"second"'} + + @pytest.mark.asyncio + async def test_missing_and_corrupt_entries_give_no_headers(self, tmp_path: Path): + manager = ConditionalRequestManager(tmp_path) + assert await manager.get_headers(b"\x07" * 20) == {} + + fp = b"\x08" * 20 + (tmp_path / f"{fp.hex()}.json").write_text("not json{{") + assert await manager.get_headers(fp) == {} + + @pytest.mark.asyncio + async def test_clear(self, tmp_path: Path): + manager = ConditionalRequestManager(tmp_path) + fp = b"\x09" * 20 + + await manager.put(fp, _make_response(headers={"ETag": '"x"'})) + await manager.clear() + assert await manager.get_headers(fp) == {} + + +class TestConditionalCrawl: + def test_second_run_skips_the_unchanged_page(self, tmp_path: Path): + site = FakeSite(FEED) + + first = MonitorSpider(site, tmp_path) + result = first.start() + assert site.sent_headers == [{}] + assert first.parsed == ["https://example.com/feed"] + assert result.stats.not_modified_count == 0 + + second = MonitorSpider(site, tmp_path) + result = second.start() + assert site.sent_headers[1] == { + "If-None-Match": '"v1"', + "If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT", + } + assert second.parsed == [] + assert second.not_modified == ["https://example.com/feed"] + assert result.stats.not_modified_count == 1 + assert result.stats.response_status_count == {"status_304": 1} + assert len(result.items) == 0 + + def test_changed_page_is_processed_and_its_new_validators_saved(self, tmp_path: Path): + site = FakeSite(FEED) + MonitorSpider(site, tmp_path).start() + + site.pages["https://example.com/feed"] = (b"v2", '"v2"', None) + changed = MonitorSpider(site, tmp_path) + result = changed.start() + assert [item["body"] for item in result.items] == [b"v2"] + assert changed.not_modified == [] + + MonitorSpider(site, tmp_path).start() + assert site.sent_headers[-1] == {"If-None-Match": '"v2"'} + + def test_last_modified_alone_is_enough(self, tmp_path: Path): + site = FakeSite({"https://example.com/feed": (b"", None, "Sat, 24 Oct 2026 10:00:00 GMT")}) + MonitorSpider(site, tmp_path).start() + + second = MonitorSpider(site, tmp_path) + second.start() + assert site.sent_headers[-1] == {"If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT"} + assert second.not_modified == ["https://example.com/feed"] + + def test_failed_callback_keeps_the_page_unconditional(self, tmp_path: Path): + class FailingSpider(MonitorSpider): + async def parse(self, response: Response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]: + raise RuntimeError("parser broke") + yield + + site = FakeSite(FEED) + FailingSpider(site, tmp_path).start() + + retry = MonitorSpider(site, tmp_path) + retry.start() + assert site.sent_headers == [{}, {}] + assert retry.parsed == ["https://example.com/feed"] + + def test_blocked_response_is_not_remembered(self, tmp_path: Path): + class BlockedSpider(MonitorSpider): + max_blocked_retries = 0 + + async def is_blocked(self, response: Response) -> bool: + return True + + site = FakeSite(FEED) + BlockedSpider(site, tmp_path).start() + MonitorSpider(site, tmp_path).start() + assert site.sent_headers == [{}, {}] + + def test_followed_requests_dont_inherit_the_validators(self, tmp_path: Path): + pages = { + **FEED, + "https://example.com/article": (b"article", None, None), + } + + class FollowingSpider(MonitorSpider): + async def parse(self, response: Response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]: + self.parsed.append(response.url) + if response.url.endswith("/feed"): + yield response.follow("/article", dont_filter=True) + + site = FakeSite(pages) + FollowingSpider(site, tmp_path).start() + + # The feed changed, so the second run fetches the article from it again + site.pages["https://example.com/feed"] = (b"v2", '"v2"', None) + FollowingSpider(site, tmp_path).start() + article_headers = site.sent_headers[-1] + assert {key.lower() for key in article_headers} == {"referer"} + + def test_request_kwargs_are_left_untouched(self, tmp_path: Path): + shared_headers = {"X-Token": "abc"} + + class HeadersSpider(MonitorSpider): + async def start_requests(self) -> AsyncGenerator[Request, None]: + yield Request("https://example.com/feed", sid="default", headers=shared_headers) + + site = FakeSite(FEED) + HeadersSpider(site, tmp_path).start() + HeadersSpider(site, tmp_path).start() + assert site.sent_headers[-1] == { + "X-Token": "abc", + "If-None-Match": '"v1"', + "If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT", + } + assert shared_headers == {"X-Token": "abc"} + + def test_requests_with_their_own_validators_are_left_alone(self, tmp_path: Path): + class OwnValidatorsSpider(MonitorSpider): + async def start_requests(self) -> AsyncGenerator[Request, None]: + yield Request("https://example.com/feed", sid="default", headers={"If-None-Match": '"mine"'}) + + site = FakeSite(FEED) + spider = OwnValidatorsSpider(site, tmp_path) + spider.start() + assert site.sent_headers == [{"If-None-Match": '"mine"'}] + assert spider.parsed == ["https://example.com/feed"] + assert not list(tmp_path.glob("*.json")) + + def test_post_requests_are_never_conditional(self, tmp_path: Path): + class PostSpider(MonitorSpider): + async def start_requests(self) -> AsyncGenerator[Request, None]: + yield Request("https://example.com/feed", sid="default", method="POST", data={"q": "x"}) + + site = FakeSite(FEED) + PostSpider(site, tmp_path).start() + PostSpider(site, tmp_path).start() + assert site.sent_headers == [{}, {}] + + def test_disabled_by_default(self, tmp_path: Path): + class PlainSpider(MonitorSpider): + conditional_requests = False + + site = FakeSite(FEED) + PlainSpider(site, tmp_path / "validators").start() + PlainSpider(site, tmp_path / "validators").start() + assert site.sent_headers == [{}, {}] + assert not (tmp_path / "validators").exists() + + def test_not_modified_counts_as_healthy_for_autothrottle(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + recorded: List[bool] = [] + original_record = AutoThrottle.record + + def record(self: AutoThrottle, domain: str, latency: float, ok: bool, *args: Any) -> float: + recorded.append(ok) + return original_record(self, domain, latency, ok, *args) + + monkeypatch.setattr(AutoThrottle, "record", record) + + class ThrottledSpider(MonitorSpider): + autothrottle_enabled = True + autothrottle_start_delay = 0.01 + autothrottle_max_delay = 1.0 + + site = FakeSite(FEED) + ThrottledSpider(site, tmp_path).start() + result = ThrottledSpider(site, tmp_path).start() + assert result.stats.not_modified_count == 1 + # A `304` is a healthy answer, so it must not make the throttle back off like a failed response would + assert recorded == [True, True] + + +class TestWithHttpSession: + """The same flow through the default `FetcherSession` against a real server.""" + + def test_not_modified_from_a_real_server(self, httpbin, tmp_path: Path): + url = f"{httpbin.url}/etag/v1" + + class HttpSpider(Spider): + name = "http_monitor" + start_urls = [url] + conditional_requests = True + conditional_requests_dir = str(tmp_path) + + def __init__(self, **kwargs: Any): + self.not_modified: List[str] = [] + super().__init__(**kwargs) + + async def parse(self, response: Response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]: + yield {"status": response.status} + + async def on_not_modified(self, request: Request, response: Response) -> None: + self.not_modified.append(request.url) + + first = HttpSpider().start() + assert [item["status"] for item in first.items] == [200] + + spider = HttpSpider() + second = spider.start() + assert second.stats.not_modified_count == 1 + assert second.stats.response_status_count == {"status_304": 1} + assert spider.not_modified == [url] + assert len(second.items) == 0 + + +class TestDevelopmentModeInteraction: + def test_disabled_while_development_mode_is_on(self, tmp_path: Path): + site = FakeSite(FEED) + + class DevSpider(MonitorSpider): + development_mode = True + + def __init__(self, *args: Any, **kwargs: Any): + self.development_cache_dir = str(tmp_path / "cache") + super().__init__(*args, **kwargs) + + spider = DevSpider(site, tmp_path / "validators") + engine = CrawlerEngine(spider, spider._session_manager) + assert engine._cache_manager is not None + assert engine._conditional_manager is None + + first = DevSpider(site, tmp_path / "validators").start() + second = DevSpider(site, tmp_path / "validators").start() + # Only the first run touched the website; the second one was replayed from the response cache + assert site.sent_headers == [{}] + assert second.stats.cache_hits == 1 + assert [item["body"] for item in first.items] == [item["body"] for item in second.items] == [b"v1"] + assert not (tmp_path / "validators").exists() + + +class TestSessionManagerHeaders: + @pytest.mark.asyncio + async def test_fetch_headers_are_merged_without_touching_the_request(self): + site = FakeSite(FEED) + manager = SessionManager() + manager.add("default", FakeSession(site)) + request = Request("https://example.com/feed", sid="default", headers={"X-Token": "abc"}) + + async with manager: + response = await manager.fetch(request, headers={"If-None-Match": '"v1"'}) + + assert site.sent_headers == [{"X-Token": "abc", "If-None-Match": '"v1"'}] + assert request._session_kwargs == {"headers": {"X-Token": "abc"}} + assert response.request is request + assert response.status == 304