diff --git a/docs/spiders/advanced.md b/docs/spiders/advanced.md
index f1814e830..e6ad6c497 100644
--- a/docs/spiders/advanced.md
+++ b/docs/spiders/advanced.md
@@ -216,6 +216,37 @@ There's no automatic expiration. To force a fresh crawl, delete the cache direct
Development mode is meant for development, not production. Cached responses never expire, and replay bypasses rate limiting and blocked-request retries. Don't ship a spider with `development_mode = True`.
+## Conditional Requests
+
+Spiders that run on a schedule, like one that checks a set of feeds every few minutes, mostly download pages that haven't changed since the last run. With `conditional_requests = True`, the spider saves the `ETag` and `Last-Modified` headers of each page it processed, and the next run sends them back as `If-None-Match` / `If-Modified-Since`. A website that supports them answers `304 Not Modified` with an empty body instead of the whole page.
+
+```python
+class FeedMonitor(Spider):
+ name = "feed_monitor"
+ start_urls = ["https://example.com/feed.xml"]
+ conditional_requests = True
+
+ async def parse(self, response: Response):
+ for item in response.css("item"):
+ yield {"title": item.css("title::text").get("")}
+
+ async def on_not_modified(self, request, response):
+ self.logger.info(f"No changes in {request.url}")
+```
+
+When the answer is `304`, the request's callback is skipped, since there's nothing new for it to process, and the `on_not_modified()` hook is called instead. The `not_modified_count` stat counts these responses, and AutoThrottle treats them as healthy ones.
+
+### How It Works
+
+1. **Which requests**: GET and HEAD requests made with HTTP sessions (`FetcherSession`). Browser sessions fetch normally, and requests that already set their own `If-None-Match` / `If-Modified-Since` headers are left as they are.
+2. **When validators are saved**: Only after the callback of a `2xx` response finished without raising. A page whose callback failed, or that was blocked, is downloaded in full on the next run.
+3. **Following links**: The conditional headers are sent for that single fetch only, so requests created with `response.follow()` don't inherit them.
+4. **Storage**: One small JSON file per request fingerprint in `.scrapling_validators/{spider.name}/`, or in the directory set with `conditional_requests_dir`. Delete it to make the next run download everything again.
+
+!!! note
+
+ Conditional requests are disabled while `development_mode` is on, since the response cache already replays pages without fetching them, and a `304` would leave it nothing to cache.
+
## Streaming
For long-running spiders or applications that need real-time access to scraped items, use the `stream()` method instead of `start()`:
@@ -357,6 +388,7 @@ print(f"Offsite filtered: {stats.offsite_requests_count}")
print(f"Robots.txt disallowed: {stats.robots_disallowed_count}")
print(f"Cache hits: {stats.cache_hits}")
print(f"Cache misses: {stats.cache_misses}")
+print(f"Not modified: {stats.not_modified_count}")
print(f"Items scraped: {stats.items_scraped}")
print(f"Items dropped: {stats.items_dropped}")
print(f"Response bytes: {stats.response_bytes}")
diff --git a/scrapling/spiders/conditional.py b/scrapling/spiders/conditional.py
new file mode 100644
index 000000000..15b1bab64
--- /dev/null
+++ b/scrapling/spiders/conditional.py
@@ -0,0 +1,83 @@
+from pathlib import Path
+
+import orjson
+import anyio
+from anyio import Path as AsyncPath
+
+from scrapling.core.utils import log
+from scrapling.core._types import Any, Dict, Mapping, Optional
+from scrapling.engines.toolbelt.custom import Response
+
+# Headers that make a request conditional; a request that already sends one is left to its author
+CONDITIONAL_HEADERS = frozenset({"if-none-match", "if-modified-since", "if-match", "if-unmodified-since", "if-range"})
+
+
+def _get_header(headers: Mapping[str, Any], name: str) -> Optional[str]:
+ """Case-insensitive header lookup, since engines don't agree on the case of header names."""
+ for key, value in headers.items():
+ if key.lower() == name:
+ return str(value)
+ return None
+
+
+class ConditionalRequestManager:
+ """Saves the validators (`ETag` / `Last-Modified`) of processed responses to disk, so the next run can ask the
+ website whether each page changed instead of downloading it again."""
+
+ def __init__(self, validators_dir: str | Path):
+ self._validators_dir = AsyncPath(validators_dir)
+
+ def _validators_path(self, fingerprint: bytes) -> AsyncPath:
+ return self._validators_dir / f"{fingerprint.hex()}.json"
+
+ async def get_headers(self, fingerprint: bytes) -> Dict[str, str]:
+ """Return the `If-None-Match` / `If-Modified-Since` headers for a request, empty if nothing was saved for it."""
+ path = self._validators_path(fingerprint)
+ if not await path.exists():
+ return {}
+
+ try:
+ async with await anyio.open_file(path, "rb") as f:
+ data: Dict[str, Any] = orjson.loads(await f.read())
+ except Exception as e:
+ log.warning(f"Failed to read saved validators for {fingerprint.hex()}: {e}")
+ return {}
+
+ headers = {}
+ if data.get("etag"):
+ headers["If-None-Match"] = data["etag"]
+ if data.get("last_modified"):
+ headers["If-Modified-Since"] = data["last_modified"]
+ return headers
+
+ async def put(self, fingerprint: bytes, response: Response) -> None:
+ """Save the response's validators, or drop the saved ones when the website stopped sending any."""
+ path = self._validators_path(fingerprint)
+ headers = response.headers or {}
+ etag = _get_header(headers, "etag")
+ last_modified = _get_header(headers, "last-modified")
+ if not etag and not last_modified:
+ if await path.exists():
+ await path.unlink()
+ return
+
+ await self._validators_dir.mkdir(parents=True, exist_ok=True)
+ temp_path = path.with_suffix(".tmp")
+ try:
+ serialized = orjson.dumps({"url": response.url, "etag": etag, "last_modified": last_modified})
+ async with await anyio.open_file(temp_path, "wb") as f:
+ await f.write(serialized)
+
+ await temp_path.replace(path)
+ except Exception as e:
+ if await temp_path.exists():
+ await temp_path.unlink()
+ log.warning(f"Failed to save validators for {fingerprint.hex()}: {e}")
+
+ async def clear(self) -> None:
+ if not await self._validators_dir.exists():
+ return
+ async for entry in self._validators_dir.iterdir():
+ if entry.suffix == ".json":
+ await entry.unlink()
+ log.info(f"Cleared saved validators at {self._validators_dir}")
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index b9eade19b..817d36058 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -15,6 +15,7 @@
from scrapling.spiders.result import CrawlStats, ItemList
from scrapling.spiders.throttle import AutoThrottle, parse_retry_after
from scrapling.spiders.cache import ResponseCacheManager
+from scrapling.spiders.conditional import ConditionalRequestManager, CONDITIONAL_HEADERS
from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData
from scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator
@@ -62,6 +63,15 @@ async def _fetch_robots(url: str, sid: str) -> Response:
else:
self._cache_manager = None
+ self._conditional_manager: Optional[ConditionalRequestManager] = None
+ if getattr(self.spider, "conditional_requests", False):
+ if self._cache_manager:
+ # A cache miss needs the full body to cache it, so a 304 would leave nothing to replay
+ log.warning("Conditional requests are disabled in development mode, the response cache is used instead")
+ else:
+ validators_dir = self.spider.conditional_requests_dir or f".scrapling_validators/{self.spider.name}"
+ self._conditional_manager = ConditionalRequestManager(validators_dir)
+
if self.spider.autothrottle_enabled:
self._autothrottle: Optional[AutoThrottle] = AutoThrottle(
start_delay=self.spider.autothrottle_start_delay,
@@ -152,8 +162,19 @@ def _normalize_request(self, request: Request) -> None:
if not request.sid:
request.sid = self.session_manager.default_session_id
- async def _run_callbacks(self, request: Request, response: Response) -> None:
- """Dispatch response to the request's callback and process yielded items/requests."""
+ @staticmethod
+ def _uses_validators(request: Request) -> bool:
+ """Whether conditional requests apply to this request: only GET/HEAD ones that don't set their own validators."""
+ if str(request._session_kwargs.get("method", "GET")).upper() not in ("GET", "HEAD"):
+ return False
+ headers = request._session_kwargs.get("headers") or {}
+ return not any(key.lower() in CONDITIONAL_HEADERS for key in headers)
+
+ async def _run_callbacks(self, request: Request, response: Response) -> bool:
+ """Dispatch response to the request's callback and process yielded items/requests.
+
+ Returns whether the callback finished without raising.
+ """
callback = request.callback if request.callback else self.spider.parse
try:
async for result in callback(response):
@@ -182,6 +203,8 @@ async def _run_callbacks(self, request: Request, response: Response) -> None:
msg = f"Spider error processing {request}:\n {e}"
log.error(msg, exc_info=e)
await self.spider.on_error(request, e)
+ return False
+ return True
async def _process_request(self, request: Request) -> None:
"""Download and process a single request."""
@@ -211,6 +234,10 @@ async def _process_request(self, request: Request) -> None:
await self._run_callbacks(request, cached)
return
+ conditional_headers: Dict[str, str] = {}
+ if self._conditional_manager and request._fp is not None and self._uses_validators(request):
+ conditional_headers = await self._conditional_manager.get_headers(request._fp)
+
async with self._rate_limiter(request.domain):
if self._autothrottle:
delay = self._autothrottle.delay_for(request.domain, floor)
@@ -224,7 +251,7 @@ async def _process_request(self, request: Request) -> None:
self.stats.proxies.append(dict(request._session_kwargs["proxies"]))
try:
started_at = anyio.current_time()
- response = await self.session_manager.fetch(request)
+ response = await self.session_manager.fetch(request, headers=conditional_headers)
latency = anyio.current_time() - started_at
self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
self.stats.increment_response_bytes(request.domain, len(response.body))
@@ -238,6 +265,15 @@ async def _process_request(self, request: Request) -> None:
if self._cache_manager and request._fp is not None:
self.stats.cache_misses += 1
+ if conditional_headers and response.status == 304:
+ # Nothing changed since a previous run processed this page, so the callback has nothing new to work on
+ self.stats.not_modified_count += 1
+ if self._autothrottle:
+ self._autothrottle.record(request.domain, latency, True, floor)
+ log.debug(f"Not modified since the last run: {request.url}")
+ await self.spider.on_not_modified(request, response)
+ return
+
blocked = await self.spider.is_blocked(response)
if self._autothrottle:
ok = 200 <= response.status < 300 and not blocked
@@ -267,7 +303,16 @@ async def _process_request(self, request: Request) -> None:
if self._cache_manager and request._fp is not None:
await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET"))
- await self._run_callbacks(request, response)
+ processed = await self._run_callbacks(request, response)
+ # Validators are saved only once the callback succeeded, so a page that failed is downloaded in full next time
+ if (
+ processed
+ and self._conditional_manager
+ and request._fp is not None
+ and 200 <= response.status < 300
+ and self._uses_validators(request)
+ ):
+ await self._conditional_manager.put(request._fp, response)
async def _task_wrapper(self, request: Request) -> None:
"""Wrapper to track active task count."""
diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py
index 29bda89e3..43afcbfe7 100644
--- a/scrapling/spiders/result.py
+++ b/scrapling/spiders/result.py
@@ -130,6 +130,7 @@ class CrawlStats:
robots_disallowed_count: int = 0
cache_hits: int = 0
cache_misses: int = 0
+ not_modified_count: int = 0
response_bytes: int = 0
items_scraped: int = 0
items_dropped: int = 0
@@ -185,6 +186,7 @@ def to_dict(self) -> dict[str, Any]:
"robots_disallowed_count": self.robots_disallowed_count,
"cache_hits": self.cache_hits,
"cache_misses": self.cache_misses,
+ "not_modified_count": self.not_modified_count,
"blocked_requests_count": self.blocked_requests_count,
"response_status_count": self.response_status_count,
"response_bytes": self.response_bytes,
diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py
index 89e008b68..7ea088373 100644
--- a/scrapling/spiders/session.py
+++ b/scrapling/spiders/session.py
@@ -3,7 +3,7 @@
from scrapling.spiders.request import Request
from scrapling.engines.static import _ASyncSessionLogic
from scrapling.engines.toolbelt.convertor import Response
-from scrapling.core._types import Set, cast, SUPPORTED_HTTP_METHODS
+from scrapling.core._types import Dict, Optional, Set, cast, SUPPORTED_HTTP_METHODS
from scrapling.fetchers import AsyncStealthySession, FetcherSession
Session = FetcherSession | AsyncStealthySession
@@ -100,7 +100,13 @@ async def close(self) -> None:
self._started = False
- async def fetch(self, request: Request) -> Response:
+ async def fetch(self, request: Request, headers: Optional[Dict[str, str]] = None) -> Response:
+ """Fetch a request with the session its `sid` points to.
+
+ :param request: The request to fetch.
+ :param headers: Headers to add for this fetch only. They aren't saved on the request, so requests followed
+ from the response don't inherit them. Browser sessions read `extra_headers` instead, so they ignore these.
+ """
sid = request.sid if request.sid else self.default_session_id
session = self.get(sid)
@@ -110,11 +116,15 @@ async def fetch(self, request: Request) -> Response:
if not session._is_alive:
await session.__aenter__()
+ session_kwargs = request._session_kwargs
+ if headers:
+ session_kwargs = {**session_kwargs, "headers": {**(session_kwargs.get("headers") or {}), **headers}}
+
if isinstance(session, FetcherSession):
client = session._client
if isinstance(client, _ASyncSessionLogic):
- kwargs = request._session_kwargs.copy()
+ kwargs = session_kwargs.copy()
method = cast(SUPPORTED_HTTP_METHODS, kwargs.pop("method", "GET"))
if "stealthy_headers" in kwargs:
# Same mapping as the public session methods (`get`, `post`, ...), so it doesn't reach curl_cffi
@@ -128,7 +138,7 @@ async def fetch(self, request: Request) -> Response:
# Sync session or other types - shouldn't happen in async context
raise TypeError(f"Session type {type(client)} not supported for async fetch")
else:
- response = await session.fetch(url=request.url, **request._session_kwargs)
+ response = await session.fetch(url=request.url, **session_kwargs)
response.request = request
# Merge request meta into response meta (response meta takes priority)
diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py
index 48750772e..3f4ff5d62 100644
--- a/scrapling/spiders/spider.py
+++ b/scrapling/spiders/spider.py
@@ -79,6 +79,10 @@ class Spider(ABC):
development_mode: bool = False
development_cache_dir: Optional[str] = None
+ # Conditional requests (ETag / Last-Modified validators saved between runs)
+ conditional_requests: bool = False
+ conditional_requests_dir: Optional[str] = None
+
# Concurrency settings
concurrent_requests: int = 4
concurrent_requests_per_domain: int = 0
@@ -197,6 +201,14 @@ async def on_error(self, request: Request, error: Exception) -> None:
"""
pass
+ async def on_not_modified(self, request: Request, response: "Response") -> None:
+ """Called instead of the request's callback when `conditional_requests` is enabled and the website answered
+ `304 Not Modified`, meaning the page didn't change since a previous run processed it.
+
+ Override to act on unchanged pages.
+ """
+ pass
+
async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None:
"""A hook to be overridden by users to do some processing on scraped items, return `None` to drop the item silently."""
return item
diff --git a/tests/spiders/test_conditional.py b/tests/spiders/test_conditional.py
new file mode 100644
index 000000000..7788afb89
--- /dev/null
+++ b/tests/spiders/test_conditional.py
@@ -0,0 +1,418 @@
+"""Tests for conditional requests (`ETag` / `Last-Modified` validators saved between runs)."""
+
+from pathlib import Path
+
+import pytest
+
+from scrapling.spiders.conditional import ConditionalRequestManager
+from scrapling.spiders.engine import CrawlerEngine
+from scrapling.spiders.request import Request
+from scrapling.spiders.session import SessionManager
+from scrapling.spiders.spider import Spider
+from scrapling.spiders.throttle import AutoThrottle
+from scrapling.engines.toolbelt.custom import Response
+from scrapling.core._types import Any, Dict, List, Optional, AsyncGenerator
+
+
+def _make_response(
+ url: str = "https://example.com/feed",
+ status: int = 200,
+ body: bytes = b"",
+ headers: Optional[Dict[str, str]] = None,
+) -> Response:
+ return Response(
+ url=url,
+ content=body,
+ status=status,
+ reason="",
+ cookies={},
+ headers=headers or {},
+ request_headers={},
+ _log=False,
+ )
+
+
+class FakeSite:
+ """A website that answers conditional requests like a real server would.
+
+ `pages` maps each URL to its `(body, etag, last_modified)`; either validator can be `None`.
+ """
+
+ def __init__(self, pages: Dict[str, tuple]):
+ self.pages = dict(pages)
+ self.sent_headers: List[Dict[str, str]] = []
+
+ def respond(self, url: str, headers: Dict[str, str]) -> Response:
+ self.sent_headers.append(dict(headers))
+ body, etag, last_modified = self.pages[url]
+ sent = {key.lower(): value for key, value in headers.items()}
+ response_headers = {}
+ if etag:
+ response_headers["ETag"] = etag
+ if last_modified:
+ response_headers["Last-Modified"] = last_modified
+
+ # If-None-Match wins over If-Modified-Since when both are sent (RFC 9110, section 13.2.2)
+ if "if-none-match" in sent:
+ unchanged = sent["if-none-match"] == etag
+ else:
+ unchanged = "if-modified-since" in sent and sent["if-modified-since"] == last_modified
+
+ if unchanged:
+ return _make_response(url, status=304, body=b"", headers=response_headers)
+ return _make_response(url, body=body, headers=response_headers)
+
+
+class FakeSession:
+ def __init__(self, site: FakeSite):
+ self.site = site
+ self._is_alive = False
+
+ async def __aenter__(self):
+ self._is_alive = True
+ return self
+
+ async def __aexit__(self, *args):
+ self._is_alive = False
+
+ async def fetch(self, url: str, **kwargs: Any) -> Response:
+ return self.site.respond(url, kwargs.get("headers") or {})
+
+
+class MonitorSpider(Spider):
+ name = "monitor"
+ start_urls = ["https://example.com/feed"]
+ conditional_requests = True
+
+ def __init__(self, site: FakeSite, validators_dir: Path, **kwargs: Any):
+ self.site = site
+ self.conditional_requests_dir = str(validators_dir)
+ self.not_modified: List[str] = []
+ self.parsed: List[str] = []
+ super().__init__(**kwargs)
+
+ def configure_sessions(self, manager: SessionManager) -> None:
+ manager.add("default", FakeSession(self.site))
+
+ async def parse(self, response: Response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:
+ self.parsed.append(response.url)
+ yield {"url": response.url, "body": response.body}
+
+ async def on_not_modified(self, request: Request, response: Response) -> None:
+ self.not_modified.append(request.url)
+
+
+FEED = {"https://example.com/feed": (b"v1", '"v1"', "Sat, 24 Oct 2026 10:00:00 GMT")}
+
+
+class TestConditionalRequestManager:
+ @pytest.mark.asyncio
+ async def test_saved_validators_become_conditional_headers(self, tmp_path: Path):
+ manager = ConditionalRequestManager(tmp_path)
+ fp = b"\x01" * 20
+ response = _make_response(headers={"ETag": 'W/"abc"', "Last-Modified": "Sat, 24 Oct 2026 10:00:00 GMT"})
+
+ await manager.put(fp, response)
+ assert await manager.get_headers(fp) == {
+ "If-None-Match": 'W/"abc"',
+ "If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT",
+ }
+
+ @pytest.mark.asyncio
+ async def test_header_names_are_matched_in_any_case(self, tmp_path: Path):
+ manager = ConditionalRequestManager(tmp_path)
+ fp = b"\x02" * 20
+
+ await manager.put(fp, _make_response(headers={"etag": '"lower"'}))
+ assert await manager.get_headers(fp) == {"If-None-Match": '"lower"'}
+
+ @pytest.mark.asyncio
+ async def test_only_last_modified(self, tmp_path: Path):
+ manager = ConditionalRequestManager(tmp_path)
+ fp = b"\x03" * 20
+
+ await manager.put(fp, _make_response(headers={"last-modified": "Sat, 24 Oct 2026 10:00:00 GMT"}))
+ assert await manager.get_headers(fp) == {"If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT"}
+
+ @pytest.mark.asyncio
+ async def test_response_without_validators_drops_the_saved_ones(self, tmp_path: Path):
+ manager = ConditionalRequestManager(tmp_path)
+ fp = b"\x04" * 20
+
+ await manager.put(fp, _make_response(headers={"ETag": '"old"'}))
+ await manager.put(fp, _make_response(headers={}))
+ assert await manager.get_headers(fp) == {}
+ assert not list(tmp_path.glob("*.json"))
+
+ @pytest.mark.asyncio
+ async def test_response_without_validators_writes_nothing(self, tmp_path: Path):
+ validators_dir = tmp_path / "validators"
+ manager = ConditionalRequestManager(validators_dir)
+
+ await manager.put(b"\x05" * 20, _make_response(headers={"content-type": "text/html"}))
+ assert not validators_dir.exists()
+
+ @pytest.mark.asyncio
+ async def test_put_overwrites(self, tmp_path: Path):
+ manager = ConditionalRequestManager(tmp_path)
+ fp = b"\x06" * 20
+
+ await manager.put(fp, _make_response(headers={"ETag": '"first"'}))
+ await manager.put(fp, _make_response(headers={"ETag": '"second"'}))
+ assert await manager.get_headers(fp) == {"If-None-Match": '"second"'}
+
+ @pytest.mark.asyncio
+ async def test_missing_and_corrupt_entries_give_no_headers(self, tmp_path: Path):
+ manager = ConditionalRequestManager(tmp_path)
+ assert await manager.get_headers(b"\x07" * 20) == {}
+
+ fp = b"\x08" * 20
+ (tmp_path / f"{fp.hex()}.json").write_text("not json{{")
+ assert await manager.get_headers(fp) == {}
+
+ @pytest.mark.asyncio
+ async def test_clear(self, tmp_path: Path):
+ manager = ConditionalRequestManager(tmp_path)
+ fp = b"\x09" * 20
+
+ await manager.put(fp, _make_response(headers={"ETag": '"x"'}))
+ await manager.clear()
+ assert await manager.get_headers(fp) == {}
+
+
+class TestConditionalCrawl:
+ def test_second_run_skips_the_unchanged_page(self, tmp_path: Path):
+ site = FakeSite(FEED)
+
+ first = MonitorSpider(site, tmp_path)
+ result = first.start()
+ assert site.sent_headers == [{}]
+ assert first.parsed == ["https://example.com/feed"]
+ assert result.stats.not_modified_count == 0
+
+ second = MonitorSpider(site, tmp_path)
+ result = second.start()
+ assert site.sent_headers[1] == {
+ "If-None-Match": '"v1"',
+ "If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT",
+ }
+ assert second.parsed == []
+ assert second.not_modified == ["https://example.com/feed"]
+ assert result.stats.not_modified_count == 1
+ assert result.stats.response_status_count == {"status_304": 1}
+ assert len(result.items) == 0
+
+ def test_changed_page_is_processed_and_its_new_validators_saved(self, tmp_path: Path):
+ site = FakeSite(FEED)
+ MonitorSpider(site, tmp_path).start()
+
+ site.pages["https://example.com/feed"] = (b"v2", '"v2"', None)
+ changed = MonitorSpider(site, tmp_path)
+ result = changed.start()
+ assert [item["body"] for item in result.items] == [b"v2"]
+ assert changed.not_modified == []
+
+ MonitorSpider(site, tmp_path).start()
+ assert site.sent_headers[-1] == {"If-None-Match": '"v2"'}
+
+ def test_last_modified_alone_is_enough(self, tmp_path: Path):
+ site = FakeSite({"https://example.com/feed": (b"", None, "Sat, 24 Oct 2026 10:00:00 GMT")})
+ MonitorSpider(site, tmp_path).start()
+
+ second = MonitorSpider(site, tmp_path)
+ second.start()
+ assert site.sent_headers[-1] == {"If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT"}
+ assert second.not_modified == ["https://example.com/feed"]
+
+ def test_failed_callback_keeps_the_page_unconditional(self, tmp_path: Path):
+ class FailingSpider(MonitorSpider):
+ async def parse(self, response: Response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:
+ raise RuntimeError("parser broke")
+ yield
+
+ site = FakeSite(FEED)
+ FailingSpider(site, tmp_path).start()
+
+ retry = MonitorSpider(site, tmp_path)
+ retry.start()
+ assert site.sent_headers == [{}, {}]
+ assert retry.parsed == ["https://example.com/feed"]
+
+ def test_blocked_response_is_not_remembered(self, tmp_path: Path):
+ class BlockedSpider(MonitorSpider):
+ max_blocked_retries = 0
+
+ async def is_blocked(self, response: Response) -> bool:
+ return True
+
+ site = FakeSite(FEED)
+ BlockedSpider(site, tmp_path).start()
+ MonitorSpider(site, tmp_path).start()
+ assert site.sent_headers == [{}, {}]
+
+ def test_followed_requests_dont_inherit_the_validators(self, tmp_path: Path):
+ pages = {
+ **FEED,
+ "https://example.com/article": (b"article", None, None),
+ }
+
+ class FollowingSpider(MonitorSpider):
+ async def parse(self, response: Response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:
+ self.parsed.append(response.url)
+ if response.url.endswith("/feed"):
+ yield response.follow("/article", dont_filter=True)
+
+ site = FakeSite(pages)
+ FollowingSpider(site, tmp_path).start()
+
+ # The feed changed, so the second run fetches the article from it again
+ site.pages["https://example.com/feed"] = (b"v2", '"v2"', None)
+ FollowingSpider(site, tmp_path).start()
+ article_headers = site.sent_headers[-1]
+ assert {key.lower() for key in article_headers} == {"referer"}
+
+ def test_request_kwargs_are_left_untouched(self, tmp_path: Path):
+ shared_headers = {"X-Token": "abc"}
+
+ class HeadersSpider(MonitorSpider):
+ async def start_requests(self) -> AsyncGenerator[Request, None]:
+ yield Request("https://example.com/feed", sid="default", headers=shared_headers)
+
+ site = FakeSite(FEED)
+ HeadersSpider(site, tmp_path).start()
+ HeadersSpider(site, tmp_path).start()
+ assert site.sent_headers[-1] == {
+ "X-Token": "abc",
+ "If-None-Match": '"v1"',
+ "If-Modified-Since": "Sat, 24 Oct 2026 10:00:00 GMT",
+ }
+ assert shared_headers == {"X-Token": "abc"}
+
+ def test_requests_with_their_own_validators_are_left_alone(self, tmp_path: Path):
+ class OwnValidatorsSpider(MonitorSpider):
+ async def start_requests(self) -> AsyncGenerator[Request, None]:
+ yield Request("https://example.com/feed", sid="default", headers={"If-None-Match": '"mine"'})
+
+ site = FakeSite(FEED)
+ spider = OwnValidatorsSpider(site, tmp_path)
+ spider.start()
+ assert site.sent_headers == [{"If-None-Match": '"mine"'}]
+ assert spider.parsed == ["https://example.com/feed"]
+ assert not list(tmp_path.glob("*.json"))
+
+ def test_post_requests_are_never_conditional(self, tmp_path: Path):
+ class PostSpider(MonitorSpider):
+ async def start_requests(self) -> AsyncGenerator[Request, None]:
+ yield Request("https://example.com/feed", sid="default", method="POST", data={"q": "x"})
+
+ site = FakeSite(FEED)
+ PostSpider(site, tmp_path).start()
+ PostSpider(site, tmp_path).start()
+ assert site.sent_headers == [{}, {}]
+
+ def test_disabled_by_default(self, tmp_path: Path):
+ class PlainSpider(MonitorSpider):
+ conditional_requests = False
+
+ site = FakeSite(FEED)
+ PlainSpider(site, tmp_path / "validators").start()
+ PlainSpider(site, tmp_path / "validators").start()
+ assert site.sent_headers == [{}, {}]
+ assert not (tmp_path / "validators").exists()
+
+ def test_not_modified_counts_as_healthy_for_autothrottle(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
+ recorded: List[bool] = []
+ original_record = AutoThrottle.record
+
+ def record(self: AutoThrottle, domain: str, latency: float, ok: bool, *args: Any) -> float:
+ recorded.append(ok)
+ return original_record(self, domain, latency, ok, *args)
+
+ monkeypatch.setattr(AutoThrottle, "record", record)
+
+ class ThrottledSpider(MonitorSpider):
+ autothrottle_enabled = True
+ autothrottle_start_delay = 0.01
+ autothrottle_max_delay = 1.0
+
+ site = FakeSite(FEED)
+ ThrottledSpider(site, tmp_path).start()
+ result = ThrottledSpider(site, tmp_path).start()
+ assert result.stats.not_modified_count == 1
+ # A `304` is a healthy answer, so it must not make the throttle back off like a failed response would
+ assert recorded == [True, True]
+
+
+class TestWithHttpSession:
+ """The same flow through the default `FetcherSession` against a real server."""
+
+ def test_not_modified_from_a_real_server(self, httpbin, tmp_path: Path):
+ url = f"{httpbin.url}/etag/v1"
+
+ class HttpSpider(Spider):
+ name = "http_monitor"
+ start_urls = [url]
+ conditional_requests = True
+ conditional_requests_dir = str(tmp_path)
+
+ def __init__(self, **kwargs: Any):
+ self.not_modified: List[str] = []
+ super().__init__(**kwargs)
+
+ async def parse(self, response: Response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:
+ yield {"status": response.status}
+
+ async def on_not_modified(self, request: Request, response: Response) -> None:
+ self.not_modified.append(request.url)
+
+ first = HttpSpider().start()
+ assert [item["status"] for item in first.items] == [200]
+
+ spider = HttpSpider()
+ second = spider.start()
+ assert second.stats.not_modified_count == 1
+ assert second.stats.response_status_count == {"status_304": 1}
+ assert spider.not_modified == [url]
+ assert len(second.items) == 0
+
+
+class TestDevelopmentModeInteraction:
+ def test_disabled_while_development_mode_is_on(self, tmp_path: Path):
+ site = FakeSite(FEED)
+
+ class DevSpider(MonitorSpider):
+ development_mode = True
+
+ def __init__(self, *args: Any, **kwargs: Any):
+ self.development_cache_dir = str(tmp_path / "cache")
+ super().__init__(*args, **kwargs)
+
+ spider = DevSpider(site, tmp_path / "validators")
+ engine = CrawlerEngine(spider, spider._session_manager)
+ assert engine._cache_manager is not None
+ assert engine._conditional_manager is None
+
+ first = DevSpider(site, tmp_path / "validators").start()
+ second = DevSpider(site, tmp_path / "validators").start()
+ # Only the first run touched the website; the second one was replayed from the response cache
+ assert site.sent_headers == [{}]
+ assert second.stats.cache_hits == 1
+ assert [item["body"] for item in first.items] == [item["body"] for item in second.items] == [b"v1"]
+ assert not (tmp_path / "validators").exists()
+
+
+class TestSessionManagerHeaders:
+ @pytest.mark.asyncio
+ async def test_fetch_headers_are_merged_without_touching_the_request(self):
+ site = FakeSite(FEED)
+ manager = SessionManager()
+ manager.add("default", FakeSession(site))
+ request = Request("https://example.com/feed", sid="default", headers={"X-Token": "abc"})
+
+ async with manager:
+ response = await manager.fetch(request, headers={"If-None-Match": '"v1"'})
+
+ assert site.sent_headers == [{"X-Token": "abc", "If-None-Match": '"v1"'}]
+ assert request._session_kwargs == {"headers": {"X-Token": "abc"}}
+ assert response.request is request
+ assert response.status == 304