feat(drivers-web-search-firecrawl): add FirecrawlWebSearchDriver

.github/workflows/docs-integration-tests.yml

@@ -57,6 +57,7 @@ jobs:
       DYNAMODB_TABLE_NAME: ${{ secrets.INTEG_DYNAMODB_TABLE_NAME }}
       ELEVEN_LABS_API_KEY: ${{ secrets.INTEG_ELEVEN_LABS_API_KEY }}
       EXA_API_KEY: ${{ secrets.INTEG_EXA_API_KEY }}
+      FIRECRAWL_API_KEY: ${{ secrets.INTEG_FIRECRAWL_API_KEY }}
       GOOGLE_API_KEY: ${{ secrets.INTEG_GOOGLE_API_KEY }}
       GOOGLE_API_SEARCH_ID: ${{ secrets.INTEG_GOOGLE_API_SEARCH_ID }}
       GOOGLE_OWNER_EMAIL: ${{ secrets.INTEG_GOOGLE_OWNER_EMAIL }}

docs/griptape-framework/drivers/src/web_search_drivers_firecrawl.py

@@ -0,0 +1,7 @@
+import os
+
+from griptape.drivers.web_search.firecrawl import FirecrawlWebSearchDriver
+
+driver = FirecrawlWebSearchDriver(api_key=os.environ["FIRECRAWL_API_KEY"])
+
+driver.search("griptape ai")

docs/griptape-framework/drivers/web-search-drivers.md

@@ -102,6 +102,19 @@ The [DuckDuckGoWebSearchDriver](../../reference/griptape/drivers/web_search/duck
 --8<-- "docs/griptape-framework/drivers/src/web_search_drivers_6.py"
 ```
 
+### Firecrawl
+
+The [FirecrawlWebSearchDriver](../../reference/griptape/drivers/web_search/firecrawl_web_search_driver.md) uses the Firecrawl Search API for web searching.
+
+!!! info
+
+    This driver requires the `drivers-web-search-firecrawl` [extra](../index.md#extras),
+    and a Firecrawl [api key](https://www.firecrawl.dev/search?utm_source=griptape&utm_medium=integration).
+
+```python
+--8<-- "docs/griptape-framework/drivers/src/web_search_drivers_firecrawl.py"
+```
+
 ### Perplexity
 
 The [PerplexityWebSearchDriver](../../reference/griptape/drivers/web_search/perplexity_web_search_driver.md) uses the [Perplexity Prompt Driver](../drivers/prompt-drivers.md#perplexity) internally.

griptape/drivers/__init__.py

@@ -85,6 +85,7 @@
 from .web_search.google import GoogleWebSearchDriver
 from .web_search.duck_duck_go import DuckDuckGoWebSearchDriver
 from .web_search.exa import ExaWebSearchDriver
+from .web_search.firecrawl import FirecrawlWebSearchDriver
 from .web_search.tavily import TavilyWebSearchDriver
 from .web_search.perplexity import PerplexityWebSearchDriver
 
@@ -190,6 +191,7 @@
     "DummyVectorStoreDriver",
     "ElevenLabsTextToSpeechDriver",
     "ExaWebSearchDriver",
+    "FirecrawlWebSearchDriver",
     "GoogleEmbeddingDriver",
     "GooglePromptDriver",
     "GoogleWebSearchDriver",
@@ -261,6 +263,7 @@
     "GoogleWebSearchDriver",
     "DuckDuckGoWebSearchDriver",
     "ExaWebSearchDriver",
+    "FirecrawlWebSearchDriver",
     "TavilyWebSearchDriver",
     "TrafilaturaWebScraperDriver",
     "VoyageAiEmbeddingDriver",

griptape/drivers/web_search/firecrawl/__init__.py

@@ -0,0 +1,3 @@
+from griptape.drivers.web_search.firecrawl_web_search_driver import FirecrawlWebSearchDriver
+
+__all__ = ["FirecrawlWebSearchDriver"]

griptape/drivers/web_search/firecrawl_web_search_driver.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from attrs import define, field
+
+from griptape.artifacts import JsonArtifact, ListArtifact
+from griptape.drivers.web_search import BaseWebSearchDriver
+from griptape.utils import import_optional_dependency
+from griptape.utils.decorators import lazy_property
+
+if TYPE_CHECKING:
+    from firecrawl import Firecrawl
+
+
+@define
+class FirecrawlWebSearchDriver(BaseWebSearchDriver):
+    """Web Search Driver for the Firecrawl Search API.
+
+    Returns web results with their title, url and description. When `scrape_options` is set in `params`,
+    each result also carries the requested page content (markdown, summary or json).
+
+    Attributes:
+        api_key: Firecrawl API key.
+        api_url: Firecrawl API base URL. Override it to use a self-hosted Firecrawl instance.
+        params: Extra keyword arguments passed to `Firecrawl.search` on every call,
+            for example `{"tbs": "qdr:w"}` or `{"scrape_options": {"formats": ["markdown"]}}`.
+    """
+
+    api_key: str = field(kw_only=True)
+    api_url: str = field(default="https://api.firecrawl.dev", kw_only=True, metadata={"serializable": True})
+    params: dict[str, Any] = field(factory=dict, kw_only=True, metadata={"serializable": True})
+    _client: Firecrawl | None = field(default=None, kw_only=True, alias="client")
+
+    @lazy_property()
+    def client(self) -> Firecrawl:
+        return import_optional_dependency("firecrawl").Firecrawl(
+            api_key=self.api_key, api_url=self.api_url, origin="griptape"
+        )
+
+    def search(self, query: str, **kwargs) -> ListArtifact[JsonArtifact]:
+        response = self.client.search(query, limit=self.results_count, **self.params, **kwargs)
+        results = [self._to_dict(result) for result in response.web or []]
+        return ListArtifact([JsonArtifact(result) for result in results])
+
+    def _to_dict(self, result: Any) -> dict:
+        if not isinstance(result, import_optional_dependency("firecrawl.types").Document):
+            return {"title": result.title, "url": result.url, "description": result.description}
+
+        # With scrape_options set, results come back as scraped Documents that keep title and url in their metadata.
+        metadata = result.metadata_typed
+        return {
+            "title": metadata.title,
+            "url": metadata.url or metadata.source_url,
+            "description": metadata.description,
+            **{key: value for key in ("markdown", "summary", "json") if (value := getattr(result, key)) is not None},
+        }

griptape/utils/import_utils.py

@@ -11,6 +11,7 @@
     "pinecone": "pinecone-client",
     "opensearchpy": "opensearch-py",
     "google.genai": "google-genai",
+    "firecrawl": "firecrawl-py",
 }
 
 

pyproject.toml

@@ -81,6 +81,7 @@ drivers-web-scraper-markdownify = [
 drivers-web-search-duckduckgo = ["ddgs>=9.0.0"]
 drivers-web-search-tavily = ["tavily-python>=0.5.0"]
 drivers-web-search-exa = ["exa-py>=1.1.4"]
+drivers-web-search-firecrawl = ["firecrawl-py>=4.46.0"]
 drivers-event-listener-amazon-sqs = ["boto3>=1.34.119"]
 drivers-event-listener-amazon-iot = ["boto3>=1.34.119"]
 drivers-event-listener-pusher = ["pusher>=3.3.2"]
@@ -145,6 +146,7 @@ all = [
   "ddgs>=9.0.0",
   "tavily-python>=0.5.0",
   "exa-py>=1.1.4",
+  "firecrawl-py>=4.46.0",
   "opentelemetry-sdk>=1.25.0",
   "opentelemetry-api>=1.25.0",
   "opentelemetry-instrumentation>=0.49b2",

tests/unit/drivers/web_search/test_firecrawl_web_search_driver.py

@@ -0,0 +1,100 @@
+import pytest
+from firecrawl.types import Document, DocumentMetadata, SearchResultWeb
+
+from griptape.artifacts import ListArtifact
+from griptape.drivers.web_search.firecrawl import FirecrawlWebSearchDriver
+
+
+class TestFirecrawlWebSearchDriver:
+    @pytest.fixture()
+    def mock_firecrawl_client(self, mocker):
+        return mocker.patch("firecrawl.Firecrawl")
+
+    @pytest.fixture()
+    def driver(self, mock_firecrawl_client, mocker):
+        mock_response = mocker.Mock()
+        mock_response.web = [
+            SearchResultWeb(title="foo", url="bar", description="baz"),
+            SearchResultWeb(title="foo", url="bar", description="baz"),
+        ]
+        mock_firecrawl_client.return_value.search.return_value = mock_response
+        return FirecrawlWebSearchDriver(api_key="test")
+
+    def test_client(self, driver, mock_firecrawl_client):
+        assert driver.client == mock_firecrawl_client.return_value
+        mock_firecrawl_client.assert_called_once_with(
+            api_key="test", api_url="https://api.firecrawl.dev", origin="griptape"
+        )
+
+    def test_client_uses_api_url(self, mock_firecrawl_client):
+        driver = FirecrawlWebSearchDriver(api_key="test", api_url="http://localhost:3002")
+        _ = driver.client
+        mock_firecrawl_client.assert_called_once_with(
+            api_key="test", api_url="http://localhost:3002", origin="griptape"
+        )
+
+    def test_search_returns_results(self, driver, mock_firecrawl_client):
+        results = driver.search("test")
+        assert isinstance(results, ListArtifact)
+        output = [result.value for result in results]
+        assert len(output) == 2
+        assert output[0] == {"title": "foo", "url": "bar", "description": "baz"}
+        mock_firecrawl_client.return_value.search.assert_called_once_with("test", limit=5)
+
+    def test_search_with_results_count(self, mock_firecrawl_client):
+        driver = FirecrawlWebSearchDriver(api_key="test", results_count=10)
+        driver.search("test")
+        mock_firecrawl_client.return_value.search.assert_called_once_with("test", limit=10)
+
+    def test_search_returns_markdown_for_scraped_results(self, mock_firecrawl_client, mocker):
+        mock_response = mocker.Mock()
+        mock_response.web = [
+            Document(markdown="# foo", metadata=DocumentMetadata(title="foo", url="bar", description="baz")),
+            Document(markdown="# foo", metadata=DocumentMetadata(title="foo", source_url="bar")),
+        ]
+        mock_firecrawl_client.return_value.search.return_value = mock_response
+        driver = FirecrawlWebSearchDriver(api_key="test", params={"scrape_options": {"formats": ["markdown"]}})
+
+        output = [result.value for result in driver.search("test")]
+
+        assert output[0] == {"title": "foo", "url": "bar", "description": "baz", "markdown": "# foo"}
+        assert output[1]["url"] == "bar"
+        mock_firecrawl_client.return_value.search.assert_called_once_with(
+            "test", limit=5, scrape_options={"formats": ["markdown"]}
+        )
+
+    def test_search_returns_other_scraped_formats(self, mock_firecrawl_client, mocker):
+        mock_response = mocker.Mock()
+        mock_response.web = [Document(summary="qux", metadata=DocumentMetadata(title="foo", url="bar"))]
+        mock_firecrawl_client.return_value.search.return_value = mock_response
+        driver = FirecrawlWebSearchDriver(api_key="test", params={"scrape_options": {"formats": ["summary"]}})
+
+        output = [result.value for result in driver.search("test")]
+
+        assert output[0] == {"title": "foo", "url": "bar", "description": None, "summary": "qux"}
+
+    def test_search_handles_document_without_metadata(self, mock_firecrawl_client, mocker):
+        mock_response = mocker.Mock()
+        mock_response.web = [Document(markdown="# foo")]
+        mock_firecrawl_client.return_value.search.return_value = mock_response
+        driver = FirecrawlWebSearchDriver(api_key="test")
+
+        output = [result.value for result in driver.search("test")]
+
+        assert output[0] == {"title": None, "url": None, "description": None, "markdown": "# foo"}
+
+    def test_search_handles_no_web_results(self, driver, mock_firecrawl_client):
+        mock_firecrawl_client.return_value.search.return_value.web = None
+        assert len(driver.search("test")) == 0
+
+    def test_search_raises_error(self, driver, mock_firecrawl_client):
+        mock_firecrawl_client.return_value.search.side_effect = Exception("test_error")
+        with pytest.raises(Exception, match="test_error"):
+            driver.search("test")
+
+    def test_search_with_params(self, driver, mock_firecrawl_client):
+        driver.params = {"tbs": "qdr:w"}
+        driver.search("test", location="Germany")
+        mock_firecrawl_client.return_value.search.assert_called_once_with(
+            "test", limit=5, tbs="qdr:w", location="Germany"
+        )