diff --git a/.github/workflows/docs-integration-tests.yml b/.github/workflows/docs-integration-tests.yml index aaeaf9413..ef7d3f7a9 100644 --- a/.github/workflows/docs-integration-tests.yml +++ b/.github/workflows/docs-integration-tests.yml @@ -57,6 +57,7 @@ jobs: DYNAMODB_TABLE_NAME: ${{ secrets.INTEG_DYNAMODB_TABLE_NAME }} ELEVEN_LABS_API_KEY: ${{ secrets.INTEG_ELEVEN_LABS_API_KEY }} EXA_API_KEY: ${{ secrets.INTEG_EXA_API_KEY }} + FIRECRAWL_API_KEY: ${{ secrets.INTEG_FIRECRAWL_API_KEY }} GOOGLE_API_KEY: ${{ secrets.INTEG_GOOGLE_API_KEY }} GOOGLE_API_SEARCH_ID: ${{ secrets.INTEG_GOOGLE_API_SEARCH_ID }} GOOGLE_OWNER_EMAIL: ${{ secrets.INTEG_GOOGLE_OWNER_EMAIL }} diff --git a/docs/griptape-framework/drivers/src/web_search_drivers_firecrawl.py b/docs/griptape-framework/drivers/src/web_search_drivers_firecrawl.py new file mode 100644 index 000000000..688e5e656 --- /dev/null +++ b/docs/griptape-framework/drivers/src/web_search_drivers_firecrawl.py @@ -0,0 +1,7 @@ +import os + +from griptape.drivers.web_search.firecrawl import FirecrawlWebSearchDriver + +driver = FirecrawlWebSearchDriver(api_key=os.environ["FIRECRAWL_API_KEY"]) + +driver.search("griptape ai") diff --git a/docs/griptape-framework/drivers/web-search-drivers.md b/docs/griptape-framework/drivers/web-search-drivers.md index 8b5703cfb..a35703ee5 100644 --- a/docs/griptape-framework/drivers/web-search-drivers.md +++ b/docs/griptape-framework/drivers/web-search-drivers.md @@ -102,6 +102,19 @@ The [DuckDuckGoWebSearchDriver](../../reference/griptape/drivers/web_search/duck --8<-- "docs/griptape-framework/drivers/src/web_search_drivers_6.py" ``` +### Firecrawl + +The [FirecrawlWebSearchDriver](../../reference/griptape/drivers/web_search/firecrawl_web_search_driver.md) uses the Firecrawl Search API for web searching. + +!!! info + + This driver requires the `drivers-web-search-firecrawl` [extra](../index.md#extras), + and a Firecrawl [api key](https://www.firecrawl.dev/search?utm_source=griptape&utm_medium=integration). + +```python +--8<-- "docs/griptape-framework/drivers/src/web_search_drivers_firecrawl.py" +``` + ### Perplexity The [PerplexityWebSearchDriver](../../reference/griptape/drivers/web_search/perplexity_web_search_driver.md) uses the [Perplexity Prompt Driver](../drivers/prompt-drivers.md#perplexity) internally. diff --git a/griptape/drivers/__init__.py b/griptape/drivers/__init__.py index 24ffefe04..4662c1a11 100644 --- a/griptape/drivers/__init__.py +++ b/griptape/drivers/__init__.py @@ -85,6 +85,7 @@ from .web_search.google import GoogleWebSearchDriver from .web_search.duck_duck_go import DuckDuckGoWebSearchDriver from .web_search.exa import ExaWebSearchDriver +from .web_search.firecrawl import FirecrawlWebSearchDriver from .web_search.tavily import TavilyWebSearchDriver from .web_search.perplexity import PerplexityWebSearchDriver @@ -190,6 +191,7 @@ "DummyVectorStoreDriver", "ElevenLabsTextToSpeechDriver", "ExaWebSearchDriver", + "FirecrawlWebSearchDriver", "GoogleEmbeddingDriver", "GooglePromptDriver", "GoogleWebSearchDriver", @@ -261,6 +263,7 @@ "GoogleWebSearchDriver", "DuckDuckGoWebSearchDriver", "ExaWebSearchDriver", + "FirecrawlWebSearchDriver", "TavilyWebSearchDriver", "TrafilaturaWebScraperDriver", "VoyageAiEmbeddingDriver", diff --git a/griptape/drivers/web_search/firecrawl/__init__.py b/griptape/drivers/web_search/firecrawl/__init__.py new file mode 100644 index 000000000..cc74279fb --- /dev/null +++ b/griptape/drivers/web_search/firecrawl/__init__.py @@ -0,0 +1,3 @@ +from griptape.drivers.web_search.firecrawl_web_search_driver import FirecrawlWebSearchDriver + +__all__ = ["FirecrawlWebSearchDriver"] diff --git a/griptape/drivers/web_search/firecrawl_web_search_driver.py b/griptape/drivers/web_search/firecrawl_web_search_driver.py new file mode 100644 index 000000000..04443d358 --- /dev/null +++ b/griptape/drivers/web_search/firecrawl_web_search_driver.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from attrs import define, field + +from griptape.artifacts import JsonArtifact, ListArtifact +from griptape.drivers.web_search import BaseWebSearchDriver +from griptape.utils import import_optional_dependency +from griptape.utils.decorators import lazy_property + +if TYPE_CHECKING: + from firecrawl import Firecrawl + + +@define +class FirecrawlWebSearchDriver(BaseWebSearchDriver): + """Web Search Driver for the Firecrawl Search API. + + Returns web results with their title, url and description. When `scrape_options` is set in `params`, + each result also carries the requested page content (markdown, summary or json). + + Attributes: + api_key: Firecrawl API key. + api_url: Firecrawl API base URL. Override it to use a self-hosted Firecrawl instance. + params: Extra keyword arguments passed to `Firecrawl.search` on every call, + for example `{"tbs": "qdr:w"}` or `{"scrape_options": {"formats": ["markdown"]}}`. + """ + + api_key: str = field(kw_only=True) + api_url: str = field(default="https://api.firecrawl.dev", kw_only=True, metadata={"serializable": True}) + params: dict[str, Any] = field(factory=dict, kw_only=True, metadata={"serializable": True}) + _client: Firecrawl | None = field(default=None, kw_only=True, alias="client") + + @lazy_property() + def client(self) -> Firecrawl: + return import_optional_dependency("firecrawl").Firecrawl( + api_key=self.api_key, api_url=self.api_url, origin="griptape" + ) + + def search(self, query: str, **kwargs) -> ListArtifact[JsonArtifact]: + response = self.client.search(query, limit=self.results_count, **self.params, **kwargs) + results = [self._to_dict(result) for result in response.web or []] + return ListArtifact([JsonArtifact(result) for result in results]) + + def _to_dict(self, result: Any) -> dict: + if not isinstance(result, import_optional_dependency("firecrawl.types").Document): + return {"title": result.title, "url": result.url, "description": result.description} + + # With scrape_options set, results come back as scraped Documents that keep title and url in their metadata. + metadata = result.metadata_typed + return { + "title": metadata.title, + "url": metadata.url or metadata.source_url, + "description": metadata.description, + **{key: value for key in ("markdown", "summary", "json") if (value := getattr(result, key)) is not None}, + } diff --git a/griptape/utils/import_utils.py b/griptape/utils/import_utils.py index e2206e9bc..8eefa9473 100644 --- a/griptape/utils/import_utils.py +++ b/griptape/utils/import_utils.py @@ -11,6 +11,7 @@ "pinecone": "pinecone-client", "opensearchpy": "opensearch-py", "google.genai": "google-genai", + "firecrawl": "firecrawl-py", } diff --git a/pyproject.toml b/pyproject.toml index 4593e6c27..66fa302c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,7 @@ drivers-web-scraper-markdownify = [ drivers-web-search-duckduckgo = ["ddgs>=9.0.0"] drivers-web-search-tavily = ["tavily-python>=0.5.0"] drivers-web-search-exa = ["exa-py>=1.1.4"] +drivers-web-search-firecrawl = ["firecrawl-py>=4.46.0"] drivers-event-listener-amazon-sqs = ["boto3>=1.34.119"] drivers-event-listener-amazon-iot = ["boto3>=1.34.119"] drivers-event-listener-pusher = ["pusher>=3.3.2"] @@ -145,6 +146,7 @@ all = [ "ddgs>=9.0.0", "tavily-python>=0.5.0", "exa-py>=1.1.4", + "firecrawl-py>=4.46.0", "opentelemetry-sdk>=1.25.0", "opentelemetry-api>=1.25.0", "opentelemetry-instrumentation>=0.49b2", diff --git a/tests/unit/drivers/web_search/test_firecrawl_web_search_driver.py b/tests/unit/drivers/web_search/test_firecrawl_web_search_driver.py new file mode 100644 index 000000000..2d06c8f06 --- /dev/null +++ b/tests/unit/drivers/web_search/test_firecrawl_web_search_driver.py @@ -0,0 +1,100 @@ +import pytest +from firecrawl.types import Document, DocumentMetadata, SearchResultWeb + +from griptape.artifacts import ListArtifact +from griptape.drivers.web_search.firecrawl import FirecrawlWebSearchDriver + + +class TestFirecrawlWebSearchDriver: + @pytest.fixture() + def mock_firecrawl_client(self, mocker): + return mocker.patch("firecrawl.Firecrawl") + + @pytest.fixture() + def driver(self, mock_firecrawl_client, mocker): + mock_response = mocker.Mock() + mock_response.web = [ + SearchResultWeb(title="foo", url="bar", description="baz"), + SearchResultWeb(title="foo", url="bar", description="baz"), + ] + mock_firecrawl_client.return_value.search.return_value = mock_response + return FirecrawlWebSearchDriver(api_key="test") + + def test_client(self, driver, mock_firecrawl_client): + assert driver.client == mock_firecrawl_client.return_value + mock_firecrawl_client.assert_called_once_with( + api_key="test", api_url="https://api.firecrawl.dev", origin="griptape" + ) + + def test_client_uses_api_url(self, mock_firecrawl_client): + driver = FirecrawlWebSearchDriver(api_key="test", api_url="http://localhost:3002") + _ = driver.client + mock_firecrawl_client.assert_called_once_with( + api_key="test", api_url="http://localhost:3002", origin="griptape" + ) + + def test_search_returns_results(self, driver, mock_firecrawl_client): + results = driver.search("test") + assert isinstance(results, ListArtifact) + output = [result.value for result in results] + assert len(output) == 2 + assert output[0] == {"title": "foo", "url": "bar", "description": "baz"} + mock_firecrawl_client.return_value.search.assert_called_once_with("test", limit=5) + + def test_search_with_results_count(self, mock_firecrawl_client): + driver = FirecrawlWebSearchDriver(api_key="test", results_count=10) + driver.search("test") + mock_firecrawl_client.return_value.search.assert_called_once_with("test", limit=10) + + def test_search_returns_markdown_for_scraped_results(self, mock_firecrawl_client, mocker): + mock_response = mocker.Mock() + mock_response.web = [ + Document(markdown="# foo", metadata=DocumentMetadata(title="foo", url="bar", description="baz")), + Document(markdown="# foo", metadata=DocumentMetadata(title="foo", source_url="bar")), + ] + mock_firecrawl_client.return_value.search.return_value = mock_response + driver = FirecrawlWebSearchDriver(api_key="test", params={"scrape_options": {"formats": ["markdown"]}}) + + output = [result.value for result in driver.search("test")] + + assert output[0] == {"title": "foo", "url": "bar", "description": "baz", "markdown": "# foo"} + assert output[1]["url"] == "bar" + mock_firecrawl_client.return_value.search.assert_called_once_with( + "test", limit=5, scrape_options={"formats": ["markdown"]} + ) + + def test_search_returns_other_scraped_formats(self, mock_firecrawl_client, mocker): + mock_response = mocker.Mock() + mock_response.web = [Document(summary="qux", metadata=DocumentMetadata(title="foo", url="bar"))] + mock_firecrawl_client.return_value.search.return_value = mock_response + driver = FirecrawlWebSearchDriver(api_key="test", params={"scrape_options": {"formats": ["summary"]}}) + + output = [result.value for result in driver.search("test")] + + assert output[0] == {"title": "foo", "url": "bar", "description": None, "summary": "qux"} + + def test_search_handles_document_without_metadata(self, mock_firecrawl_client, mocker): + mock_response = mocker.Mock() + mock_response.web = [Document(markdown="# foo")] + mock_firecrawl_client.return_value.search.return_value = mock_response + driver = FirecrawlWebSearchDriver(api_key="test") + + output = [result.value for result in driver.search("test")] + + assert output[0] == {"title": None, "url": None, "description": None, "markdown": "# foo"} + + def test_search_handles_no_web_results(self, driver, mock_firecrawl_client): + mock_firecrawl_client.return_value.search.return_value.web = None + assert len(driver.search("test")) == 0 + + def test_search_raises_error(self, driver, mock_firecrawl_client): + mock_firecrawl_client.return_value.search.side_effect = Exception("test_error") + with pytest.raises(Exception, match="test_error"): + driver.search("test") + + def test_search_with_params(self, driver, mock_firecrawl_client): + driver.params = {"tbs": "qdr:w"} + driver.search("test", location="Germany") + mock_firecrawl_client.return_value.search.assert_called_once_with( + "test", limit=5, tbs="qdr:w", location="Germany" + ) diff --git a/uv.lock b/uv.lock index a16519c9e..900a6c3c0 100644 --- a/uv.lock +++ b/uv.lock @@ -1189,6 +1189,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970, upload-time = "2022-11-02T17:34:01.425Z" }, ] +[[package]] +name = "firecrawl-py" +version = "4.46.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "httpx" }, + { name = "nest-asyncio" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/bb/0776d4431c3b8f206321b1e514417a6438513a0ac6552634d160ce4e301e/firecrawl_py-4.46.0.tar.gz", hash = "sha256:e0c84ec5d3470269108923d4edf3d363268b1e17e6876392ee0d9ac6e5ba7ea2", size = 256174, upload-time = "2026-10-01T17:46:23.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/d3/02c5fc9a4c2ec8a09e6ca4c87d9066242fb79dd8ddffc7155d1286eb1919/firecrawl_py-4.46.0-py3-none-any.whl", hash = "sha256:51d788e3d68731e21ca669ed2f3bbd2826221559bbef7639a5029ff3dbfe7c70", size = 313540, upload-time = "2026-10-01T17:46:22.561Z" }, +] + [[package]] name = "frozenlist" version = "1.5.0" @@ -1438,6 +1456,7 @@ all = [ { name = "diffusers" }, { name = "elevenlabs" }, { name = "exa-py" }, + { name = "firecrawl-py" }, { name = "google-genai" }, { name = "huggingface-hub" }, { name = "mail-parser" }, @@ -1631,6 +1650,9 @@ drivers-web-search-duckduckgo = [ drivers-web-search-exa = [ { name = "exa-py" }, ] +drivers-web-search-firecrawl = [ + { name = "firecrawl-py" }, +] drivers-web-search-tavily = [ { name = "tavily-python" }, ] @@ -1723,6 +1745,8 @@ requires-dist = [ { name = "exa-py", marker = "extra == 'all'", specifier = ">=1.1.4" }, { name = "exa-py", marker = "extra == 'drivers-web-search-exa'", specifier = ">=1.1.4" }, { name = "filetype", specifier = ">=1.2" }, + { name = "firecrawl-py", marker = "extra == 'all'", specifier = ">=4.46.0" }, + { name = "firecrawl-py", marker = "extra == 'drivers-web-search-firecrawl'", specifier = ">=4.46.0" }, { name = "google-genai", marker = "extra == 'all'", specifier = ">=1.73.1" }, { name = "google-genai", marker = "extra == 'drivers-embedding-google'", specifier = ">=1.73.1" }, { name = "google-genai", marker = "extra == 'drivers-prompt-google'", specifier = ">=1.73.1" }, @@ -1823,7 +1847,7 @@ requires-dist = [ { name = "voyageai", marker = "extra == 'drivers-rerank-voyageai'", specifier = ">=0.2.1" }, { name = "wrapt", specifier = ">=1.16.0" }, ] -provides-extras = ["drivers-prompt-cohere", "drivers-prompt-anthropic", "drivers-prompt-huggingface-hub", "drivers-prompt-huggingface-pipeline", "drivers-prompt-amazon-bedrock", "drivers-prompt-amazon-sagemaker", "drivers-prompt-google", "drivers-prompt-ollama", "drivers-sql", "drivers-sql-amazon-redshift", "drivers-sql-snowflake", "drivers-memory-conversation-amazon-dynamodb", "drivers-memory-conversation-redis", "drivers-vector-marqo", "drivers-vector-pinecone", "drivers-vector-mongodb", "drivers-vector-redis", "drivers-vector-opensearch", "drivers-vector-amazon-opensearch", "drivers-vector-pgvector", "drivers-vector-qdrant", "drivers-vector-astra-db", "drivers-vector-pgai", "drivers-embedding-amazon-bedrock", "drivers-embedding-amazon-sagemaker", "drivers-embedding-huggingface", "drivers-embedding-voyageai", "drivers-embedding-google", "drivers-embedding-cohere", "drivers-embedding-ollama", "drivers-web-scraper-trafilatura", "drivers-web-scraper-markdownify", "drivers-web-search-duckduckgo", "drivers-web-search-tavily", "drivers-web-search-exa", "drivers-event-listener-amazon-sqs", "drivers-event-listener-amazon-iot", "drivers-event-listener-pusher", "drivers-text-to-speech-elevenlabs", "drivers-rerank-cohere", "drivers-rerank-amazon-bedrock", "drivers-rerank-voyageai", "drivers-observability-opentelemetry", "drivers-observability-griptape-cloud", "drivers-observability-datadog", "drivers-image-generation-huggingface", "drivers-file-manager-amazon-s3", "loaders-pdf", "loaders-image", "loaders-email", "loaders-sql", "all"] +provides-extras = ["drivers-prompt-cohere", "drivers-prompt-anthropic", "drivers-prompt-huggingface-hub", "drivers-prompt-huggingface-pipeline", "drivers-prompt-amazon-bedrock", "drivers-prompt-amazon-sagemaker", "drivers-prompt-google", "drivers-prompt-ollama", "drivers-sql", "drivers-sql-amazon-redshift", "drivers-sql-snowflake", "drivers-memory-conversation-amazon-dynamodb", "drivers-memory-conversation-redis", "drivers-vector-marqo", "drivers-vector-pinecone", "drivers-vector-mongodb", "drivers-vector-redis", "drivers-vector-opensearch", "drivers-vector-amazon-opensearch", "drivers-vector-pgvector", "drivers-vector-qdrant", "drivers-vector-astra-db", "drivers-vector-pgai", "drivers-embedding-amazon-bedrock", "drivers-embedding-amazon-sagemaker", "drivers-embedding-huggingface", "drivers-embedding-voyageai", "drivers-embedding-google", "drivers-embedding-cohere", "drivers-embedding-ollama", "drivers-web-scraper-trafilatura", "drivers-web-scraper-markdownify", "drivers-web-search-duckduckgo", "drivers-web-search-tavily", "drivers-web-search-exa", "drivers-web-search-firecrawl", "drivers-event-listener-amazon-sqs", "drivers-event-listener-amazon-iot", "drivers-event-listener-pusher", "drivers-text-to-speech-elevenlabs", "drivers-rerank-cohere", "drivers-rerank-amazon-bedrock", "drivers-rerank-voyageai", "drivers-observability-opentelemetry", "drivers-observability-griptape-cloud", "drivers-observability-datadog", "drivers-image-generation-huggingface", "drivers-file-manager-amazon-s3", "loaders-pdf", "loaders-image", "loaders-email", "loaders-sql", "all"] [package.metadata.requires-dev] dev = [ @@ -3182,6 +3206,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/67/c2f508c00ed2a6911541494504b7cac16fe0b0473912568df65fd1801132/ndg_httpsclient-0.5.1-py3-none-any.whl", hash = "sha256:dd174c11d971b6244a891f7be2b32ca9853d3797a72edb34fa5d7b07d8fff7d4", size = 34042, upload-time = "2018-07-23T16:04:46.553Z" }, ] +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + [[package]] name = "networkx" version = "3.4.2"