From 17bb9efa376abcb8db7077b7ca625a25b055a330 Mon Sep 17 00:00:00 2001 From: Rakshith Ramprakash <33714643+rakshith48@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:11:18 +0530 Subject: [PATCH] Add Firecrawl as a web_search provider Same keyed httpx pattern as Tavily/Brave, hitting Firecrawl's v2 /search endpoint. Key resolves from the SecretStore or FIRECRAWL_API_KEY. Also documents web_search_provider in config.example.toml. --- coworker/config.py | 2 +- coworker/web/__init__.py | 2 ++ coworker/web/providers.py | 36 +++++++++++++++++++++++++++++++++--- docs/config.example.toml | 6 ++++++ pyproject.toml | 2 +- tests/test_web_search.py | 39 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 81 insertions(+), 6 deletions(-) diff --git a/coworker/config.py b/coworker/config.py index b13547b6..79798f8d 100644 --- a/coworker/config.py +++ b/coworker/config.py @@ -49,7 +49,7 @@ class Config: auto_allow: list[str] = field(default_factory=list) host: str = "127.0.0.1" port: int = 8765 - # Web search provider: "duckduckgo" (keyless default) | "tavily" | "brave" (need a key). + # Web search provider: "duckduckgo" (keyless default) | "tavily" | "brave" | "firecrawl" (need a key). web_search_provider: str = "duckduckgo" # OpenWorker Cloud (sign-in + managed connectors). Config, never constants: # dev/staging/BYO-VPC deployments point these at their own instances. diff --git a/coworker/web/__init__.py b/coworker/web/__init__.py index e397eab8..232ef199 100644 --- a/coworker/web/__init__.py +++ b/coworker/web/__init__.py @@ -5,6 +5,7 @@ from __future__ import annotations from .providers import ( BraveProvider, DuckDuckGoProvider, + FirecrawlProvider, SearchResult, TavilyProvider, WebSearchProvider, @@ -20,6 +21,7 @@ __all__ = [ "DuckDuckGoProvider", "TavilyProvider", "BraveProvider", + "FirecrawlProvider", "build_provider", "provider_names", "make_web_search_tool", diff --git a/coworker/web/providers.py b/coworker/web/providers.py index 23cad857..289d4622 100644 --- a/coworker/web/providers.py +++ b/coworker/web/providers.py @@ -1,8 +1,8 @@ """Web search providers — a keyless default + pluggable third-party services. -`duckduckgo` works with no API key (our "starting version of our own"). `tavily` and `brave` -give better results but need a key (configured via the SecretStore / env). All providers -return a uniform `list[SearchResult]`; the heavy client libs are lazy-imported. +`duckduckgo` works with no API key (our "starting version of our own"). `tavily`, `brave`, +and `firecrawl` give better results but need a key (configured via the SecretStore / env). +All providers return a uniform `list[SearchResult]`; the heavy client libs are lazy-imported. """ from __future__ import annotations @@ -108,10 +108,40 @@ class BraveProvider(WebSearchProvider): ] +class FirecrawlProvider(WebSearchProvider): + """Firecrawl web search (https://docs.firecrawl.dev) via the v2 REST API.""" + + name = "firecrawl" + requires_key = True + + def __init__(self, api_key: str) -> None: + self.api_key = api_key + + def search(self, query: str, max_results: int = 5) -> list[SearchResult]: + import httpx + + resp = httpx.post( + "https://api.firecrawl.dev/v2/search", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={"query": query, "limit": max_results}, + timeout=_TIMEOUT, + ) + data = resp.json() + return [ + SearchResult( + title=r.get("title", ""), + url=r.get("url", ""), + snippet=r.get("description", ""), + ) + for r in (data.get("data", {}) or {}).get("web", []) + ] + + _PROVIDERS = { "duckduckgo": DuckDuckGoProvider, "tavily": TavilyProvider, "brave": BraveProvider, + "firecrawl": FirecrawlProvider, } diff --git a/docs/config.example.toml b/docs/config.example.toml index 9638ad23..c8e11c44 100644 --- a/docs/config.example.toml +++ b/docs/config.example.toml @@ -8,6 +8,12 @@ model = "gpt-5.5" # default model id (override per session in the UI/ mode = "interactive" # plan | interactive | auto | custom max_iterations = 12 # max model<->tool iterations per turn before stopping +# Web search provider for the `web_search` tool. +# duckduckgo (keyless default) | tavily | brave | firecrawl +# The keyed providers read their key from the SecretStore (set in the UI) or a +# _API_KEY env var, e.g. FIRECRAWL_API_KEY. See https://docs.firecrawl.dev +web_search_provider = "duckduckgo" + # Commands auto-allowed without an approval prompt (prefix match). allowed_commands = [ "ls", "cat", "pwd", "grep", "find", diff --git a/pyproject.toml b/pyproject.toml index aa04eef4..8e0cda74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "mcp>=1.1", # MCP client (stdio + streamable-http); we use our own async layer on it "httpx>=0.27", # sync outbound senders for messaging connectors (send_message tool) "websockets>=13", # managed Slack relay client transport (relay_client.py) - "ddgs>=9", # keyless default web-search provider (DuckDuckGo); Tavily/Brave use httpx + "ddgs>=9", # keyless default web-search provider (DuckDuckGo); Tavily/Brave/Firecrawl use httpx "croniter>=2", # cron next-fire math for the automation scheduler # PDF attachments for models without native PDF support (pdf_support.py): # pypdf = pure-python text extraction; pypdfium2 = page rasterization (BSD pdfium, diff --git a/tests/test_web_search.py b/tests/test_web_search.py index d1c2d0fd..4cb52b6f 100644 --- a/tests/test_web_search.py +++ b/tests/test_web_search.py @@ -1,7 +1,7 @@ """Tests for web search — provider abstraction, the tool, and config resolution. No network: a FakeProvider is injected; third-party key handling and the REST config path -are exercised without hitting DuckDuckGo/Tavily/Brave. +are exercised without hitting DuckDuckGo/Tavily/Brave/Firecrawl. """ from __future__ import annotations @@ -18,6 +18,7 @@ from coworker.web import ( from coworker.web.providers import ( BraveProvider, DuckDuckGoProvider, + FirecrawlProvider, TavilyProvider, WebSearchProvider, ) @@ -79,6 +80,42 @@ def test_build_provider_third_party_requires_key(): build_provider("tavily") # no key assert isinstance(build_provider("tavily", "tvly-x"), TavilyProvider) assert isinstance(build_provider("brave", "brv-x"), BraveProvider) + with pytest.raises(ValueError): + build_provider("firecrawl") # no key + assert isinstance(build_provider("firecrawl", "fc-x"), FirecrawlProvider) + + +def test_firecrawl_maps_v2_search_response(monkeypatch): + import httpx + + captured = {} + + class _Resp: + def json(self): + return { + "success": True, + "data": { + "web": [ + {"title": "t0", "url": "https://x/0", "description": "d0"}, + {"title": "t1", "url": "https://x/1", "description": "d1"}, + ] + }, + } + + def fake_post(url, headers=None, json=None, timeout=None): + captured.update(url=url, headers=headers, json=json) + return _Resp() + + monkeypatch.setattr(httpx, "post", fake_post) + results = FirecrawlProvider("fc-key").search("claude api", max_results=2) + + assert captured["url"] == "https://api.firecrawl.dev/v2/search" + assert captured["headers"]["Authorization"] == "Bearer fc-key" + assert captured["json"] == {"query": "claude api", "limit": 2} + assert [(r.title, r.url, r.snippet) for r in results] == [ + ("t0", "https://x/0", "d0"), + ("t1", "https://x/1", "d1"), + ] def test_tool_surfaces_missing_key_error(tmp_path):