From 82a816a5b53c6f3fb3381efe028d77d569395202 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Sun, 11 Jan 2026 14:09:21 +0100 Subject: [PATCH] feat: add web search tool using SearXNG Add WebSearchTool that queries the self-hosted SearXNG metasearch engine for current information, documentation, and facts beyond training data. - Add SEARXNG_URL and SEARXNG_TIMEOUT config settings - Create WebSearchTool with query, num_results, categories params - Register web_search tool with explore agent - Add 10 tests for search functionality Usage: Agents can now use web_search(query="...") to find current info. Co-Authored-By: Claude Opus 4.5 --- .../src/domains/agents/explore/tools.py | 40 ++++ webber-api/src/domains/tools/__init__.py | 3 +- .../src/domains/tools/search/__init__.py | 3 +- webber-api/src/domains/tools/search/web.py | 169 ++++++++++++++ webber-api/src/shared/config.py | 4 + webber-api/tests/test_web_search.py | 217 ++++++++++++++++++ 6 files changed, 434 insertions(+), 2 deletions(-) create mode 100644 webber-api/src/domains/tools/search/web.py create mode 100644 webber-api/tests/test_web_search.py diff --git a/webber-api/src/domains/agents/explore/tools.py b/webber-api/src/domains/agents/explore/tools.py index d236b32..a5c840b 100644 --- a/webber-api/src/domains/agents/explore/tools.py +++ b/webber-api/src/domains/agents/explore/tools.py @@ -11,6 +11,7 @@ from src.domains.tools.file.glob import GlobFilesTool from src.domains.tools.file.edit import EditFileTool from src.domains.tools.file.write import WriteFileTool from src.domains.tools.search.grep import GrepContentTool +from src.domains.tools.search.web import WebSearchTool from src.domains.tools.shell.bash import BashReadOnlyTool from src.domains.tools.shell.bash_full import BashTool @@ -271,3 +272,42 @@ def register_explore_tools(agent: Agent[AgentContext, str]) -> None: timeout=min(timeout, ctx.deps.timeout_seconds) ) return result.to_string() + + # === Web search === + + @agent.tool + async def web_search( + ctx: RunContext[AgentContext], + query: str, + num_results: int = 5, + categories: str | None = None + ) -> str: + """Search the web for current information. + + Args: + query: Search query (e.g., "Python 3.12 new features") + num_results: Number of results to return (1-10, default: 5) + categories: Optional category filter ("general", "it", "news", "science") + + Returns: + Search results with titles, URLs, and snippets. + + Use this for: + - Current events or recent information + - Documentation updates since your training + - Facts you're uncertain about + - Technical references with URLs + + IMPORTANT: Always include a "Sources:" section with URLs in your response. + + Examples: + - query="FastAPI best practices 2024" + - query="CVE-2024" categories="it" + """ + tool = WebSearchTool() + result = await tool.execute( + query=query, + num_results=num_results, + categories=categories + ) + return result.to_string() diff --git a/webber-api/src/domains/tools/__init__.py b/webber-api/src/domains/tools/__init__.py index 9b9d847..e1bf1bb 100644 --- a/webber-api/src/domains/tools/__init__.py +++ b/webber-api/src/domains/tools/__init__.py @@ -5,7 +5,7 @@ All tools inherit from BaseTool and return ToolResult. """ from src.domains.tools.base import BaseTool, ToolResult from src.domains.tools.file import ReadFileTool, GlobFilesTool, EditFileTool, WriteFileTool -from src.domains.tools.search import GrepContentTool +from src.domains.tools.search import GrepContentTool, WebSearchTool from src.domains.tools.shell import BashReadOnlyTool, BashTool __all__ = [ @@ -16,6 +16,7 @@ __all__ = [ "EditFileTool", "WriteFileTool", "GrepContentTool", + "WebSearchTool", "BashReadOnlyTool", "BashTool", ] diff --git a/webber-api/src/domains/tools/search/__init__.py b/webber-api/src/domains/tools/search/__init__.py index 1050da9..1b6aa99 100644 --- a/webber-api/src/domains/tools/search/__init__.py +++ b/webber-api/src/domains/tools/search/__init__.py @@ -2,5 +2,6 @@ Search tools. """ from src.domains.tools.search.grep import GrepContentTool +from src.domains.tools.search.web import WebSearchTool -__all__ = ["GrepContentTool"] +__all__ = ["GrepContentTool", "WebSearchTool"] diff --git a/webber-api/src/domains/tools/search/web.py b/webber-api/src/domains/tools/search/web.py new file mode 100644 index 0000000..85b5c4b --- /dev/null +++ b/webber-api/src/domains/tools/search/web.py @@ -0,0 +1,169 @@ +""" +Web search tool using SearXNG. +""" +from dataclasses import dataclass +from datetime import datetime + +import httpx + +from src.domains.tools.base import BaseTool, ToolResult +from src.shared.config import get_settings +from src.shared.logging import logged, get_logger + +logger = get_logger(__name__) + + +@dataclass +class SearchResult: + """A single search result.""" + title: str + url: str + content: str + engine: str + published_date: str | None = None + + +class WebSearchTool(BaseTool): + """ + Search the web using SearXNG metasearch engine. + + Returns relevant web results for queries about current events, + documentation, or anything beyond the LLM's knowledge cutoff. + """ + + name = "web_search" + description = """Search the web for current information. + +Args: + query: Search query string + num_results: Maximum results to return (default: 5, max: 10) + engines: Comma-separated engine list (optional, e.g., "google,brave,duckduckgo") + categories: Search category (optional: "general", "images", "news", "science", "it") + +Returns: + List of search results with title, URL, and snippet. + Include a "Sources:" section with URLs in your response. + +Examples: + - query="Python 3.12 new features" - Find latest Python docs + - query="FastAPI best practices 2024" - Find recent tutorials + - query="CVE-2024" categories="it" - Search IT/security news + +IMPORTANT: +- Use this for current events, recent documentation, or facts you're unsure about +- Always cite sources with URLs in your response +- Today's date is {date} - use current year in queries for recent info +""".format(date=datetime.now().strftime("%Y-%m-%d")) + + def __init__( + self, + searxng_url: str | None = None, + timeout: int | None = None, + max_results: int = 10, + ): + """ + Initialize WebSearchTool. + + Args: + searxng_url: SearXNG instance URL (default: from config) + timeout: Request timeout in seconds (default: from config) + max_results: Maximum results to return + """ + settings = get_settings() + self.searxng_url = (searxng_url or settings.searxng_url).rstrip("/") + self.timeout = timeout or settings.searxng_timeout + self.max_results = max_results + + @logged() + async def execute( + self, + query: str, + num_results: int = 5, + engines: str | None = None, + categories: str | None = None, + ) -> ToolResult: + """ + Execute web search. + + Args: + query: Search query + num_results: Number of results (1-10) + engines: Specific engines to use + categories: Search category + + Returns: + ToolResult with search results + """ + if not query or not query.strip(): + return self._error("Query cannot be empty") + + num_results = min(max(1, num_results), self.max_results) + + # Build SearXNG API request + params = { + "q": query.strip(), + "format": "json", + } + + if engines: + params["engines"] = engines + if categories: + params["categories"] = categories + + try: + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.get( + f"{self.searxng_url}/search", + params=params, + ) + response.raise_for_status() + data = response.json() + + except httpx.TimeoutException: + return self._error(f"Search timed out after {self.timeout}s") + except httpx.HTTPStatusError as e: + return self._error(f"Search failed: HTTP {e.response.status_code}") + except httpx.RequestError as e: + return self._error(f"Search request failed: {e}") + except Exception as e: + logger.exception(f"Unexpected search error: {e}") + return self._error(f"Search error: {e}") + + # Parse results + raw_results = data.get("results", [])[:num_results] + + if not raw_results: + return self._success( + f"No results found for: {query}", + result_count=0, + query=query, + ) + + # Format results for LLM consumption + results = [] + for r in raw_results: + result = SearchResult( + title=r.get("title", "Untitled"), + url=r.get("url", ""), + content=r.get("content", "No description"), + engine=r.get("engine", "unknown"), + published_date=r.get("publishedDate"), + ) + results.append(result) + + # Format as readable text + output_lines = [f"Search results for: {query}", ""] + for i, r in enumerate(results, 1): + output_lines.append(f"{i}. **{r.title}**") + output_lines.append(f" URL: {r.url}") + output_lines.append(f" {r.content}") + if r.published_date: + output_lines.append(f" Published: {r.published_date}") + output_lines.append("") + + return self._success( + "\n".join(output_lines), + result_count=len(results), + query=query, + engines_used=list({r.engine for r in results}), + ) diff --git a/webber-api/src/shared/config.py b/webber-api/src/shared/config.py index b8ad55e..1cf9d9e 100644 --- a/webber-api/src/shared/config.py +++ b/webber-api/src/shared/config.py @@ -71,6 +71,10 @@ class Settings(BaseSettings): tatlock_api_url: str | None = "http://192.168.86.149:8000" internal_api_key: str | None = None + # Web search - SearXNG (use SEARXNG_URL env var to override) + searxng_url: str = "http://192.168.86.149:8087" + searxng_timeout: int = 10 + # Tool execution tool_timeout_seconds: int = 120 sandbox_enabled: bool = True diff --git a/webber-api/tests/test_web_search.py b/webber-api/tests/test_web_search.py new file mode 100644 index 0000000..a52c392 --- /dev/null +++ b/webber-api/tests/test_web_search.py @@ -0,0 +1,217 @@ +""" +Tests for WebSearchTool. +""" +import pytest +from unittest.mock import AsyncMock, patch, MagicMock + +from src.domains.tools.search.web import WebSearchTool + + +class TestWebSearchTool: + """Tests for WebSearchTool.""" + + @pytest.fixture + def tool(self): + return WebSearchTool(searxng_url="http://localhost:8087", timeout=5) + + @pytest.fixture + def mock_search_response(self): + """Sample SearXNG response.""" + return { + "query": "test query", + "number_of_results": 3, + "results": [ + { + "title": "First Result", + "url": "https://example.com/1", + "content": "This is the first result content.", + "engine": "google", + "publishedDate": "2024-01-15", + }, + { + "title": "Second Result", + "url": "https://example.com/2", + "content": "This is the second result content.", + "engine": "brave", + "publishedDate": None, + }, + { + "title": "Third Result", + "url": "https://example.com/3", + "content": "This is the third result content.", + "engine": "duckduckgo", + "publishedDate": "2024-01-10", + }, + ], + } + + @pytest.mark.anyio + async def test_search_success(self, tool, mock_search_response): + """Test successful search.""" + with patch("httpx.AsyncClient") as mock_client: + mock_response = MagicMock() + mock_response.json.return_value = mock_search_response + mock_response.raise_for_status = MagicMock() + + mock_instance = AsyncMock() + mock_instance.get.return_value = mock_response + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await tool.execute(query="test query", num_results=3) + + assert result.success + assert "First Result" in result.data + assert "https://example.com/1" in result.data + assert result.metadata["result_count"] == 3 + + @pytest.mark.anyio + async def test_search_empty_query(self, tool): + """Test empty query is rejected.""" + result = await tool.execute(query="") + assert not result.success + assert "empty" in result.error.lower() + + @pytest.mark.anyio + async def test_search_whitespace_query(self, tool): + """Test whitespace-only query is rejected.""" + result = await tool.execute(query=" ") + assert not result.success + assert "empty" in result.error.lower() + + @pytest.mark.anyio + async def test_search_no_results(self, tool): + """Test when search returns no results.""" + with patch("httpx.AsyncClient") as mock_client: + mock_response = MagicMock() + mock_response.json.return_value = {"query": "obscure", "results": []} + mock_response.raise_for_status = MagicMock() + + mock_instance = AsyncMock() + mock_instance.get.return_value = mock_response + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await tool.execute(query="obscure nonexistent thing") + + assert result.success + assert "No results" in result.data + assert result.metadata["result_count"] == 0 + + @pytest.mark.anyio + async def test_search_num_results_limit(self, tool, mock_search_response): + """Test num_results limits output.""" + with patch("httpx.AsyncClient") as mock_client: + mock_response = MagicMock() + mock_response.json.return_value = mock_search_response + mock_response.raise_for_status = MagicMock() + + mock_instance = AsyncMock() + mock_instance.get.return_value = mock_response + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await tool.execute(query="test", num_results=2) + + assert result.success + assert result.metadata["result_count"] == 2 + + @pytest.mark.anyio + async def test_search_num_results_clamped(self, tool, mock_search_response): + """Test num_results is clamped to max.""" + with patch("httpx.AsyncClient") as mock_client: + mock_response = MagicMock() + mock_response.json.return_value = mock_search_response + mock_response.raise_for_status = MagicMock() + + mock_instance = AsyncMock() + mock_instance.get.return_value = mock_response + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + # Request 100 but max is 10 + result = await tool.execute(query="test", num_results=100) + + assert result.success + # Should only get 3 (what's in mock response, capped at 10) + assert result.metadata["result_count"] <= 10 + + @pytest.mark.anyio + async def test_search_timeout_error(self, tool): + """Test timeout handling.""" + import httpx + + with patch("httpx.AsyncClient") as mock_client: + mock_instance = AsyncMock() + mock_instance.get.side_effect = httpx.TimeoutException("timeout") + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await tool.execute(query="test") + + assert not result.success + assert "timed out" in result.error.lower() + + @pytest.mark.anyio + async def test_search_request_error(self, tool): + """Test network error handling.""" + import httpx + + with patch("httpx.AsyncClient") as mock_client: + mock_instance = AsyncMock() + mock_instance.get.side_effect = httpx.RequestError("connection failed") + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await tool.execute(query="test") + + assert not result.success + assert "failed" in result.error.lower() + + @pytest.mark.anyio + async def test_search_includes_engines_metadata(self, tool, mock_search_response): + """Test engines used are included in metadata.""" + with patch("httpx.AsyncClient") as mock_client: + mock_response = MagicMock() + mock_response.json.return_value = mock_search_response + mock_response.raise_for_status = MagicMock() + + mock_instance = AsyncMock() + mock_instance.get.return_value = mock_response + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await tool.execute(query="test", num_results=3) + + assert result.success + engines = result.metadata.get("engines_used", []) + assert "google" in engines or "brave" in engines + + @pytest.mark.anyio + async def test_search_with_categories(self, tool, mock_search_response): + """Test category parameter is passed.""" + with patch("httpx.AsyncClient") as mock_client: + mock_response = MagicMock() + mock_response.json.return_value = mock_search_response + mock_response.raise_for_status = MagicMock() + + mock_instance = AsyncMock() + mock_instance.get.return_value = mock_response + mock_instance.__aenter__.return_value = mock_instance + mock_instance.__aexit__.return_value = None + mock_client.return_value = mock_instance + + result = await tool.execute(query="test", categories="it") + + assert result.success + # Verify get was called with categories parameter + call_args = mock_instance.get.call_args + assert "categories" in call_args.kwargs["params"] + assert call_args.kwargs["params"]["categories"] == "it"