feat: add web search tool using SearXNG

Add WebSearchTool that queries the self-hosted SearXNG metasearch engine for current information, documentation, and facts beyond training data. - Add SEARXNG_URL and SEARXNG_TIMEOUT config settings - Create WebSearchTool with query, num_results, categories params - Register web_search tool with explore agent - Add 10 tests for search functionality Usage: Agents can now use web_search(query="...") to find current info. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-11 14:09:21 +01:00
parent f6256363a2
commit 82a816a5b5
6 changed files with 434 additions and 2 deletions
@@ -11,6 +11,7 @@ from src.domains.tools.file.glob import GlobFilesTool
 from src.domains.tools.file.edit import EditFileTool
 from src.domains.tools.file.write import WriteFileTool
 from src.domains.tools.search.grep import GrepContentTool
+from src.domains.tools.search.web import WebSearchTool
 from src.domains.tools.shell.bash import BashReadOnlyTool
 from src.domains.tools.shell.bash_full import BashTool

@@ -271,3 +272,42 @@ def register_explore_tools(agent: Agent[AgentContext, str]) -> None:
            timeout=min(timeout, ctx.deps.timeout_seconds)
        )
        return result.to_string()
+
+    # === Web search ===
+
+    @agent.tool
+    async def web_search(
+        ctx: RunContext[AgentContext],
+        query: str,
+        num_results: int = 5,
+        categories: str | None = None
+    ) -> str:
+        """Search the web for current information.
+
+        Args:
+            query: Search query (e.g., "Python 3.12 new features")
+            num_results: Number of results to return (1-10, default: 5)
+            categories: Optional category filter ("general", "it", "news", "science")
+
+        Returns:
+            Search results with titles, URLs, and snippets.
+
+        Use this for:
+        - Current events or recent information
+        - Documentation updates since your training
+        - Facts you're uncertain about
+        - Technical references with URLs
+
+        IMPORTANT: Always include a "Sources:" section with URLs in your response.
+
+        Examples:
+            - query="FastAPI best practices 2024"
+            - query="CVE-2024" categories="it"
+        """
+        tool = WebSearchTool()
+        result = await tool.execute(
+            query=query,
+            num_results=num_results,
+            categories=categories
+        )
+        return result.to_string()
@@ -5,7 +5,7 @@ All tools inherit from BaseTool and return ToolResult.
 """
 from src.domains.tools.base import BaseTool, ToolResult
 from src.domains.tools.file import ReadFileTool, GlobFilesTool, EditFileTool, WriteFileTool
-from src.domains.tools.search import GrepContentTool
+from src.domains.tools.search import GrepContentTool, WebSearchTool
 from src.domains.tools.shell import BashReadOnlyTool, BashTool

 __all__ = [
@@ -16,6 +16,7 @@ __all__ = [
    "EditFileTool",
    "WriteFileTool",
    "GrepContentTool",
+    "WebSearchTool",
    "BashReadOnlyTool",
    "BashTool",
 ]
@@ -2,5 +2,6 @@
 Search tools.
 """
 from src.domains.tools.search.grep import GrepContentTool
+from src.domains.tools.search.web import WebSearchTool

-__all__ = ["GrepContentTool"]
+__all__ = ["GrepContentTool", "WebSearchTool"]
@@ -0,0 +1,169 @@
+"""
+Web search tool using SearXNG.
+"""
+from dataclasses import dataclass
+from datetime import datetime
+
+import httpx
+
+from src.domains.tools.base import BaseTool, ToolResult
+from src.shared.config import get_settings
+from src.shared.logging import logged, get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class SearchResult:
+    """A single search result."""
+    title: str
+    url: str
+    content: str
+    engine: str
+    published_date: str | None = None
+
+
+class WebSearchTool(BaseTool):
+    """
+    Search the web using SearXNG metasearch engine.
+
+    Returns relevant web results for queries about current events,
+    documentation, or anything beyond the LLM's knowledge cutoff.
+    """
+
+    name = "web_search"
+    description = """Search the web for current information.
+
+Args:
+    query: Search query string
+    num_results: Maximum results to return (default: 5, max: 10)
+    engines: Comma-separated engine list (optional, e.g., "google,brave,duckduckgo")
+    categories: Search category (optional: "general", "images", "news", "science", "it")
+
+Returns:
+    List of search results with title, URL, and snippet.
+    Include a "Sources:" section with URLs in your response.
+
+Examples:
+    - query="Python 3.12 new features" - Find latest Python docs
+    - query="FastAPI best practices 2024" - Find recent tutorials
+    - query="CVE-2024" categories="it" - Search IT/security news
+
+IMPORTANT:
+- Use this for current events, recent documentation, or facts you're unsure about
+- Always cite sources with URLs in your response
+- Today's date is {date} - use current year in queries for recent info
+""".format(date=datetime.now().strftime("%Y-%m-%d"))
+
+    def __init__(
+        self,
+        searxng_url: str | None = None,
+        timeout: int | None = None,
+        max_results: int = 10,
+    ):
+        """
+        Initialize WebSearchTool.
+
+        Args:
+            searxng_url: SearXNG instance URL (default: from config)
+            timeout: Request timeout in seconds (default: from config)
+            max_results: Maximum results to return
+        """
+        settings = get_settings()
+        self.searxng_url = (searxng_url or settings.searxng_url).rstrip("/")
+        self.timeout = timeout or settings.searxng_timeout
+        self.max_results = max_results
+
+    @logged()
+    async def execute(
+        self,
+        query: str,
+        num_results: int = 5,
+        engines: str | None = None,
+        categories: str | None = None,
+    ) -> ToolResult:
+        """
+        Execute web search.
+
+        Args:
+            query: Search query
+            num_results: Number of results (1-10)
+            engines: Specific engines to use
+            categories: Search category
+
+        Returns:
+            ToolResult with search results
+        """
+        if not query or not query.strip():
+            return self._error("Query cannot be empty")
+
+        num_results = min(max(1, num_results), self.max_results)
+
+        # Build SearXNG API request
+        params = {
+            "q": query.strip(),
+            "format": "json",
+        }
+
+        if engines:
+            params["engines"] = engines
+        if categories:
+            params["categories"] = categories
+
+        try:
+            async with httpx.AsyncClient(timeout=self.timeout) as client:
+                response = await client.get(
+                    f"{self.searxng_url}/search",
+                    params=params,
+                )
+                response.raise_for_status()
+                data = response.json()
+
+        except httpx.TimeoutException:
+            return self._error(f"Search timed out after {self.timeout}s")
+        except httpx.HTTPStatusError as e:
+            return self._error(f"Search failed: HTTP {e.response.status_code}")
+        except httpx.RequestError as e:
+            return self._error(f"Search request failed: {e}")
+        except Exception as e:
+            logger.exception(f"Unexpected search error: {e}")
+            return self._error(f"Search error: {e}")
+
+        # Parse results
+        raw_results = data.get("results", [])[:num_results]
+
+        if not raw_results:
+            return self._success(
+                f"No results found for: {query}",
+                result_count=0,
+                query=query,
+            )
+
+        # Format results for LLM consumption
+        results = []
+        for r in raw_results:
+            result = SearchResult(
+                title=r.get("title", "Untitled"),
+                url=r.get("url", ""),
+                content=r.get("content", "No description"),
+                engine=r.get("engine", "unknown"),
+                published_date=r.get("publishedDate"),
+            )
+            results.append(result)
+
+        # Format as readable text
+        output_lines = [f"Search results for: {query}", ""]
+        for i, r in enumerate(results, 1):
+            output_lines.append(f"{i}. **{r.title}**")
+            output_lines.append(f"   URL: {r.url}")
+            output_lines.append(f"   {r.content}")
+            if r.published_date:
+                output_lines.append(f"   Published: {r.published_date}")
+            output_lines.append("")
+
+        return self._success(
+            "\n".join(output_lines),
+            result_count=len(results),
+            query=query,
+            engines_used=list({r.engine for r in results}),
+        )
@@ -71,6 +71,10 @@ class Settings(BaseSettings):
    tatlock_api_url: str | None = "http://192.168.86.149:8000"
    internal_api_key: str | None = None

+    # Web search - SearXNG (use SEARXNG_URL env var to override)
+    searxng_url: str = "http://192.168.86.149:8087"
+    searxng_timeout: int = 10
+
    # Tool execution
    tool_timeout_seconds: int = 120
    sandbox_enabled: bool = True
@@ -0,0 +1,217 @@
+"""
+Tests for WebSearchTool.
+"""
+import pytest
+from unittest.mock import AsyncMock, patch, MagicMock
+
+from src.domains.tools.search.web import WebSearchTool
+
+
+class TestWebSearchTool:
+    """Tests for WebSearchTool."""
+
+    @pytest.fixture
+    def tool(self):
+        return WebSearchTool(searxng_url="http://localhost:8087", timeout=5)
+
+    @pytest.fixture
+    def mock_search_response(self):
+        """Sample SearXNG response."""
+        return {
+            "query": "test query",
+            "number_of_results": 3,
+            "results": [
+                {
+                    "title": "First Result",
+                    "url": "https://example.com/1",
+                    "content": "This is the first result content.",
+                    "engine": "google",
+                    "publishedDate": "2024-01-15",
+                },
+                {
+                    "title": "Second Result",
+                    "url": "https://example.com/2",
+                    "content": "This is the second result content.",
+                    "engine": "brave",
+                    "publishedDate": None,
+                },
+                {
+                    "title": "Third Result",
+                    "url": "https://example.com/3",
+                    "content": "This is the third result content.",
+                    "engine": "duckduckgo",
+                    "publishedDate": "2024-01-10",
+                },
+            ],
+        }
+
+    @pytest.mark.anyio
+    async def test_search_success(self, tool, mock_search_response):
+        """Test successful search."""
+        with patch("httpx.AsyncClient") as mock_client:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_search_response
+            mock_response.raise_for_status = MagicMock()
+
+            mock_instance = AsyncMock()
+            mock_instance.get.return_value = mock_response
+            mock_instance.__aenter__.return_value = mock_instance
+            mock_instance.__aexit__.return_value = None
+            mock_client.return_value = mock_instance
+
+            result = await tool.execute(query="test query", num_results=3)
+
+            assert result.success
+            assert "First Result" in result.data
+            assert "https://example.com/1" in result.data
+            assert result.metadata["result_count"] == 3
+
+    @pytest.mark.anyio
+    async def test_search_empty_query(self, tool):
+        """Test empty query is rejected."""
+        result = await tool.execute(query="")
+        assert not result.success
+        assert "empty" in result.error.lower()
+
+    @pytest.mark.anyio
+    async def test_search_whitespace_query(self, tool):
+        """Test whitespace-only query is rejected."""
+        result = await tool.execute(query="   ")
+        assert not result.success
+        assert "empty" in result.error.lower()
+
+    @pytest.mark.anyio
+    async def test_search_no_results(self, tool):
+        """Test when search returns no results."""
+        with patch("httpx.AsyncClient") as mock_client:
+            mock_response = MagicMock()
+            mock_response.json.return_value = {"query": "obscure", "results": []}
+            mock_response.raise_for_status = MagicMock()
+
+            mock_instance = AsyncMock()
+            mock_instance.get.return_value = mock_response
+            mock_instance.__aenter__.return_value = mock_instance
+            mock_instance.__aexit__.return_value = None
+            mock_client.return_value = mock_instance
+
+            result = await tool.execute(query="obscure nonexistent thing")
+
+            assert result.success
+            assert "No results" in result.data
+            assert result.metadata["result_count"] == 0
+
+    @pytest.mark.anyio
+    async def test_search_num_results_limit(self, tool, mock_search_response):
+        """Test num_results limits output."""
+        with patch("httpx.AsyncClient") as mock_client:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_search_response
+            mock_response.raise_for_status = MagicMock()
+
+            mock_instance = AsyncMock()
+            mock_instance.get.return_value = mock_response
+            mock_instance.__aenter__.return_value = mock_instance
+            mock_instance.__aexit__.return_value = None
+            mock_client.return_value = mock_instance
+
+            result = await tool.execute(query="test", num_results=2)
+
+            assert result.success
+            assert result.metadata["result_count"] == 2
+
+    @pytest.mark.anyio
+    async def test_search_num_results_clamped(self, tool, mock_search_response):
+        """Test num_results is clamped to max."""
+        with patch("httpx.AsyncClient") as mock_client:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_search_response
+            mock_response.raise_for_status = MagicMock()
+
+            mock_instance = AsyncMock()
+            mock_instance.get.return_value = mock_response
+            mock_instance.__aenter__.return_value = mock_instance
+            mock_instance.__aexit__.return_value = None
+            mock_client.return_value = mock_instance
+
+            # Request 100 but max is 10
+            result = await tool.execute(query="test", num_results=100)
+
+            assert result.success
+            # Should only get 3 (what's in mock response, capped at 10)
+            assert result.metadata["result_count"] <= 10
+
+    @pytest.mark.anyio
+    async def test_search_timeout_error(self, tool):
+        """Test timeout handling."""
+        import httpx
+
+        with patch("httpx.AsyncClient") as mock_client:
+            mock_instance = AsyncMock()
+            mock_instance.get.side_effect = httpx.TimeoutException("timeout")
+            mock_instance.__aenter__.return_value = mock_instance
+            mock_instance.__aexit__.return_value = None
+            mock_client.return_value = mock_instance
+
+            result = await tool.execute(query="test")
+
+            assert not result.success
+            assert "timed out" in result.error.lower()
+
+    @pytest.mark.anyio
+    async def test_search_request_error(self, tool):
+        """Test network error handling."""
+        import httpx
+
+        with patch("httpx.AsyncClient") as mock_client:
+            mock_instance = AsyncMock()
+            mock_instance.get.side_effect = httpx.RequestError("connection failed")
+            mock_instance.__aenter__.return_value = mock_instance
+            mock_instance.__aexit__.return_value = None
+            mock_client.return_value = mock_instance
+
+            result = await tool.execute(query="test")
+
+            assert not result.success
+            assert "failed" in result.error.lower()
+
+    @pytest.mark.anyio
+    async def test_search_includes_engines_metadata(self, tool, mock_search_response):
+        """Test engines used are included in metadata."""
+        with patch("httpx.AsyncClient") as mock_client:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_search_response
+            mock_response.raise_for_status = MagicMock()
+
+            mock_instance = AsyncMock()
+            mock_instance.get.return_value = mock_response
+            mock_instance.__aenter__.return_value = mock_instance
+            mock_instance.__aexit__.return_value = None
+            mock_client.return_value = mock_instance
+
+            result = await tool.execute(query="test", num_results=3)
+
+            assert result.success
+            engines = result.metadata.get("engines_used", [])
+            assert "google" in engines or "brave" in engines
+
+    @pytest.mark.anyio
+    async def test_search_with_categories(self, tool, mock_search_response):
+        """Test category parameter is passed."""
+        with patch("httpx.AsyncClient") as mock_client:
+            mock_response = MagicMock()
+            mock_response.json.return_value = mock_search_response
+            mock_response.raise_for_status = MagicMock()
+
+            mock_instance = AsyncMock()
+            mock_instance.get.return_value = mock_response
+            mock_instance.__aenter__.return_value = mock_instance
+            mock_instance.__aexit__.return_value = None
+            mock_client.return_value = mock_instance
+
+            result = await tool.execute(query="test", categories="it")
+
+            assert result.success
+            # Verify get was called with categories parameter
+            call_args = mock_instance.get.call_args
+            assert "categories" in call_args.kwargs["params"]
+            assert call_args.kwargs["params"]["categories"] == "it"