From 100ebeae5270bcedee659cdc4849b8d8fc9b9a86 Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Mon, 15 Dec 2025 18:30:16 +0100 Subject: [PATCH] feat: migrate web search from tatlock_core to Librarian MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move web search functionality to The Librarian agent, integrating with the library-desk /rag/search endpoint for enhanced search capabilities. Changes: - Add search_web, read_url, read_urls_batch tools to Librarian - Add WebSearchResult, ContentExtractionResult models to client - Add search_web, extract_content, extract_content_batch client methods - Update Librarian capability with web/url/internet domains - Remove search_web from tatlock_core tools and toolset - Update Tatlock system prompt to delegate web search to Librarian - Add comprehensive unit tests for new Librarian tools - Clean up legacy src/agents/tools.py 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- CHANGELOG.md | 24 ++ docs/rag_search_implementation.md | 348 +++++++++++++++++++++ src/agents/librarian/capability.py | 10 +- src/agents/librarian/client.py | 221 +++++++++++++ src/agents/librarian/tools.py | 239 ++++++++++++++- src/agents/tatlock.py | 36 +-- src/agents/tatlock_core/__init__.py | 5 +- src/agents/tatlock_core/capability.py | 6 +- src/agents/tatlock_core/tools.py | 95 +----- src/agents/tatlock_core/toolset.py | 14 +- src/agents/tools.py | 100 +----- tests/agents/librarian/test_tools.py | 426 ++++++++++++++++++++++++++ tests/agents/test_tools.py | 188 +----------- 13 files changed, 1287 insertions(+), 425 deletions(-) create mode 100644 docs/rag_search_implementation.md create mode 100644 tests/agents/librarian/test_tools.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d69ef48..9063073 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +#### Web Search Migration to Librarian +- **`search_web()`** tool in Librarian for web search via library-desk `/rag/search` endpoint +- **`read_url()`** tool for single URL content extraction via Trafilatura +- **`read_urls_batch()`** tool for parallel batch URL extraction (max 20 URLs) +- `WebSearchResult`, `WebSearchResponse` models in LibraryDeskClient +- `ContentExtractionResult`, `BatchExtractionResponse` models for content extraction +- `search_web()`, `extract_content()`, `extract_content_batch()` methods in LibraryDeskClient +- Comprehensive unit tests for new Librarian tools (`tests/agents/librarian/test_tools.py`) + +### Changed + +- Librarian capability updated with web search domains: "web", "url", "internet" +- Tatlock system prompt now delegates web search to Librarian +- `tatlock_core` capability reduced to computation/datetime only (no longer requires network) + +### Removed + +- `search_web` function from `src/agents/tatlock_core/tools.py` +- `web_search_tool` from `tatlock_core_tools` list +- `search_web` from legacy `src/agents/tools.py` +- Search tests from `tests/agents/test_tools.py` (moved to Librarian tests) + ## [1.6.0] - 2025-12-15 ### Added diff --git a/docs/rag_search_implementation.md b/docs/rag_search_implementation.md new file mode 100644 index 0000000..14572fe --- /dev/null +++ b/docs/rag_search_implementation.md @@ -0,0 +1,348 @@ +# Tatlock Integration Guide + +Implementation instructions for integrating Library Desk search and content extraction endpoints into the Tatlock project. + +## Base Configuration + +``` +BASE_URL: http://library-desk:8089 (or your deployment URL) +AUTH_HEADER: Authorization: Bearer +``` + +--- + +## 1. RAG Search Endpoint + +**Use case:** Librarian needs to research a topic by searching the web. + +### Endpoint + +``` +POST /rag/search +``` + +### Request + +```json +{ + "query": "Python async programming best practices", + "search_type": "web", + "limit": 10, + "user": "tatlock-librarian" +} +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `query` | string | required | Search query (1-500 chars) | +| `search_type` | enum | `"web"` | `"web"`, `"news"`, or `"images"` | +| `limit` | int | 10 | Results to return (1-20) | +| `user` | string | `"default"` | User identifier for tracking | + +### Response + +```json +{ + "query": "Python async programming best practices", + "search_type": "web", + "results": [ + { + "title": "Async IO in Python: A Complete Walkthrough", + "url": "https://realpython.com/async-io-python/", + "content": "Full extracted article text via Trafilatura (~2000 chars max)...", + "snippet": "Original search engine snippet (150-300 chars)...", + "source": "realpython.com", + "published_date": "2023-05-15" + } + ], + "total_results": 10, + "search_time_ms": 2340, + "sources_summary": "## Sources\n- [Async IO in Python](https://realpython.com/async-io-python/)\n- ..." +} +``` + +### Key Fields for Tatlock + +| Field | Usage | +|-------|-------| +| `results[].content` | Full extracted text - use this for LLM context | +| `results[].snippet` | Fallback if content extraction failed | +| `sources_summary` | Pre-formatted markdown for citations | + +### Error Handling + +| HTTP Code | Meaning | Action | +|-----------|---------|--------| +| 400 | Invalid query | Check query length/format | +| 502 | SearXNG unavailable | Retry with backoff | +| 504 | Search timeout | Retry or reduce limit | +| 500 | Internal error | Log and notify | + +### Example Usage (Python) + +```python +import httpx + +async def search_web(query: str, limit: int = 10) -> dict: + async with httpx.AsyncClient() as client: + response = await client.post( + f"{BASE_URL}/rag/search", + headers={"Authorization": f"Bearer {API_KEY}"}, + json={ + "query": query, + "search_type": "web", + "limit": limit, + "user": "tatlock-librarian" + }, + timeout=30.0 + ) + response.raise_for_status() + return response.json() + +# Usage +results = await search_web("machine learning transformers") +for r in results["results"]: + # Prefer full content, fall back to snippet + text = r["content"] or r["snippet"] + print(f"{r['title']}: {len(text)} chars") +``` + +--- + +## 2. Content Extraction Endpoint + +**Use case:** Librarian has a specific URL and needs to read its content. + +### Single URL Extraction + +``` +POST /content/extract +``` + +#### Request + +```json +{ + "url": "https://example.com/article", + "include_metadata": true, + "max_length": 2000 +} +``` + +#### Response + +```json +{ + "result": { + "url": "https://example.com/article", + "title": "Article Title", + "content": "Extracted main text content...", + "author": "John Doe", + "date": "2024-01-15", + "language": "en", + "success": true, + "error": null + }, + "extraction_time_ms": 1250 +} +``` + +### Batch URL Extraction + +``` +POST /content/extract/batch +``` + +#### Request + +```json +{ + "urls": [ + "https://example.com/article1", + "https://example.com/article2", + "https://example.com/article3" + ], + "include_metadata": true, + "max_length": 2000 +} +``` + +#### Response + +```json +{ + "results": [ + { + "url": "https://example.com/article1", + "title": "Article 1", + "content": "Extracted content...", + "success": true, + "error": null + }, + { + "url": "https://example.com/article2", + "title": null, + "content": "", + "success": false, + "error": "Connection timeout" + } + ], + "total_urls": 3, + "successful": 2, + "failed": 1, + "extraction_time_ms": 3500 +} +``` + +--- + +## 3. Error Pattern: Soft Failures + +> **Important:** Content extraction uses a **soft failure pattern** - individual URL failures do NOT throw HTTP errors. + +### Why Soft Failures? + +When extracting content from multiple URLs (batch) or even single URLs: +- Some sites block bots +- Some URLs are temporarily down +- Some pages have no extractable content + +Instead of failing the entire request, we return: +- `success: true/false` per result +- `error: "reason"` when failed +- Empty `content: ""` on failure + +### Handling Soft Failures + +```python +async def extract_with_fallback(url: str) -> str: + response = await client.post( + f"{BASE_URL}/content/extract", + headers={"Authorization": f"Bearer {API_KEY}"}, + json={"url": url} + ) + response.raise_for_status() # Only throws on 4xx/5xx + + data = response.json() + result = data["result"] + + if result["success"]: + return result["content"] + else: + # Log the failure, return empty or handle gracefully + logger.warning(f"Extraction failed for {url}: {result['error']}") + return "" # Or raise, or use cached version, etc. +``` + +### Batch Processing Example + +```python +async def extract_batch_with_stats(urls: list[str]) -> dict: + response = await client.post( + f"{BASE_URL}/content/extract/batch", + headers={"Authorization": f"Bearer {API_KEY}"}, + json={"urls": urls, "max_length": 3000} + ) + response.raise_for_status() + + data = response.json() + + # Separate successful and failed + successful = [r for r in data["results"] if r["success"]] + failed = [r for r in data["results"] if not r["success"]] + + if failed: + logger.warning(f"{len(failed)} URLs failed extraction:") + for f in failed: + logger.warning(f" {f['url']}: {f['error']}") + + return { + "contents": {r["url"]: r["content"] for r in successful}, + "failed_urls": [f["url"] for f in failed], + "success_rate": data["successful"] / data["total_urls"] + } +``` + +--- + +## 4. Recommended Patterns for Tatlock + +### Research Flow + +```python +async def librarian_research(topic: str) -> dict: + """ + Full research flow: search + extract additional context. + """ + # 1. Search for relevant pages + search_results = await search_web(topic, limit=10) + + # 2. RAG search already includes extracted content + # Only extract more if you need deeper content + + # 3. Build context for LLM + context_parts = [] + for r in search_results["results"]: + content = r["content"] or r["snippet"] + if content: + context_parts.append(f"## {r['title']}\nSource: {r['url']}\n\n{content}") + + return { + "context": "\n\n---\n\n".join(context_parts), + "sources": search_results["sources_summary"], + "result_count": search_results["total_results"] + } +``` + +### Reading a Specific Page + +```python +async def librarian_read_page(url: str) -> str: + """ + Read a specific URL the user provided. + """ + response = await client.post( + f"{BASE_URL}/content/extract", + headers={"Authorization": f"Bearer {API_KEY}"}, + json={"url": url, "max_length": 5000} # Longer for deep reads + ) + response.raise_for_status() + + result = response.json()["result"] + + if not result["success"]: + raise ValueError(f"Could not read page: {result['error']}") + + # Format for LLM + header = f"# {result['title'] or 'Untitled'}\n" + if result["author"]: + header += f"Author: {result['author']}\n" + if result["date"]: + header += f"Date: {result['date']}\n" + + return header + "\n" + result["content"] +``` + +--- + +## 5. Rate Limits & Best Practices + +| Recommendation | Reason | +|----------------|--------| +| Use `limit: 5-10` for searches | More results = longer extraction time | +| Batch URLs when possible | More efficient than sequential calls | +| Max 20 URLs per batch | Server limit | +| Set reasonable timeouts (30s) | Content extraction can be slow | +| Cache results client-side | Same URL rarely changes content | +| Use `user` parameter | Helps with debugging and rate limiting | + +--- + +## 6. Quick Reference + +| Endpoint | Method | Use Case | +|----------|--------|----------| +| `/rag/search` | POST | Search web + get extracted content | +| `/content/extract` | POST | Read a single URL | +| `/content/extract/batch` | POST | Read multiple URLs | +| `/health` | GET | Check service status | diff --git a/src/agents/librarian/capability.py b/src/agents/librarian/capability.py index 2e3494d..24ab07b 100644 --- a/src/agents/librarian/capability.py +++ b/src/agents/librarian/capability.py @@ -21,10 +21,11 @@ LIBRARIAN_CAPABILITY = HouseholdCapability( role="The Librarian", category="research", description=( - "Research and wiki management: can CREATE wiki pages about topics " + "Research, web search, and wiki management: can SEARCH the web for current " + "information, READ URLs/articles, CREATE wiki pages about topics " "(with automatic HybridRAG research), UPDATE existing pages, " - "SEARCH wiki/knowledge graph/web, and synthesize information. " - "Use for: 'create a page about X', 'update wiki', 'find info on X'" + "and synthesize information from multiple sources. " + "Use for: 'search for X', 'what is X', 'create a page about X', 'read this URL'" ), domains=[ "research", @@ -33,6 +34,9 @@ LIBRARIAN_CAPABILITY = HouseholdCapability( "wiki", "documents", "search", + "web", + "url", + "internet", "synthesis", "create", "write", diff --git a/src/agents/librarian/client.py b/src/agents/librarian/client.py index 7be3673..5256909 100644 --- a/src/agents/librarian/client.py +++ b/src/agents/librarian/client.py @@ -98,6 +98,47 @@ class ResearchSummary(BaseModel): timing_ms: int = 0 +class WebSearchResult(BaseModel): + """Result from web search via /rag/search.""" + title: str + url: str + content: str = "" # Full extracted text via Trafilatura + snippet: str = "" # Original search engine snippet + source: str = "" # Domain name + published_date: Optional[str] = None + + +class WebSearchResponse(BaseModel): + """Response from /rag/search endpoint.""" + query: str + search_type: str + results: list[WebSearchResult] = Field(default_factory=list) + total_results: int = 0 + search_time_ms: int = 0 + sources_summary: str = "" # Pre-formatted markdown citations + + +class ContentExtractionResult(BaseModel): + """Result from content extraction.""" + url: str + title: Optional[str] = None + content: str = "" + author: Optional[str] = None + date: Optional[str] = None + language: Optional[str] = None + success: bool = True + error: Optional[str] = None + + +class BatchExtractionResponse(BaseModel): + """Response from batch content extraction.""" + results: list[ContentExtractionResult] = Field(default_factory=list) + total_urls: int = 0 + successful: int = 0 + failed: int = 0 + extraction_time_ms: int = 0 + + class EntityLinking(BaseModel): """Entity linking results from smart-create.""" forward_links: int = 0 @@ -685,6 +726,186 @@ class LibraryDeskClient: logger.warning("library_desk_health_check_failed", error=str(e)) return False + # ======================================================================== + # RAG Search (Web Search with Content Extraction) + # ======================================================================== + + async def search_web( + self, + query: str, + user: str | None = None, + search_type: str = "web", + limit: int = 10, + ) -> WebSearchResponse: + """ + Search the web and extract content from results. + + Uses SearXNG for search and Trafilatura for content extraction. + Returns both snippets and full extracted text. + + Args: + query: Search query (1-500 chars) + user: User identifier for tracking + search_type: "web", "news", or "images" + limit: Number of results (1-20) + + Returns: + WebSearchResponse with results and pre-formatted sources + """ + user = user or get_user() + client = self._ensure_client() + + payload = { + "query": query, + "search_type": search_type, + "limit": limit, + "user": user or "tatlock-librarian", + } + + logger.info("library_desk_web_search", query=query, limit=limit) + + response = await client.post("/rag/search", json=payload, timeout=30.0) + response.raise_for_status() + + data = response.json() + + results = [ + WebSearchResult( + title=r.get("title", ""), + url=r.get("url", ""), + content=r.get("content", ""), + snippet=r.get("snippet", ""), + source=r.get("source", ""), + published_date=r.get("published_date"), + ) + for r in data.get("results", []) + ] + + return WebSearchResponse( + query=data.get("query", query), + search_type=data.get("search_type", search_type), + results=results, + total_results=data.get("total_results", len(results)), + search_time_ms=data.get("search_time_ms", 0), + sources_summary=data.get("sources_summary", ""), + ) + + # ======================================================================== + # Content Extraction + # ======================================================================== + + async def extract_content( + self, + url: str, + include_metadata: bool = True, + max_length: int = 5000, + ) -> ContentExtractionResult: + """ + Extract main content from a URL. + + Uses Trafilatura for intelligent content extraction, + removing boilerplate, ads, and navigation. + + Note: Uses soft failure pattern - check result.success field. + + Args: + url: URL to extract content from + include_metadata: Whether to extract author, date, etc. + max_length: Maximum content length + + Returns: + ContentExtractionResult (check .success and .error fields) + """ + client = self._ensure_client() + + payload = { + "url": url, + "include_metadata": include_metadata, + "max_length": max_length, + } + + logger.debug("library_desk_extract_content", url=url) + + response = await client.post("/content/extract", json=payload, timeout=30.0) + response.raise_for_status() + + data = response.json() + result = data.get("result", {}) + + return ContentExtractionResult( + url=result.get("url", url), + title=result.get("title"), + content=result.get("content", ""), + author=result.get("author"), + date=result.get("date"), + language=result.get("language"), + success=result.get("success", False), + error=result.get("error"), + ) + + async def extract_content_batch( + self, + urls: list[str], + include_metadata: bool = True, + max_length: int = 2000, + ) -> BatchExtractionResponse: + """ + Extract content from multiple URLs in parallel. + + More efficient than sequential calls. Max 20 URLs per batch. + + Note: Uses soft failure pattern - individual failures don't + throw errors, check each result's .success field. + + Args: + urls: List of URLs to extract (max 20) + include_metadata: Whether to extract author, date, etc. + max_length: Maximum content length per URL + + Returns: + BatchExtractionResponse with results and stats + """ + client = self._ensure_client() + + payload = { + "urls": urls[:20], # Server limit + "include_metadata": include_metadata, + "max_length": max_length, + } + + logger.info("library_desk_extract_batch", url_count=len(urls)) + + response = await client.post( + "/content/extract/batch", + json=payload, + timeout=60.0, # Longer timeout for batch + ) + response.raise_for_status() + + data = response.json() + + results = [ + ContentExtractionResult( + url=r.get("url", ""), + title=r.get("title"), + content=r.get("content", ""), + author=r.get("author"), + date=r.get("date"), + language=r.get("language"), + success=r.get("success", False), + error=r.get("error"), + ) + for r in data.get("results", []) + ] + + return BatchExtractionResponse( + results=results, + total_urls=data.get("total_urls", len(urls)), + successful=data.get("successful", 0), + failed=data.get("failed", 0), + extraction_time_ms=data.get("extraction_time_ms", 0), + ) + # Global client factory async def get_library_client() -> LibraryDeskClient: diff --git a/src/agents/librarian/tools.py b/src/agents/librarian/tools.py index 9dc26a3..d07f183 100644 --- a/src/agents/librarian/tools.py +++ b/src/agents/librarian/tools.py @@ -432,6 +432,239 @@ async def find_related_entities( return f"Error finding related entities: {str(e)}" +# ============================================================================ +# Web Search & Content Extraction +# ============================================================================ + +async def search_web( + query: str, + limit: int = 10, + search_type: str = "web", +) -> str: + """ + Search the web and extract content from results. + + This is the primary tool for finding current information online. + Results include both snippets and full extracted text from pages. + + Search types: + - "web": General web search (default) + - "news": News articles + - "images": Image search + + Args: + query: Search query (1-500 chars) + limit: Number of results (1-20, default: 10) + search_type: Type of search ("web", "news", or "images") + + Returns: + Formatted search results with sources and extracted content + + Examples: + search_web("Python 3.12 new features") + search_web("latest tech news", search_type="news", limit=5) + """ + try: + async with LibraryDeskClient() as client: + response = await client.search_web( + query=query, + limit=limit, + search_type=search_type, + ) + + if not response.results: + return f"No results found for '{query}'" + + output_parts = [f"## Web Search: {query}\n"] + output_parts.append(f"*Found {response.total_results} results in {response.search_time_ms}ms*\n") + + for i, result in enumerate(response.results, 1): + output_parts.append(f"### {i}. {result.title}") + output_parts.append(f"**Source:** {result.source}") + output_parts.append(f"**URL:** {result.url}") + + if result.published_date: + output_parts.append(f"**Date:** {result.published_date}") + + # Use full content if available, otherwise snippet + content = result.content or result.snippet + if content: + # Truncate for readability + if len(content) > 500: + content = content[:500] + "..." + output_parts.append(f"\n{content}") + + output_parts.append("") + + # Add pre-formatted sources for citations + if response.sources_summary: + output_parts.append("---") + output_parts.append(response.sources_summary) + + logger.info( + "librarian_web_search", + query=query, + result_count=response.total_results, + search_type=search_type, + ) + + return "\n".join(output_parts) + + except Exception as e: + logger.error("librarian_web_search_error", error=str(e), query=query) + return f"Error searching web: {str(e)}" + + +async def read_url( + url: str, + max_length: int = 5000, +) -> str: + """ + Read and extract the main content from a URL. + + Use this when you have a specific URL to read, such as: + - A link the user provided + - A URL from search results you want to read in full + - Documentation or article pages + + Extracts the main content, removing ads, navigation, and boilerplate. + + Args: + url: The URL to read + max_length: Maximum content length (default: 5000) + + Returns: + Extracted page content with metadata + + Examples: + read_url("https://docs.python.org/3/library/asyncio.html") + read_url("https://example.com/article", max_length=10000) + """ + try: + async with LibraryDeskClient() as client: + result = await client.extract_content( + url=url, + include_metadata=True, + max_length=max_length, + ) + + if not result.success: + return f"Could not read page: {result.error or 'Unknown error'}" + + output_parts = [] + + # Header with metadata + if result.title: + output_parts.append(f"# {result.title}") + else: + output_parts.append(f"# Content from {url}") + + output_parts.append(f"**URL:** {url}") + + if result.author: + output_parts.append(f"**Author:** {result.author}") + + if result.date: + output_parts.append(f"**Date:** {result.date}") + + if result.language and result.language != "en": + output_parts.append(f"**Language:** {result.language}") + + output_parts.append("") + + # Main content + if result.content: + output_parts.append(result.content) + else: + output_parts.append("(No content could be extracted)") + + logger.info( + "librarian_read_url", + url=url, + content_length=len(result.content) if result.content else 0, + ) + + return "\n".join(output_parts) + + except Exception as e: + logger.error("librarian_read_url_error", error=str(e), url=url) + return f"Error reading URL: {str(e)}" + + +async def read_urls_batch( + urls: list[str], + max_length: int = 2000, +) -> str: + """ + Read and extract content from multiple URLs in parallel. + + More efficient than calling read_url multiple times. + Max 20 URLs per batch. + + Note: Individual failures don't fail the entire batch - + failed URLs are reported but other content is still returned. + + Args: + urls: List of URLs to read (max 20) + max_length: Maximum content length per URL (default: 2000) + + Returns: + Extracted content from all successful URLs with failure report + + Examples: + read_urls_batch(["https://example.com/1", "https://example.com/2"]) + """ + try: + async with LibraryDeskClient() as client: + response = await client.extract_content_batch( + urls=urls, + include_metadata=True, + max_length=max_length, + ) + + output_parts = [ + f"## Batch Content Extraction", + f"*Extracted {response.successful}/{response.total_urls} URLs in {response.extraction_time_ms}ms*\n", + ] + + # Show successful extractions + for result in response.results: + if result.success: + title = result.title or result.url + output_parts.append(f"### {title}") + output_parts.append(f"**URL:** {result.url}") + + if result.content: + # Truncate for readability in batch mode + content = result.content + if len(content) > max_length: + content = content[:max_length] + "..." + output_parts.append(f"\n{content}") + + output_parts.append("") + + # Report failures + failed = [r for r in response.results if not r.success] + if failed: + output_parts.append("---") + output_parts.append("### Failed Extractions") + for result in failed: + output_parts.append(f"- {result.url}: {result.error}") + + logger.info( + "librarian_read_urls_batch", + total=response.total_urls, + successful=response.successful, + failed=response.failed, + ) + + return "\n".join(output_parts) + + except Exception as e: + logger.error("librarian_read_urls_batch_error", error=str(e)) + return f"Error reading URLs: {str(e)}" + + # ============================================================================ # Wiki Write Operations # ============================================================================ @@ -685,7 +918,7 @@ async def smart_create_wiki_page( # All tools available to The Librarian LIBRARIAN_TOOLS = [ - # Research tools + # Research tools (internal knowledge) hybrid_search, search_wiki, get_wiki_page, @@ -694,6 +927,10 @@ LIBRARIAN_TOOLS = [ semantic_search, explore_knowledge_graph, find_related_entities, + # Web search & content extraction + search_web, + read_url, + read_urls_batch, # Write tools create_wiki_page, update_wiki_page, diff --git a/src/agents/tatlock.py b/src/agents/tatlock.py index 6944fce..362917c 100644 --- a/src/agents/tatlock.py +++ b/src/agents/tatlock.py @@ -17,7 +17,6 @@ from src.agents.tatlock_core.tools import ( get_current_datetime, calculate_time_offset, time_difference, - search_web, ) from src.core.config import config from src.core.logging_config import get_logger @@ -76,18 +75,17 @@ You have direct access to several permanent tools that you should USE whenever a - time_difference: Calculate the time between two dates - Use these for ANY date/time queries - never guess at dates or times -3. **Web Search** (search_web): Search for current, volatile, or factual information - - Use this for ANY information that might be current, factual, or outside your training data +3. **Web Search** (via Librarian): For current, volatile, or factual information + - Delegate to the Librarian for web searches and research - Examples: news, current events, recent developments, specific facts, technical documentation - - Always prefer searching over guessing or using potentially outdated knowledge - - For extensive research questions, note that this will later be delegated to the librarian + - Use: delegate_to_librarian(task="search the web for ...") ## Tool Usage Guidelines - **Mathematics**: ALWAYS use the calculator tool, even for simple arithmetic - **Dates/Times**: ALWAYS use the date/time tools, never guess or estimate -- **Current Information**: ALWAYS search for facts, news, or volatile information -- **Verification**: When facts are important, use search to verify rather than rely on memory alone +- **Current Information**: Delegate web searches to the Librarian +- **Verification**: When facts are important, delegate to Librarian for research - When you use a tool, explain what you're doing in a butler-appropriate manner - Present tool results naturally in your response @@ -238,28 +236,8 @@ class TatlockAgent(AgentInterface): ctx.deps.log_call(f"🕐 Calculating time difference between {date1_str} and {date2_str}") return time_difference(date1_str, date2_str) - # Web search tool - @self._agent.tool - async def web_search(ctx: RunContext[ToolCallTracker], query: str, num_results: int = 5) -> str: - """ - Search the web using SearXNG for current information. - - Use this tool for ANY information that might be: - - Current or time-sensitive (news, events, recent developments) - - Factual and verifiable (statistics, technical specs, definitions) - - Outside your training data or knowledge cutoff - - Args: - query: Search query string - num_results: Number of results to return (default: 5, max: 10) - - Returns: - Formatted search results with titles, URLs, and snippets - """ - # Log the search query to reasoning output - if ctx.deps: - ctx.deps.log_call(f"🔍 Searching for: '{query}'") - return await search_web(query, num_results) + # NOTE: Web search has been moved to The Librarian agent. + # Use delegate_to_librarian(task="search web for ...") for web search. @property def agent(self): diff --git a/src/agents/tatlock_core/__init__.py b/src/agents/tatlock_core/__init__.py index 19ae66c..ce6677a 100644 --- a/src/agents/tatlock_core/__init__.py +++ b/src/agents/tatlock_core/__init__.py @@ -1,7 +1,8 @@ """ Tatlock's core tools package. -Provides calculator, date/time, and web search capabilities. +Provides calculator and date/time capabilities. +Web search has been moved to The Librarian agent. Organized as a household member with toolset and capability registration. """ from .capability import TATLOCK_CORE_CAPABILITY, get_capability @@ -10,7 +11,6 @@ from .tools import ( calculate, calculate_time_offset, get_current_datetime, - search_web, time_difference, ) @@ -20,7 +20,6 @@ __all__ = [ "get_current_datetime", "calculate_time_offset", "time_difference", - "search_web", # Toolset "tatlock_core_tools", "get_core_tools", diff --git a/src/agents/tatlock_core/capability.py b/src/agents/tatlock_core/capability.py index 573aa31..ade3b85 100644 --- a/src/agents/tatlock_core/capability.py +++ b/src/agents/tatlock_core/capability.py @@ -11,10 +11,10 @@ TATLOCK_CORE_CAPABILITY = HouseholdCapability( name="tatlock_core", role="Butler's Core Tools", category="core", - description="Essential tools for computation, date/time operations, and web searches", - domains=["computation", "datetime", "information", "research"], + description="Essential tools for computation and date/time operations", + domains=["computation", "datetime", "math", "calculator"], cost="low", - requires_network=True, # For web search + requires_network=False, # Web search moved to Librarian ) diff --git a/src/agents/tatlock_core/tools.py b/src/agents/tatlock_core/tools.py index fb7265c..12e9ffc 100644 --- a/src/agents/tatlock_core/tools.py +++ b/src/agents/tatlock_core/tools.py @@ -256,96 +256,5 @@ def time_difference(date1_str: str, date2_str: str = "now") -> str: return f"Error calculating time difference: {str(e)}" -# ============================================================================ -# SearXNG Search Tool -# ============================================================================ - -async def search_web(query: str, num_results: int = 5) -> str: - """ - Search the web using SearXNG. - - Args: - query: Search query string - num_results: Number of results to return (default: 5, max: 10) - - Returns: - Formatted search results as a string with titles, URLs, and snippets - - Examples: - search_web("Python async programming") -> "1. Title: ...\n URL: ...\n ..." - """ - try: - # Limit results - num_results = min(num_results, 10) - - # Get SearXNG host with fallback logic - searxng_host = str(config.SEARXNG_HOST) - - # Try production host first, fall back to localhost in development - hosts_to_try = [searxng_host] - if config.ENVIRONMENT.value == "development" and "localhost" not in searxng_host: - # Add localhost fallback for development - hosts_to_try.append("http://localhost:8087") - - last_error = None - - for host in hosts_to_try: - try: - logger.debug("searxng_search_attempt", host=host, query=query) - - async with httpx.AsyncClient(timeout=config.SEARXNG_TIMEOUT) as client: - response = await client.get( - f"{host}/search", - params={ - "q": query, - "format": "json", - "pageno": 1, - } - ) - - if response.status_code == 200: - data = response.json() - results = data.get("results", []) - - if not results: - return f"No results found for '{query}'" - - # Format results - formatted_results = [] - for i, result in enumerate(results[:num_results], 1): - title = result.get("title", "No title") - url = result.get("url", "") - content = result.get("content", "No description available") - - formatted_results.append( - f"{i}. {title}\n" - f" URL: {url}\n" - f" {content}\n" - ) - - logger.info( - "searxng_search_success", - host=host, - query=query, - result_count=len(results), - ) - return "\n".join(formatted_results) - else: - last_error = f"SearXNG returned status {response.status_code}" - - except httpx.ConnectError: - last_error = f"Cannot connect to SearXNG at {host}" - logger.warning("searxng_connection_failed", host=host) - continue - except Exception as e: - last_error = str(e) - logger.warning("searxng_error", host=host, error=str(e)) - continue - - # All hosts failed - logger.error("searxng_all_hosts_failed", error=last_error) - return f"Error searching: {last_error}. Please check that SearXNG is running." - - except Exception as e: - logger.error("searxng_unexpected_error", error=str(e), exc_info=True) - return f"Error searching: {str(e)}" +# NOTE: Web search has been moved to The Librarian agent. +# Use delegate_to_librarian(task="search web for ...") for web search. diff --git a/src/agents/tatlock_core/toolset.py b/src/agents/tatlock_core/toolset.py index 7b4f19f..737f124 100644 --- a/src/agents/tatlock_core/toolset.py +++ b/src/agents/tatlock_core/toolset.py @@ -55,17 +55,8 @@ time_difference_tool = Tool( ), ) -web_search_tool = Tool( - function=tools.search_web, - name="search_web", - description=( - "Search the web using SearXNG for current information. " - "Use this to find recent events, current data, or verify facts. " - "Returns formatted results with titles, URLs, and snippets. " - "Useful for information that may have changed since training data." - ), - takes_ctx=False, -) +# NOTE: Web search has been moved to The Librarian agent. +# Use delegate_to_librarian(task="search web for ...") for web search. # Combined toolset of all core tools @@ -74,7 +65,6 @@ tatlock_core_tools = [ current_datetime_tool, time_offset_tool, time_difference_tool, - web_search_tool, ] diff --git a/src/agents/tools.py b/src/agents/tools.py index 9f8a28a..58efe3d 100644 --- a/src/agents/tools.py +++ b/src/agents/tools.py @@ -4,20 +4,14 @@ Tatlock's permanent tools. These tools are always available to the butler agent: - Calculator: For all mathematical operations - Date/Time toolkit: For current time and time calculations -- SearXNG search: For searching the web for current information + +Note: Web search has been moved to The Librarian agent. +See src/agents/librarian/tools.py for search_web functionality. """ -import logging import math import re from datetime import datetime, timedelta -from typing import Any - -import httpx - -from src.core.config import config - -logger = logging.getLogger(__name__) # ============================================================================ @@ -256,91 +250,3 @@ def time_difference(date1_str: str, date2_str: str = "now") -> str: except Exception as e: return f"Error calculating time difference: {str(e)}" - - -# ============================================================================ -# SearXNG Search Tool -# ============================================================================ - -async def search_web(query: str, num_results: int = 5) -> str: - """ - Search the web using SearXNG. - - Args: - query: Search query string - num_results: Number of results to return (default: 5, max: 10) - - Returns: - Formatted search results as a string with titles, URLs, and snippets - - Examples: - search_web("Python async programming") -> "1. Title: ...\n URL: ...\n ..." - """ - try: - # Limit results - num_results = min(num_results, 10) - - # Get SearXNG host with fallback logic - searxng_host = str(config.SEARXNG_HOST) - - # Try production host first, fall back to localhost in development - hosts_to_try = [searxng_host] - if config.ENVIRONMENT.value == "development" and "localhost" not in searxng_host: - # Add localhost fallback for development - hosts_to_try.append("http://localhost:8087") - - last_error = None - - for host in hosts_to_try: - try: - logger.info(f"Attempting SearXNG search at {host}") - - async with httpx.AsyncClient(timeout=config.SEARXNG_TIMEOUT) as client: - response = await client.get( - f"{host}/search", - params={ - "q": query, - "format": "json", - "pageno": 1, - } - ) - - if response.status_code == 200: - data = response.json() - results = data.get("results", []) - - if not results: - return f"No results found for '{query}'" - - # Format results - formatted_results = [] - for i, result in enumerate(results[:num_results], 1): - title = result.get("title", "No title") - url = result.get("url", "") - content = result.get("content", "No description available") - - formatted_results.append( - f"{i}. {title}\n" - f" URL: {url}\n" - f" {content}\n" - ) - - return "\n".join(formatted_results) - else: - last_error = f"SearXNG returned status {response.status_code}" - - except httpx.ConnectError: - last_error = f"Cannot connect to SearXNG at {host}" - logger.warning(f"SearXNG connection failed at {host}, trying next host if available") - continue - except Exception as e: - last_error = str(e) - logger.warning(f"SearXNG error at {host}: {e}") - continue - - # All hosts failed - return f"Error searching: {last_error}. Please check that SearXNG is running." - - except Exception as e: - logger.error(f"Unexpected error in search_web: {e}", exc_info=True) - return f"Error searching: {str(e)}" diff --git a/tests/agents/librarian/test_tools.py b/tests/agents/librarian/test_tools.py new file mode 100644 index 0000000..cfaad1f --- /dev/null +++ b/tests/agents/librarian/test_tools.py @@ -0,0 +1,426 @@ +""" +Tests for Librarian tools. + +Tests the tool functions that wrap the Library-Desk API, +including the new web search and content extraction tools. +""" + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + +from src.agents.librarian.tools import ( + search_web, + read_url, + read_urls_batch, + hybrid_search, + search_wiki, +) +from src.agents.librarian.client import ( + WebSearchResult, + WebSearchResponse, + ContentExtractionResult, + BatchExtractionResponse, +) + + +@pytest.fixture +def mock_client(): + """Create a mock LibraryDeskClient.""" + client = AsyncMock() + return client + + +# ============================================================================ +# Web Search Tests +# ============================================================================ + +@pytest.mark.unit +class TestSearchWeb: + """Tests for search_web tool.""" + + @pytest.mark.asyncio + async def test_search_web_success(self, mock_client): + """Test successful web search.""" + mock_response = WebSearchResponse( + query="Python async programming", + search_type="web", + results=[ + WebSearchResult( + title="Async Python Tutorial", + url="https://example.com/async", + content="Full content about async programming...", + snippet="Learn async programming in Python", + source="example.com", + ), + WebSearchResult( + title="AsyncIO Documentation", + url="https://docs.python.org/asyncio", + content="Official asyncio docs content...", + snippet="Python asyncio library reference", + source="docs.python.org", + ), + ], + total_results=2, + search_time_ms=150, + sources_summary="**Sources:**\n- example.com\n- docs.python.org", + ) + mock_client.search_web.return_value = mock_response + + with patch( + "src.agents.librarian.tools.LibraryDeskClient" + ) as mock_client_class: + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client_class.return_value.__aexit__.return_value = None + + result = await search_web("Python async programming") + + assert "Python async programming" in result + assert "Async Python Tutorial" in result + assert "https://example.com/async" in result + assert "example.com" in result + assert "150ms" in result or "2 results" in result + + @pytest.mark.asyncio + async def test_search_web_no_results(self, mock_client): + """Test web search with no results.""" + mock_response = WebSearchResponse( + query="nonexistent query xyz123", + search_type="web", + results=[], + total_results=0, + search_time_ms=50, + ) + mock_client.search_web.return_value = mock_response + + with patch( + "src.agents.librarian.tools.LibraryDeskClient" + ) as mock_client_class: + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client_class.return_value.__aexit__.return_value = None + + result = await search_web("nonexistent query xyz123") + + assert "No results found" in result + + @pytest.mark.asyncio + async def test_search_web_error_handling(self, mock_client): + """Test web search error handling.""" + mock_client.search_web.side_effect = Exception("Connection failed") + + with patch( + "src.agents.librarian.tools.LibraryDeskClient" + ) as mock_client_class: + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client_class.return_value.__aexit__.return_value = None + + result = await search_web("test query") + + assert "Error" in result + assert "Connection failed" in result + + @pytest.mark.asyncio + async def test_search_web_with_news_type(self, mock_client): + """Test web search with news search type.""" + mock_response = WebSearchResponse( + query="latest tech news", + search_type="news", + results=[ + WebSearchResult( + title="Tech News Today", + url="https://news.example.com/tech", + snippet="Breaking tech news", + source="news.example.com", + published_date="2024-01-15", + ), + ], + total_results=1, + search_time_ms=100, + ) + mock_client.search_web.return_value = mock_response + + with patch( + "src.agents.librarian.tools.LibraryDeskClient" + ) as mock_client_class: + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client_class.return_value.__aexit__.return_value = None + + result = await search_web("latest tech news", search_type="news") + + assert "Tech News Today" in result + mock_client.search_web.assert_called_with( + query="latest tech news", + limit=10, + search_type="news", + ) + + +# ============================================================================ +# Read URL Tests +# ============================================================================ + +@pytest.mark.unit +class TestReadUrl: + """Tests for read_url tool.""" + + @pytest.mark.asyncio + async def test_read_url_success(self, mock_client): + """Test successful URL content extraction.""" + mock_result = ContentExtractionResult( + url="https://example.com/article", + title="Great Article Title", + content="This is the full article content extracted from the page.", + author="John Doe", + date="2024-01-10", + language="en", + success=True, + ) + mock_client.extract_content.return_value = mock_result + + with patch( + "src.agents.librarian.tools.LibraryDeskClient" + ) as mock_client_class: + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client_class.return_value.__aexit__.return_value = None + + result = await read_url("https://example.com/article") + + assert "Great Article Title" in result + assert "https://example.com/article" in result + assert "John Doe" in result + assert "full article content" in result + + @pytest.mark.asyncio + async def test_read_url_failure(self, mock_client): + """Test URL extraction failure.""" + mock_result = ContentExtractionResult( + url="https://example.com/blocked", + success=False, + error="403 Forbidden", + ) + mock_client.extract_content.return_value = mock_result + + with patch( + "src.agents.librarian.tools.LibraryDeskClient" + ) as mock_client_class: + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client_class.return_value.__aexit__.return_value = None + + result = await read_url("https://example.com/blocked") + + assert "Could not read page" in result + assert "403 Forbidden" in result + + @pytest.mark.asyncio + async def test_read_url_with_max_length(self, mock_client): + """Test URL extraction with custom max length.""" + mock_result = ContentExtractionResult( + url="https://example.com/long", + title="Long Article", + content="X" * 10000, + success=True, + ) + mock_client.extract_content.return_value = mock_result + + with patch( + "src.agents.librarian.tools.LibraryDeskClient" + ) as mock_client_class: + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client_class.return_value.__aexit__.return_value = None + + result = await read_url("https://example.com/long", max_length=2000) + + mock_client.extract_content.assert_called_with( + url="https://example.com/long", + include_metadata=True, + max_length=2000, + ) + + +# ============================================================================ +# Batch URL Tests +# ============================================================================ + +@pytest.mark.unit +class TestReadUrlsBatch: + """Tests for read_urls_batch tool.""" + + @pytest.mark.asyncio + async def test_batch_success(self, mock_client): + """Test successful batch extraction.""" + mock_response = BatchExtractionResponse( + results=[ + ContentExtractionResult( + url="https://example.com/1", + title="Article 1", + content="Content from article 1", + success=True, + ), + ContentExtractionResult( + url="https://example.com/2", + title="Article 2", + content="Content from article 2", + success=True, + ), + ], + total_urls=2, + successful=2, + failed=0, + extraction_time_ms=300, + ) + mock_client.extract_content_batch.return_value = mock_response + + with patch( + "src.agents.librarian.tools.LibraryDeskClient" + ) as mock_client_class: + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client_class.return_value.__aexit__.return_value = None + + result = await read_urls_batch([ + "https://example.com/1", + "https://example.com/2", + ]) + + assert "Article 1" in result + assert "Article 2" in result + assert "2/2" in result or "Extracted 2" in result + + @pytest.mark.asyncio + async def test_batch_partial_failure(self, mock_client): + """Test batch extraction with some failures.""" + mock_response = BatchExtractionResponse( + results=[ + ContentExtractionResult( + url="https://example.com/good", + title="Good Article", + content="Content extracted successfully", + success=True, + ), + ContentExtractionResult( + url="https://example.com/bad", + success=False, + error="Connection timeout", + ), + ], + total_urls=2, + successful=1, + failed=1, + extraction_time_ms=500, + ) + mock_client.extract_content_batch.return_value = mock_response + + with patch( + "src.agents.librarian.tools.LibraryDeskClient" + ) as mock_client_class: + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client_class.return_value.__aexit__.return_value = None + + result = await read_urls_batch([ + "https://example.com/good", + "https://example.com/bad", + ]) + + # Should contain successful result + assert "Good Article" in result + # Should report failure + assert "Failed" in result + assert "Connection timeout" in result + + +# ============================================================================ +# Response Model Tests +# ============================================================================ + +@pytest.mark.unit +class TestWebSearchModels: + """Tests for web search response models.""" + + def test_web_search_result_model(self): + """Test WebSearchResult model.""" + result = WebSearchResult( + title="Test Title", + url="https://example.com", + content="Full content here", + snippet="Short snippet", + source="example.com", + published_date="2024-01-15", + ) + + assert result.title == "Test Title" + assert result.url == "https://example.com" + assert result.content == "Full content here" + assert result.source == "example.com" + + def test_web_search_result_defaults(self): + """Test WebSearchResult default values.""" + result = WebSearchResult( + title="Title", + url="https://example.com", + ) + + assert result.content == "" + assert result.snippet == "" + assert result.source == "" + assert result.published_date is None + + def test_web_search_response_model(self): + """Test WebSearchResponse model.""" + response = WebSearchResponse( + query="test query", + search_type="web", + results=[ + WebSearchResult(title="R1", url="https://example.com/1"), + WebSearchResult(title="R2", url="https://example.com/2"), + ], + total_results=2, + search_time_ms=100, + sources_summary="**Sources:** example.com", + ) + + assert response.query == "test query" + assert len(response.results) == 2 + assert response.total_results == 2 + + def test_content_extraction_result_model(self): + """Test ContentExtractionResult model.""" + result = ContentExtractionResult( + url="https://example.com", + title="Title", + content="Content", + author="Author", + date="2024-01-01", + language="en", + success=True, + ) + + assert result.url == "https://example.com" + assert result.success is True + assert result.author == "Author" + + def test_content_extraction_failure(self): + """Test ContentExtractionResult for failed extraction.""" + result = ContentExtractionResult( + url="https://example.com", + success=False, + error="404 Not Found", + ) + + assert result.success is False + assert result.error == "404 Not Found" + assert result.content == "" + + def test_batch_extraction_response_model(self): + """Test BatchExtractionResponse model.""" + response = BatchExtractionResponse( + results=[ + ContentExtractionResult(url="https://1.com", success=True), + ContentExtractionResult(url="https://2.com", success=False), + ], + total_urls=2, + successful=1, + failed=1, + extraction_time_ms=500, + ) + + assert response.total_urls == 2 + assert response.successful == 1 + assert response.failed == 1 diff --git a/tests/agents/test_tools.py b/tests/agents/test_tools.py index ae38500..dcba341 100644 --- a/tests/agents/test_tools.py +++ b/tests/agents/test_tools.py @@ -1,17 +1,18 @@ """ -Tests for Tatlock's permanent tools (calculator, date/time, search). +Tests for Tatlock's permanent tools (calculator, date/time). + +Note: Web search has been moved to The Librarian agent. +See tests/agents/librarian/test_tools.py for search tests. """ import pytest from datetime import datetime -from unittest.mock import AsyncMock, patch from src.agents.tools import ( calculate, get_current_datetime, calculate_time_offset, time_difference, - search_web, ) @@ -188,184 +189,3 @@ class TestDateTime: """Test error handling for invalid dates.""" result = time_difference("invalid-date", "now") assert "Error" in result - - -# ============================================================================ -# Search Tests -# ============================================================================ - -class TestSearch: - """Tests for web search tool.""" - - @pytest.mark.asyncio - async def test_search_web_success(self): - """Test successful web search.""" - mock_response = { - "results": [ - { - "title": "Test Result 1", - "url": "https://example.com/1", - "content": "This is a test result" - }, - { - "title": "Test Result 2", - "url": "https://example.com/2", - "content": "Another test result" - } - ] - } - - with patch("src.agents.tools.httpx.AsyncClient") as mock_client_class: - # Create mock response - mock_response_obj = type('MockResponse', (), { - 'status_code': 200, - 'json': lambda *args, **kwargs: mock_response - })() - - # Create mock client with async get method - async def mock_get(*args, **kwargs): - return mock_response_obj - - mock_client_instance = type('MockClient', (), { - 'get': mock_get - })() - - # Setup async context manager - async def mock_aenter(*args, **kwargs): - return mock_client_instance - - async def mock_aexit(*args, **kwargs): - return None - - mock_client_class.return_value.__aenter__ = mock_aenter - mock_client_class.return_value.__aexit__ = mock_aexit - - result = await search_web("test query", num_results=2) - - assert "Test Result 1" in result - assert "https://example.com/1" in result - assert "Test Result 2" in result - assert "https://example.com/2" in result - - @pytest.mark.asyncio - async def test_search_web_no_results(self): - """Test web search with no results.""" - mock_response_data = {"results": []} - - with patch("src.agents.tools.httpx.AsyncClient") as mock_client_class: - mock_response_obj = type('MockResponse', (), { - 'status_code': 200, - 'json': lambda *args, **kwargs: mock_response_data - })() - - async def mock_get(*args, **kwargs): - return mock_response_obj - - mock_client_instance = type('MockClient', (), { - 'get': mock_get - })() - - async def mock_aenter(*args, **kwargs): - return mock_client_instance - - async def mock_aexit(*args, **kwargs): - return None - - mock_client_class.return_value.__aenter__ = mock_aenter - mock_client_class.return_value.__aexit__ = mock_aexit - - result = await search_web("test query") - - assert "No results found" in result - - @pytest.mark.asyncio - async def test_search_web_connection_error(self): - """Test web search with connection error.""" - with patch("httpx.AsyncClient") as mock_client: - mock_client_instance = AsyncMock() - mock_client_instance.get.side_effect = Exception("Connection failed") - mock_client.return_value.__aenter__.return_value = mock_client_instance - - result = await search_web("test query") - - assert "Error searching" in result - - @pytest.mark.asyncio - async def test_search_web_limits_results(self): - """Test that search limits results to max 10.""" - mock_response_data = { - "results": [ - {"title": f"Result {i}", "url": f"https://example.com/{i}", "content": "Test"} - for i in range(20) - ] - } - - with patch("src.agents.tools.httpx.AsyncClient") as mock_client_class: - mock_response_obj = type('MockResponse', (), { - 'status_code': 200, - 'json': lambda *args, **kwargs: mock_response_data - })() - - async def mock_get(*args, **kwargs): - return mock_response_obj - - mock_client_instance = type('MockClient', (), { - 'get': mock_get - })() - - async def mock_aenter(*args, **kwargs): - return mock_client_instance - - async def mock_aexit(*args, **kwargs): - return None - - mock_client_class.return_value.__aenter__ = mock_aenter - mock_client_class.return_value.__aexit__ = mock_aexit - - result = await search_web("test query", num_results=15) - - # Should only return 10 results (max limit) - result_count = result.count("URL:") - assert result_count == 10 - - @pytest.mark.asyncio - async def test_search_web_formats_results(self): - """Test that search results are properly formatted.""" - mock_response_data = { - "results": [ - { - "title": "Test Title", - "url": "https://example.com", - "content": "Test content description" - } - ] - } - - with patch("src.agents.tools.httpx.AsyncClient") as mock_client_class: - mock_response_obj = type('MockResponse', (), { - 'status_code': 200, - 'json': lambda *args, **kwargs: mock_response_data - })() - - async def mock_get(*args, **kwargs): - return mock_response_obj - - mock_client_instance = type('MockClient', (), { - 'get': mock_get - })() - - async def mock_aenter(*args, **kwargs): - return mock_client_instance - - async def mock_aexit(*args, **kwargs): - return None - - mock_client_class.return_value.__aenter__ = mock_aenter - mock_client_class.return_value.__aexit__ = mock_aexit - - result = await search_web("test query") - - # Check formatting - assert "1. Test Title" in result - assert "URL: https://example.com" in result - assert "Test content description" in result