feat: migrate web search from tatlock_core to Librarian
Move web search functionality to The Librarian agent, integrating with the library-desk /rag/search endpoint for enhanced search capabilities. Changes: - Add search_web, read_url, read_urls_batch tools to Librarian - Add WebSearchResult, ContentExtractionResult models to client - Add search_web, extract_content, extract_content_batch client methods - Update Librarian capability with web/url/internet domains - Remove search_web from tatlock_core tools and toolset - Update Tatlock system prompt to delegate web search to Librarian - Add comprehensive unit tests for new Librarian tools - Clean up legacy src/agents/tools.py 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.5
@@ -432,6 +432,239 @@ async def find_related_entities(
|
||||
return f"Error finding related entities: {str(e)}"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Web Search & Content Extraction
|
||||
# ============================================================================
|
||||
|
||||
async def search_web(
|
||||
query: str,
|
||||
limit: int = 10,
|
||||
search_type: str = "web",
|
||||
) -> str:
|
||||
"""
|
||||
Search the web and extract content from results.
|
||||
|
||||
This is the primary tool for finding current information online.
|
||||
Results include both snippets and full extracted text from pages.
|
||||
|
||||
Search types:
|
||||
- "web": General web search (default)
|
||||
- "news": News articles
|
||||
- "images": Image search
|
||||
|
||||
Args:
|
||||
query: Search query (1-500 chars)
|
||||
limit: Number of results (1-20, default: 10)
|
||||
search_type: Type of search ("web", "news", or "images")
|
||||
|
||||
Returns:
|
||||
Formatted search results with sources and extracted content
|
||||
|
||||
Examples:
|
||||
search_web("Python 3.12 new features")
|
||||
search_web("latest tech news", search_type="news", limit=5)
|
||||
"""
|
||||
try:
|
||||
async with LibraryDeskClient() as client:
|
||||
response = await client.search_web(
|
||||
query=query,
|
||||
limit=limit,
|
||||
search_type=search_type,
|
||||
)
|
||||
|
||||
if not response.results:
|
||||
return f"No results found for '{query}'"
|
||||
|
||||
output_parts = [f"## Web Search: {query}\n"]
|
||||
output_parts.append(f"*Found {response.total_results} results in {response.search_time_ms}ms*\n")
|
||||
|
||||
for i, result in enumerate(response.results, 1):
|
||||
output_parts.append(f"### {i}. {result.title}")
|
||||
output_parts.append(f"**Source:** {result.source}")
|
||||
output_parts.append(f"**URL:** {result.url}")
|
||||
|
||||
if result.published_date:
|
||||
output_parts.append(f"**Date:** {result.published_date}")
|
||||
|
||||
# Use full content if available, otherwise snippet
|
||||
content = result.content or result.snippet
|
||||
if content:
|
||||
# Truncate for readability
|
||||
if len(content) > 500:
|
||||
content = content[:500] + "..."
|
||||
output_parts.append(f"\n{content}")
|
||||
|
||||
output_parts.append("")
|
||||
|
||||
# Add pre-formatted sources for citations
|
||||
if response.sources_summary:
|
||||
output_parts.append("---")
|
||||
output_parts.append(response.sources_summary)
|
||||
|
||||
logger.info(
|
||||
"librarian_web_search",
|
||||
query=query,
|
||||
result_count=response.total_results,
|
||||
search_type=search_type,
|
||||
)
|
||||
|
||||
return "\n".join(output_parts)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("librarian_web_search_error", error=str(e), query=query)
|
||||
return f"Error searching web: {str(e)}"
|
||||
|
||||
|
||||
async def read_url(
|
||||
url: str,
|
||||
max_length: int = 5000,
|
||||
) -> str:
|
||||
"""
|
||||
Read and extract the main content from a URL.
|
||||
|
||||
Use this when you have a specific URL to read, such as:
|
||||
- A link the user provided
|
||||
- A URL from search results you want to read in full
|
||||
- Documentation or article pages
|
||||
|
||||
Extracts the main content, removing ads, navigation, and boilerplate.
|
||||
|
||||
Args:
|
||||
url: The URL to read
|
||||
max_length: Maximum content length (default: 5000)
|
||||
|
||||
Returns:
|
||||
Extracted page content with metadata
|
||||
|
||||
Examples:
|
||||
read_url("https://docs.python.org/3/library/asyncio.html")
|
||||
read_url("https://example.com/article", max_length=10000)
|
||||
"""
|
||||
try:
|
||||
async with LibraryDeskClient() as client:
|
||||
result = await client.extract_content(
|
||||
url=url,
|
||||
include_metadata=True,
|
||||
max_length=max_length,
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
return f"Could not read page: {result.error or 'Unknown error'}"
|
||||
|
||||
output_parts = []
|
||||
|
||||
# Header with metadata
|
||||
if result.title:
|
||||
output_parts.append(f"# {result.title}")
|
||||
else:
|
||||
output_parts.append(f"# Content from {url}")
|
||||
|
||||
output_parts.append(f"**URL:** {url}")
|
||||
|
||||
if result.author:
|
||||
output_parts.append(f"**Author:** {result.author}")
|
||||
|
||||
if result.date:
|
||||
output_parts.append(f"**Date:** {result.date}")
|
||||
|
||||
if result.language and result.language != "en":
|
||||
output_parts.append(f"**Language:** {result.language}")
|
||||
|
||||
output_parts.append("")
|
||||
|
||||
# Main content
|
||||
if result.content:
|
||||
output_parts.append(result.content)
|
||||
else:
|
||||
output_parts.append("(No content could be extracted)")
|
||||
|
||||
logger.info(
|
||||
"librarian_read_url",
|
||||
url=url,
|
||||
content_length=len(result.content) if result.content else 0,
|
||||
)
|
||||
|
||||
return "\n".join(output_parts)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("librarian_read_url_error", error=str(e), url=url)
|
||||
return f"Error reading URL: {str(e)}"
|
||||
|
||||
|
||||
async def read_urls_batch(
|
||||
urls: list[str],
|
||||
max_length: int = 2000,
|
||||
) -> str:
|
||||
"""
|
||||
Read and extract content from multiple URLs in parallel.
|
||||
|
||||
More efficient than calling read_url multiple times.
|
||||
Max 20 URLs per batch.
|
||||
|
||||
Note: Individual failures don't fail the entire batch -
|
||||
failed URLs are reported but other content is still returned.
|
||||
|
||||
Args:
|
||||
urls: List of URLs to read (max 20)
|
||||
max_length: Maximum content length per URL (default: 2000)
|
||||
|
||||
Returns:
|
||||
Extracted content from all successful URLs with failure report
|
||||
|
||||
Examples:
|
||||
read_urls_batch(["https://example.com/1", "https://example.com/2"])
|
||||
"""
|
||||
try:
|
||||
async with LibraryDeskClient() as client:
|
||||
response = await client.extract_content_batch(
|
||||
urls=urls,
|
||||
include_metadata=True,
|
||||
max_length=max_length,
|
||||
)
|
||||
|
||||
output_parts = [
|
||||
f"## Batch Content Extraction",
|
||||
f"*Extracted {response.successful}/{response.total_urls} URLs in {response.extraction_time_ms}ms*\n",
|
||||
]
|
||||
|
||||
# Show successful extractions
|
||||
for result in response.results:
|
||||
if result.success:
|
||||
title = result.title or result.url
|
||||
output_parts.append(f"### {title}")
|
||||
output_parts.append(f"**URL:** {result.url}")
|
||||
|
||||
if result.content:
|
||||
# Truncate for readability in batch mode
|
||||
content = result.content
|
||||
if len(content) > max_length:
|
||||
content = content[:max_length] + "..."
|
||||
output_parts.append(f"\n{content}")
|
||||
|
||||
output_parts.append("")
|
||||
|
||||
# Report failures
|
||||
failed = [r for r in response.results if not r.success]
|
||||
if failed:
|
||||
output_parts.append("---")
|
||||
output_parts.append("### Failed Extractions")
|
||||
for result in failed:
|
||||
output_parts.append(f"- {result.url}: {result.error}")
|
||||
|
||||
logger.info(
|
||||
"librarian_read_urls_batch",
|
||||
total=response.total_urls,
|
||||
successful=response.successful,
|
||||
failed=response.failed,
|
||||
)
|
||||
|
||||
return "\n".join(output_parts)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("librarian_read_urls_batch_error", error=str(e))
|
||||
return f"Error reading URLs: {str(e)}"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Wiki Write Operations
|
||||
# ============================================================================
|
||||
@@ -685,7 +918,7 @@ async def smart_create_wiki_page(
|
||||
|
||||
# All tools available to The Librarian
|
||||
LIBRARIAN_TOOLS = [
|
||||
# Research tools
|
||||
# Research tools (internal knowledge)
|
||||
hybrid_search,
|
||||
search_wiki,
|
||||
get_wiki_page,
|
||||
@@ -694,6 +927,10 @@ LIBRARIAN_TOOLS = [
|
||||
semantic_search,
|
||||
explore_knowledge_graph,
|
||||
find_related_entities,
|
||||
# Web search & content extraction
|
||||
search_web,
|
||||
read_url,
|
||||
read_urls_batch,
|
||||
# Write tools
|
||||
create_wiki_page,
|
||||
update_wiki_page,
|
||||
|
||||
Reference in New Issue
Block a user