feat: migrate web search from tatlock_core to Librarian
Move web search functionality to The Librarian agent, integrating with the library-desk /rag/search endpoint for enhanced search capabilities. Changes: - Add search_web, read_url, read_urls_batch tools to Librarian - Add WebSearchResult, ContentExtractionResult models to client - Add search_web, extract_content, extract_content_batch client methods - Update Librarian capability with web/url/internet domains - Remove search_web from tatlock_core tools and toolset - Update Tatlock system prompt to delegate web search to Librarian - Add comprehensive unit tests for new Librarian tools - Clean up legacy src/agents/tools.py 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.5
@@ -0,0 +1,426 @@
|
||||
"""
|
||||
Tests for Librarian tools.
|
||||
|
||||
Tests the tool functions that wrap the Library-Desk API,
|
||||
including the new web search and content extraction tools.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from src.agents.librarian.tools import (
|
||||
search_web,
|
||||
read_url,
|
||||
read_urls_batch,
|
||||
hybrid_search,
|
||||
search_wiki,
|
||||
)
|
||||
from src.agents.librarian.client import (
|
||||
WebSearchResult,
|
||||
WebSearchResponse,
|
||||
ContentExtractionResult,
|
||||
BatchExtractionResponse,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_client():
|
||||
"""Create a mock LibraryDeskClient."""
|
||||
client = AsyncMock()
|
||||
return client
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Web Search Tests
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSearchWeb:
|
||||
"""Tests for search_web tool."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_web_success(self, mock_client):
|
||||
"""Test successful web search."""
|
||||
mock_response = WebSearchResponse(
|
||||
query="Python async programming",
|
||||
search_type="web",
|
||||
results=[
|
||||
WebSearchResult(
|
||||
title="Async Python Tutorial",
|
||||
url="https://example.com/async",
|
||||
content="Full content about async programming...",
|
||||
snippet="Learn async programming in Python",
|
||||
source="example.com",
|
||||
),
|
||||
WebSearchResult(
|
||||
title="AsyncIO Documentation",
|
||||
url="https://docs.python.org/asyncio",
|
||||
content="Official asyncio docs content...",
|
||||
snippet="Python asyncio library reference",
|
||||
source="docs.python.org",
|
||||
),
|
||||
],
|
||||
total_results=2,
|
||||
search_time_ms=150,
|
||||
sources_summary="**Sources:**\n- example.com\n- docs.python.org",
|
||||
)
|
||||
mock_client.search_web.return_value = mock_response
|
||||
|
||||
with patch(
|
||||
"src.agents.librarian.tools.LibraryDeskClient"
|
||||
) as mock_client_class:
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
mock_client_class.return_value.__aexit__.return_value = None
|
||||
|
||||
result = await search_web("Python async programming")
|
||||
|
||||
assert "Python async programming" in result
|
||||
assert "Async Python Tutorial" in result
|
||||
assert "https://example.com/async" in result
|
||||
assert "example.com" in result
|
||||
assert "150ms" in result or "2 results" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_web_no_results(self, mock_client):
|
||||
"""Test web search with no results."""
|
||||
mock_response = WebSearchResponse(
|
||||
query="nonexistent query xyz123",
|
||||
search_type="web",
|
||||
results=[],
|
||||
total_results=0,
|
||||
search_time_ms=50,
|
||||
)
|
||||
mock_client.search_web.return_value = mock_response
|
||||
|
||||
with patch(
|
||||
"src.agents.librarian.tools.LibraryDeskClient"
|
||||
) as mock_client_class:
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
mock_client_class.return_value.__aexit__.return_value = None
|
||||
|
||||
result = await search_web("nonexistent query xyz123")
|
||||
|
||||
assert "No results found" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_web_error_handling(self, mock_client):
|
||||
"""Test web search error handling."""
|
||||
mock_client.search_web.side_effect = Exception("Connection failed")
|
||||
|
||||
with patch(
|
||||
"src.agents.librarian.tools.LibraryDeskClient"
|
||||
) as mock_client_class:
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
mock_client_class.return_value.__aexit__.return_value = None
|
||||
|
||||
result = await search_web("test query")
|
||||
|
||||
assert "Error" in result
|
||||
assert "Connection failed" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_web_with_news_type(self, mock_client):
|
||||
"""Test web search with news search type."""
|
||||
mock_response = WebSearchResponse(
|
||||
query="latest tech news",
|
||||
search_type="news",
|
||||
results=[
|
||||
WebSearchResult(
|
||||
title="Tech News Today",
|
||||
url="https://news.example.com/tech",
|
||||
snippet="Breaking tech news",
|
||||
source="news.example.com",
|
||||
published_date="2024-01-15",
|
||||
),
|
||||
],
|
||||
total_results=1,
|
||||
search_time_ms=100,
|
||||
)
|
||||
mock_client.search_web.return_value = mock_response
|
||||
|
||||
with patch(
|
||||
"src.agents.librarian.tools.LibraryDeskClient"
|
||||
) as mock_client_class:
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
mock_client_class.return_value.__aexit__.return_value = None
|
||||
|
||||
result = await search_web("latest tech news", search_type="news")
|
||||
|
||||
assert "Tech News Today" in result
|
||||
mock_client.search_web.assert_called_with(
|
||||
query="latest tech news",
|
||||
limit=10,
|
||||
search_type="news",
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Read URL Tests
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestReadUrl:
|
||||
"""Tests for read_url tool."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_url_success(self, mock_client):
|
||||
"""Test successful URL content extraction."""
|
||||
mock_result = ContentExtractionResult(
|
||||
url="https://example.com/article",
|
||||
title="Great Article Title",
|
||||
content="This is the full article content extracted from the page.",
|
||||
author="John Doe",
|
||||
date="2024-01-10",
|
||||
language="en",
|
||||
success=True,
|
||||
)
|
||||
mock_client.extract_content.return_value = mock_result
|
||||
|
||||
with patch(
|
||||
"src.agents.librarian.tools.LibraryDeskClient"
|
||||
) as mock_client_class:
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
mock_client_class.return_value.__aexit__.return_value = None
|
||||
|
||||
result = await read_url("https://example.com/article")
|
||||
|
||||
assert "Great Article Title" in result
|
||||
assert "https://example.com/article" in result
|
||||
assert "John Doe" in result
|
||||
assert "full article content" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_url_failure(self, mock_client):
|
||||
"""Test URL extraction failure."""
|
||||
mock_result = ContentExtractionResult(
|
||||
url="https://example.com/blocked",
|
||||
success=False,
|
||||
error="403 Forbidden",
|
||||
)
|
||||
mock_client.extract_content.return_value = mock_result
|
||||
|
||||
with patch(
|
||||
"src.agents.librarian.tools.LibraryDeskClient"
|
||||
) as mock_client_class:
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
mock_client_class.return_value.__aexit__.return_value = None
|
||||
|
||||
result = await read_url("https://example.com/blocked")
|
||||
|
||||
assert "Could not read page" in result
|
||||
assert "403 Forbidden" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_url_with_max_length(self, mock_client):
|
||||
"""Test URL extraction with custom max length."""
|
||||
mock_result = ContentExtractionResult(
|
||||
url="https://example.com/long",
|
||||
title="Long Article",
|
||||
content="X" * 10000,
|
||||
success=True,
|
||||
)
|
||||
mock_client.extract_content.return_value = mock_result
|
||||
|
||||
with patch(
|
||||
"src.agents.librarian.tools.LibraryDeskClient"
|
||||
) as mock_client_class:
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
mock_client_class.return_value.__aexit__.return_value = None
|
||||
|
||||
result = await read_url("https://example.com/long", max_length=2000)
|
||||
|
||||
mock_client.extract_content.assert_called_with(
|
||||
url="https://example.com/long",
|
||||
include_metadata=True,
|
||||
max_length=2000,
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Batch URL Tests
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestReadUrlsBatch:
|
||||
"""Tests for read_urls_batch tool."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_success(self, mock_client):
|
||||
"""Test successful batch extraction."""
|
||||
mock_response = BatchExtractionResponse(
|
||||
results=[
|
||||
ContentExtractionResult(
|
||||
url="https://example.com/1",
|
||||
title="Article 1",
|
||||
content="Content from article 1",
|
||||
success=True,
|
||||
),
|
||||
ContentExtractionResult(
|
||||
url="https://example.com/2",
|
||||
title="Article 2",
|
||||
content="Content from article 2",
|
||||
success=True,
|
||||
),
|
||||
],
|
||||
total_urls=2,
|
||||
successful=2,
|
||||
failed=0,
|
||||
extraction_time_ms=300,
|
||||
)
|
||||
mock_client.extract_content_batch.return_value = mock_response
|
||||
|
||||
with patch(
|
||||
"src.agents.librarian.tools.LibraryDeskClient"
|
||||
) as mock_client_class:
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
mock_client_class.return_value.__aexit__.return_value = None
|
||||
|
||||
result = await read_urls_batch([
|
||||
"https://example.com/1",
|
||||
"https://example.com/2",
|
||||
])
|
||||
|
||||
assert "Article 1" in result
|
||||
assert "Article 2" in result
|
||||
assert "2/2" in result or "Extracted 2" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_partial_failure(self, mock_client):
|
||||
"""Test batch extraction with some failures."""
|
||||
mock_response = BatchExtractionResponse(
|
||||
results=[
|
||||
ContentExtractionResult(
|
||||
url="https://example.com/good",
|
||||
title="Good Article",
|
||||
content="Content extracted successfully",
|
||||
success=True,
|
||||
),
|
||||
ContentExtractionResult(
|
||||
url="https://example.com/bad",
|
||||
success=False,
|
||||
error="Connection timeout",
|
||||
),
|
||||
],
|
||||
total_urls=2,
|
||||
successful=1,
|
||||
failed=1,
|
||||
extraction_time_ms=500,
|
||||
)
|
||||
mock_client.extract_content_batch.return_value = mock_response
|
||||
|
||||
with patch(
|
||||
"src.agents.librarian.tools.LibraryDeskClient"
|
||||
) as mock_client_class:
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
mock_client_class.return_value.__aexit__.return_value = None
|
||||
|
||||
result = await read_urls_batch([
|
||||
"https://example.com/good",
|
||||
"https://example.com/bad",
|
||||
])
|
||||
|
||||
# Should contain successful result
|
||||
assert "Good Article" in result
|
||||
# Should report failure
|
||||
assert "Failed" in result
|
||||
assert "Connection timeout" in result
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Response Model Tests
|
||||
# ============================================================================
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestWebSearchModels:
|
||||
"""Tests for web search response models."""
|
||||
|
||||
def test_web_search_result_model(self):
|
||||
"""Test WebSearchResult model."""
|
||||
result = WebSearchResult(
|
||||
title="Test Title",
|
||||
url="https://example.com",
|
||||
content="Full content here",
|
||||
snippet="Short snippet",
|
||||
source="example.com",
|
||||
published_date="2024-01-15",
|
||||
)
|
||||
|
||||
assert result.title == "Test Title"
|
||||
assert result.url == "https://example.com"
|
||||
assert result.content == "Full content here"
|
||||
assert result.source == "example.com"
|
||||
|
||||
def test_web_search_result_defaults(self):
|
||||
"""Test WebSearchResult default values."""
|
||||
result = WebSearchResult(
|
||||
title="Title",
|
||||
url="https://example.com",
|
||||
)
|
||||
|
||||
assert result.content == ""
|
||||
assert result.snippet == ""
|
||||
assert result.source == ""
|
||||
assert result.published_date is None
|
||||
|
||||
def test_web_search_response_model(self):
|
||||
"""Test WebSearchResponse model."""
|
||||
response = WebSearchResponse(
|
||||
query="test query",
|
||||
search_type="web",
|
||||
results=[
|
||||
WebSearchResult(title="R1", url="https://example.com/1"),
|
||||
WebSearchResult(title="R2", url="https://example.com/2"),
|
||||
],
|
||||
total_results=2,
|
||||
search_time_ms=100,
|
||||
sources_summary="**Sources:** example.com",
|
||||
)
|
||||
|
||||
assert response.query == "test query"
|
||||
assert len(response.results) == 2
|
||||
assert response.total_results == 2
|
||||
|
||||
def test_content_extraction_result_model(self):
|
||||
"""Test ContentExtractionResult model."""
|
||||
result = ContentExtractionResult(
|
||||
url="https://example.com",
|
||||
title="Title",
|
||||
content="Content",
|
||||
author="Author",
|
||||
date="2024-01-01",
|
||||
language="en",
|
||||
success=True,
|
||||
)
|
||||
|
||||
assert result.url == "https://example.com"
|
||||
assert result.success is True
|
||||
assert result.author == "Author"
|
||||
|
||||
def test_content_extraction_failure(self):
|
||||
"""Test ContentExtractionResult for failed extraction."""
|
||||
result = ContentExtractionResult(
|
||||
url="https://example.com",
|
||||
success=False,
|
||||
error="404 Not Found",
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert result.error == "404 Not Found"
|
||||
assert result.content == ""
|
||||
|
||||
def test_batch_extraction_response_model(self):
|
||||
"""Test BatchExtractionResponse model."""
|
||||
response = BatchExtractionResponse(
|
||||
results=[
|
||||
ContentExtractionResult(url="https://1.com", success=True),
|
||||
ContentExtractionResult(url="https://2.com", success=False),
|
||||
],
|
||||
total_urls=2,
|
||||
successful=1,
|
||||
failed=1,
|
||||
extraction_time_ms=500,
|
||||
)
|
||||
|
||||
assert response.total_urls == 2
|
||||
assert response.successful == 1
|
||||
assert response.failed == 1
|
||||
+4
-184
@@ -1,17 +1,18 @@
|
||||
"""
|
||||
Tests for Tatlock's permanent tools (calculator, date/time, search).
|
||||
Tests for Tatlock's permanent tools (calculator, date/time).
|
||||
|
||||
Note: Web search has been moved to The Librarian agent.
|
||||
See tests/agents/librarian/test_tools.py for search tests.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from src.agents.tools import (
|
||||
calculate,
|
||||
get_current_datetime,
|
||||
calculate_time_offset,
|
||||
time_difference,
|
||||
search_web,
|
||||
)
|
||||
|
||||
|
||||
@@ -188,184 +189,3 @@ class TestDateTime:
|
||||
"""Test error handling for invalid dates."""
|
||||
result = time_difference("invalid-date", "now")
|
||||
assert "Error" in result
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Search Tests
|
||||
# ============================================================================
|
||||
|
||||
class TestSearch:
|
||||
"""Tests for web search tool."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_web_success(self):
|
||||
"""Test successful web search."""
|
||||
mock_response = {
|
||||
"results": [
|
||||
{
|
||||
"title": "Test Result 1",
|
||||
"url": "https://example.com/1",
|
||||
"content": "This is a test result"
|
||||
},
|
||||
{
|
||||
"title": "Test Result 2",
|
||||
"url": "https://example.com/2",
|
||||
"content": "Another test result"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
with patch("src.agents.tools.httpx.AsyncClient") as mock_client_class:
|
||||
# Create mock response
|
||||
mock_response_obj = type('MockResponse', (), {
|
||||
'status_code': 200,
|
||||
'json': lambda *args, **kwargs: mock_response
|
||||
})()
|
||||
|
||||
# Create mock client with async get method
|
||||
async def mock_get(*args, **kwargs):
|
||||
return mock_response_obj
|
||||
|
||||
mock_client_instance = type('MockClient', (), {
|
||||
'get': mock_get
|
||||
})()
|
||||
|
||||
# Setup async context manager
|
||||
async def mock_aenter(*args, **kwargs):
|
||||
return mock_client_instance
|
||||
|
||||
async def mock_aexit(*args, **kwargs):
|
||||
return None
|
||||
|
||||
mock_client_class.return_value.__aenter__ = mock_aenter
|
||||
mock_client_class.return_value.__aexit__ = mock_aexit
|
||||
|
||||
result = await search_web("test query", num_results=2)
|
||||
|
||||
assert "Test Result 1" in result
|
||||
assert "https://example.com/1" in result
|
||||
assert "Test Result 2" in result
|
||||
assert "https://example.com/2" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_web_no_results(self):
|
||||
"""Test web search with no results."""
|
||||
mock_response_data = {"results": []}
|
||||
|
||||
with patch("src.agents.tools.httpx.AsyncClient") as mock_client_class:
|
||||
mock_response_obj = type('MockResponse', (), {
|
||||
'status_code': 200,
|
||||
'json': lambda *args, **kwargs: mock_response_data
|
||||
})()
|
||||
|
||||
async def mock_get(*args, **kwargs):
|
||||
return mock_response_obj
|
||||
|
||||
mock_client_instance = type('MockClient', (), {
|
||||
'get': mock_get
|
||||
})()
|
||||
|
||||
async def mock_aenter(*args, **kwargs):
|
||||
return mock_client_instance
|
||||
|
||||
async def mock_aexit(*args, **kwargs):
|
||||
return None
|
||||
|
||||
mock_client_class.return_value.__aenter__ = mock_aenter
|
||||
mock_client_class.return_value.__aexit__ = mock_aexit
|
||||
|
||||
result = await search_web("test query")
|
||||
|
||||
assert "No results found" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_web_connection_error(self):
|
||||
"""Test web search with connection error."""
|
||||
with patch("httpx.AsyncClient") as mock_client:
|
||||
mock_client_instance = AsyncMock()
|
||||
mock_client_instance.get.side_effect = Exception("Connection failed")
|
||||
mock_client.return_value.__aenter__.return_value = mock_client_instance
|
||||
|
||||
result = await search_web("test query")
|
||||
|
||||
assert "Error searching" in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_web_limits_results(self):
|
||||
"""Test that search limits results to max 10."""
|
||||
mock_response_data = {
|
||||
"results": [
|
||||
{"title": f"Result {i}", "url": f"https://example.com/{i}", "content": "Test"}
|
||||
for i in range(20)
|
||||
]
|
||||
}
|
||||
|
||||
with patch("src.agents.tools.httpx.AsyncClient") as mock_client_class:
|
||||
mock_response_obj = type('MockResponse', (), {
|
||||
'status_code': 200,
|
||||
'json': lambda *args, **kwargs: mock_response_data
|
||||
})()
|
||||
|
||||
async def mock_get(*args, **kwargs):
|
||||
return mock_response_obj
|
||||
|
||||
mock_client_instance = type('MockClient', (), {
|
||||
'get': mock_get
|
||||
})()
|
||||
|
||||
async def mock_aenter(*args, **kwargs):
|
||||
return mock_client_instance
|
||||
|
||||
async def mock_aexit(*args, **kwargs):
|
||||
return None
|
||||
|
||||
mock_client_class.return_value.__aenter__ = mock_aenter
|
||||
mock_client_class.return_value.__aexit__ = mock_aexit
|
||||
|
||||
result = await search_web("test query", num_results=15)
|
||||
|
||||
# Should only return 10 results (max limit)
|
||||
result_count = result.count("URL:")
|
||||
assert result_count == 10
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_web_formats_results(self):
|
||||
"""Test that search results are properly formatted."""
|
||||
mock_response_data = {
|
||||
"results": [
|
||||
{
|
||||
"title": "Test Title",
|
||||
"url": "https://example.com",
|
||||
"content": "Test content description"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
with patch("src.agents.tools.httpx.AsyncClient") as mock_client_class:
|
||||
mock_response_obj = type('MockResponse', (), {
|
||||
'status_code': 200,
|
||||
'json': lambda *args, **kwargs: mock_response_data
|
||||
})()
|
||||
|
||||
async def mock_get(*args, **kwargs):
|
||||
return mock_response_obj
|
||||
|
||||
mock_client_instance = type('MockClient', (), {
|
||||
'get': mock_get
|
||||
})()
|
||||
|
||||
async def mock_aenter(*args, **kwargs):
|
||||
return mock_client_instance
|
||||
|
||||
async def mock_aexit(*args, **kwargs):
|
||||
return None
|
||||
|
||||
mock_client_class.return_value.__aenter__ = mock_aenter
|
||||
mock_client_class.return_value.__aexit__ = mock_aexit
|
||||
|
||||
result = await search_web("test query")
|
||||
|
||||
# Check formatting
|
||||
assert "1. Test Title" in result
|
||||
assert "URL: https://example.com" in result
|
||||
assert "Test content description" in result
|
||||
|
||||
Reference in New Issue
Block a user