jpmschweitzerandClaude Fable 5
bd1699115a
ContentExtractor._fetch used client.get(), buffering the whole body in memory before the MAX_RESPONSE_BYTES check truncated it - the cap protected Trafilatura but not memory/bandwidth (a multi-hundred-MB URL was still fully downloaded, on up to max_urls_per_batch concurrent fetches, bounded only by the read timeout). Fetches now stream via client.stream + aiter_bytes and close the connection as soon as the cap is reached; charset still comes from the Content-Type header, available before the body is read. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
362 lines
13 KiB
Python
362 lines
13 KiB
Python
"""Tests for ContentExtractor client."""
|
|
|
|
import pytest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import httpx
|
|
|
|
from src.clients.content_extractor import ContentExtractor
|
|
from src.models.content import ContentExtractionResult
|
|
|
|
TEST_HTML = "<html><body><article>Test</article></body></html>"
|
|
|
|
|
|
def _doc(text, **meta):
|
|
"""bare_extraction-style result dict."""
|
|
return {
|
|
"text": text,
|
|
"title": meta.get("title"),
|
|
"author": meta.get("author"),
|
|
"date": meta.get("date"),
|
|
"language": meta.get("language"),
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def content_extractor():
|
|
"""Create ContentExtractor with test configuration."""
|
|
return ContentExtractor(timeout=5, max_length=2000)
|
|
|
|
|
|
class TestContentExtractor:
|
|
"""Tests for ContentExtractor client."""
|
|
|
|
def test_init(self, content_extractor):
|
|
"""Test ContentExtractor initialization."""
|
|
assert content_extractor.timeout == 5
|
|
assert content_extractor.max_length == 2000
|
|
assert content_extractor.max_urls_per_batch == (
|
|
ContentExtractor.DEFAULT_MAX_URLS_PER_BATCH
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_success(self, content_extractor):
|
|
"""Test successful content extraction."""
|
|
test_url = "https://example.com/article"
|
|
test_content = "This is the extracted article content."
|
|
|
|
with patch.object(
|
|
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
|
|
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
|
|
mock_traf.bare_extraction.return_value = _doc(
|
|
test_content,
|
|
title="Test Article",
|
|
author="John Doe",
|
|
date="2024-01-15",
|
|
language="en",
|
|
)
|
|
|
|
result = await content_extractor.extract(test_url)
|
|
|
|
assert result.success is True
|
|
assert result.url == test_url
|
|
assert result.content == test_content
|
|
assert result.title == "Test Article"
|
|
assert result.error is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extraction_runs_once_per_url(self, content_extractor):
|
|
"""Trafilatura must parse the document exactly ONCE (the old code
|
|
ran extract() twice plus bare_extraction — three full parses)."""
|
|
with patch.object(
|
|
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
|
|
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
|
|
mock_traf.bare_extraction.return_value = _doc("content")
|
|
|
|
await content_extractor.extract("https://example.com/a")
|
|
|
|
assert mock_traf.bare_extraction.call_count == 1
|
|
mock_traf.extract.assert_not_called()
|
|
mock_traf.fetch_url.assert_not_called() # httpx fetches now
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_fetch_failure(self, content_extractor):
|
|
"""Test extraction when URL fetch fails."""
|
|
test_url = "https://example.com/nonexistent"
|
|
|
|
with patch.object(
|
|
content_extractor, "_fetch", AsyncMock(return_value=None)
|
|
):
|
|
result = await content_extractor.extract(test_url)
|
|
|
|
assert result.success is False
|
|
assert result.url == test_url
|
|
assert result.content == ""
|
|
assert "Failed to fetch URL" in result.error
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_fetch_timeout(self, content_extractor):
|
|
"""A slow server hits the httpx timeout instead of pinning a
|
|
worker thread on a blind download."""
|
|
test_url = "https://example.com/slow-server"
|
|
|
|
with patch.object(
|
|
content_extractor,
|
|
"_fetch",
|
|
AsyncMock(side_effect=httpx.ReadTimeout("read timeout")),
|
|
):
|
|
result = await content_extractor.extract(test_url)
|
|
|
|
assert result.success is False
|
|
assert "timed out" in result.error.lower()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_network_error(self, content_extractor):
|
|
"""Connection errors return a failed result, not an exception."""
|
|
with patch.object(
|
|
content_extractor,
|
|
"_fetch",
|
|
AsyncMock(side_effect=httpx.ConnectError("refused")),
|
|
):
|
|
result = await content_extractor.extract("https://example.com/down")
|
|
|
|
assert result.success is False
|
|
assert "Failed to fetch URL" in result.error
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_no_content(self, content_extractor):
|
|
"""Test extraction when page has no extractable content."""
|
|
test_url = "https://example.com/empty"
|
|
|
|
with patch.object(
|
|
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
|
|
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
|
|
mock_traf.bare_extraction.return_value = None
|
|
|
|
result = await content_extractor.extract(test_url)
|
|
|
|
assert result.success is False
|
|
assert "No content extracted" in result.error
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_max_length_truncation(self, content_extractor):
|
|
"""Test that content is truncated to max length."""
|
|
test_url = "https://example.com/long-article"
|
|
# Content longer than max_length (2000)
|
|
long_content = "x" * 3000
|
|
|
|
with patch.object(
|
|
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
|
|
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
|
|
mock_traf.bare_extraction.return_value = _doc(long_content)
|
|
|
|
result = await content_extractor.extract(test_url)
|
|
|
|
assert result.success is True
|
|
assert len(result.content) <= content_extractor.max_length + 3 # +3 for "..."
|
|
assert result.content.endswith("...")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_batch(self, content_extractor):
|
|
"""Test batch extraction of multiple URLs."""
|
|
test_urls = [
|
|
"https://example.com/article1",
|
|
"https://example.com/article2",
|
|
"https://example.com/article3",
|
|
]
|
|
|
|
with patch.object(
|
|
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
|
|
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
|
|
mock_traf.bare_extraction.return_value = _doc("Extracted content")
|
|
|
|
results = await content_extractor.extract_batch(test_urls)
|
|
|
|
assert len(results) == 3
|
|
for i, result in enumerate(results):
|
|
assert result.url == test_urls[i]
|
|
assert result.success is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_batch_caps_full_page_extractions(self):
|
|
"""URLs beyond the per-call cap are skipped (callers fall back to
|
|
the search snippet) instead of fanning out unbounded downloads."""
|
|
extractor = ContentExtractor(timeout=5, max_length=2000, max_urls_per_batch=2)
|
|
test_urls = [f"https://example.com/{i}" for i in range(5)]
|
|
|
|
with patch.object(
|
|
extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
|
|
) as mock_fetch, patch(
|
|
"src.clients.content_extractor.trafilatura"
|
|
) as mock_traf:
|
|
mock_traf.bare_extraction.return_value = _doc("content")
|
|
|
|
results = await extractor.extract_batch(test_urls)
|
|
|
|
assert len(results) == 5
|
|
assert mock_fetch.await_count == 2
|
|
assert [r.url for r in results] == test_urls
|
|
assert all(r.success for r in results[:2])
|
|
for skipped in results[2:]:
|
|
assert skipped.success is False
|
|
assert "cap" in skipped.error
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_parse_timeout(self):
|
|
"""Test extraction (parse) timeout handling."""
|
|
import time
|
|
|
|
test_url = "https://example.com/slow"
|
|
|
|
# Create an extractor with very short timeout
|
|
fast_extractor = ContentExtractor(timeout=0.001, max_length=2000)
|
|
|
|
def slow_parse(*args, **kwargs):
|
|
time.sleep(1) # Sleep synchronously (this runs in thread pool)
|
|
return _doc("late content")
|
|
|
|
with patch.object(
|
|
fast_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
|
|
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
|
|
mock_traf.bare_extraction = slow_parse
|
|
|
|
result = await fast_extractor.extract(test_url)
|
|
|
|
assert result.success is False
|
|
assert "timed out" in result.error.lower()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_from_html(self, content_extractor):
|
|
"""Test extraction from raw HTML."""
|
|
test_html = "<html><body><article>Article content here.</article></body></html>"
|
|
|
|
with patch("src.clients.content_extractor.trafilatura") as mock_traf:
|
|
mock_traf.bare_extraction.return_value = _doc(
|
|
"Article content here.", title="Test"
|
|
)
|
|
|
|
result = await content_extractor.extract_from_html(
|
|
test_html, url="https://example.com"
|
|
)
|
|
|
|
assert result.success is True
|
|
assert result.content == "Article content here."
|
|
|
|
|
|
class TestFetchStreamingCap:
|
|
"""The 5MB cap must abort the DOWNLOAD, not just truncate after it."""
|
|
|
|
def _extractor_with_transport(self, handler):
|
|
extractor = ContentExtractor(timeout=5, max_length=2000)
|
|
extractor._http = httpx.AsyncClient(
|
|
transport=httpx.MockTransport(handler)
|
|
)
|
|
return extractor
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_aborts_past_cap(self):
|
|
from src.clients.content_extractor import MAX_RESPONSE_BYTES
|
|
|
|
chunk = b"x" * (1024 * 1024) # 1MB per chunk
|
|
chunks_produced = []
|
|
|
|
async def body():
|
|
for i in range(100): # 100MB on offer
|
|
chunks_produced.append(i)
|
|
yield chunk
|
|
|
|
def handler(request):
|
|
return httpx.Response(
|
|
200,
|
|
content=body(),
|
|
headers={"Content-Type": "text/html; charset=utf-8"},
|
|
)
|
|
|
|
extractor = self._extractor_with_transport(handler)
|
|
try:
|
|
text = await extractor._fetch("https://example.com/huge")
|
|
finally:
|
|
await extractor.close()
|
|
|
|
assert text is not None
|
|
assert len(text.encode()) == MAX_RESPONSE_BYTES
|
|
# Streaming stopped at the cap instead of consuming all 100 chunks
|
|
assert len(chunks_produced) <= (MAX_RESPONSE_BYTES // len(chunk)) + 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_small_response_returned_whole(self):
|
|
def handler(request):
|
|
return httpx.Response(
|
|
200,
|
|
content=TEST_HTML.encode(),
|
|
headers={"Content-Type": "text/html; charset=utf-8"},
|
|
)
|
|
|
|
extractor = self._extractor_with_transport(handler)
|
|
try:
|
|
text = await extractor._fetch("https://example.com/small")
|
|
finally:
|
|
await extractor.close()
|
|
|
|
assert text == TEST_HTML
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_200_returns_none(self):
|
|
def handler(request):
|
|
return httpx.Response(404, content=b"not found")
|
|
|
|
extractor = self._extractor_with_transport(handler)
|
|
try:
|
|
text = await extractor._fetch("https://example.com/missing")
|
|
finally:
|
|
await extractor.close()
|
|
|
|
assert text is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_body_returns_none(self):
|
|
def handler(request):
|
|
return httpx.Response(200, content=b"")
|
|
|
|
extractor = self._extractor_with_transport(handler)
|
|
try:
|
|
text = await extractor._fetch("https://example.com/empty")
|
|
finally:
|
|
await extractor.close()
|
|
|
|
assert text is None
|
|
|
|
|
|
class TestContentExtractionResult:
|
|
"""Tests for ContentExtractionResult model."""
|
|
|
|
def test_success_result(self):
|
|
"""Test creating a successful result."""
|
|
result = ContentExtractionResult(
|
|
url="https://example.com",
|
|
title="Test Article",
|
|
content="Article content",
|
|
author="John Doe",
|
|
date="2024-01-15",
|
|
language="en",
|
|
success=True,
|
|
error=None
|
|
)
|
|
|
|
assert result.url == "https://example.com"
|
|
assert result.success is True
|
|
assert result.error is None
|
|
|
|
def test_failure_result(self):
|
|
"""Test creating a failure result."""
|
|
result = ContentExtractionResult(
|
|
url="https://example.com/error",
|
|
content="",
|
|
success=False,
|
|
error="Failed to fetch URL"
|
|
)
|
|
|
|
assert result.url == "https://example.com/error"
|
|
assert result.success is False
|
|
assert result.error == "Failed to fetch URL"
|