"""Tests for ContentExtractor client."""
import pytest
from unittest.mock import AsyncMock, patch
import httpx
from src.clients.content_extractor import ContentExtractor
from src.models.content import ContentExtractionResult
TEST_HTML = "Test"
def _doc(text, **meta):
"""bare_extraction-style result dict."""
return {
"text": text,
"title": meta.get("title"),
"author": meta.get("author"),
"date": meta.get("date"),
"language": meta.get("language"),
}
@pytest.fixture
def content_extractor():
"""Create ContentExtractor with test configuration."""
return ContentExtractor(timeout=5, max_length=2000)
class TestContentExtractor:
"""Tests for ContentExtractor client."""
def test_init(self, content_extractor):
"""Test ContentExtractor initialization."""
assert content_extractor.timeout == 5
assert content_extractor.max_length == 2000
assert content_extractor.max_urls_per_batch == (
ContentExtractor.DEFAULT_MAX_URLS_PER_BATCH
)
@pytest.mark.asyncio
async def test_extract_success(self, content_extractor):
"""Test successful content extraction."""
test_url = "https://example.com/article"
test_content = "This is the extracted article content."
with patch.object(
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
mock_traf.bare_extraction.return_value = _doc(
test_content,
title="Test Article",
author="John Doe",
date="2024-01-15",
language="en",
)
result = await content_extractor.extract(test_url)
assert result.success is True
assert result.url == test_url
assert result.content == test_content
assert result.title == "Test Article"
assert result.error is None
@pytest.mark.asyncio
async def test_extraction_runs_once_per_url(self, content_extractor):
"""Trafilatura must parse the document exactly ONCE (the old code
ran extract() twice plus bare_extraction — three full parses)."""
with patch.object(
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
mock_traf.bare_extraction.return_value = _doc("content")
await content_extractor.extract("https://example.com/a")
assert mock_traf.bare_extraction.call_count == 1
mock_traf.extract.assert_not_called()
mock_traf.fetch_url.assert_not_called() # httpx fetches now
@pytest.mark.asyncio
async def test_extract_fetch_failure(self, content_extractor):
"""Test extraction when URL fetch fails."""
test_url = "https://example.com/nonexistent"
with patch.object(
content_extractor, "_fetch", AsyncMock(return_value=None)
):
result = await content_extractor.extract(test_url)
assert result.success is False
assert result.url == test_url
assert result.content == ""
assert "Failed to fetch URL" in result.error
@pytest.mark.asyncio
async def test_extract_fetch_timeout(self, content_extractor):
"""A slow server hits the httpx timeout instead of pinning a
worker thread on a blind download."""
test_url = "https://example.com/slow-server"
with patch.object(
content_extractor,
"_fetch",
AsyncMock(side_effect=httpx.ReadTimeout("read timeout")),
):
result = await content_extractor.extract(test_url)
assert result.success is False
assert "timed out" in result.error.lower()
@pytest.mark.asyncio
async def test_extract_network_error(self, content_extractor):
"""Connection errors return a failed result, not an exception."""
with patch.object(
content_extractor,
"_fetch",
AsyncMock(side_effect=httpx.ConnectError("refused")),
):
result = await content_extractor.extract("https://example.com/down")
assert result.success is False
assert "Failed to fetch URL" in result.error
@pytest.mark.asyncio
async def test_extract_no_content(self, content_extractor):
"""Test extraction when page has no extractable content."""
test_url = "https://example.com/empty"
with patch.object(
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
mock_traf.bare_extraction.return_value = None
result = await content_extractor.extract(test_url)
assert result.success is False
assert "No content extracted" in result.error
@pytest.mark.asyncio
async def test_extract_max_length_truncation(self, content_extractor):
"""Test that content is truncated to max length."""
test_url = "https://example.com/long-article"
# Content longer than max_length (2000)
long_content = "x" * 3000
with patch.object(
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
mock_traf.bare_extraction.return_value = _doc(long_content)
result = await content_extractor.extract(test_url)
assert result.success is True
assert len(result.content) <= content_extractor.max_length + 3 # +3 for "..."
assert result.content.endswith("...")
@pytest.mark.asyncio
async def test_extract_batch(self, content_extractor):
"""Test batch extraction of multiple URLs."""
test_urls = [
"https://example.com/article1",
"https://example.com/article2",
"https://example.com/article3",
]
with patch.object(
content_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
mock_traf.bare_extraction.return_value = _doc("Extracted content")
results = await content_extractor.extract_batch(test_urls)
assert len(results) == 3
for i, result in enumerate(results):
assert result.url == test_urls[i]
assert result.success is True
@pytest.mark.asyncio
async def test_extract_batch_caps_full_page_extractions(self):
"""URLs beyond the per-call cap are skipped (callers fall back to
the search snippet) instead of fanning out unbounded downloads."""
extractor = ContentExtractor(timeout=5, max_length=2000, max_urls_per_batch=2)
test_urls = [f"https://example.com/{i}" for i in range(5)]
with patch.object(
extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
) as mock_fetch, patch(
"src.clients.content_extractor.trafilatura"
) as mock_traf:
mock_traf.bare_extraction.return_value = _doc("content")
results = await extractor.extract_batch(test_urls)
assert len(results) == 5
assert mock_fetch.await_count == 2
assert [r.url for r in results] == test_urls
assert all(r.success for r in results[:2])
for skipped in results[2:]:
assert skipped.success is False
assert "cap" in skipped.error
@pytest.mark.asyncio
async def test_extract_parse_timeout(self):
"""Test extraction (parse) timeout handling."""
import time
test_url = "https://example.com/slow"
# Create an extractor with very short timeout
fast_extractor = ContentExtractor(timeout=0.001, max_length=2000)
def slow_parse(*args, **kwargs):
time.sleep(1) # Sleep synchronously (this runs in thread pool)
return _doc("late content")
with patch.object(
fast_extractor, "_fetch", AsyncMock(return_value=TEST_HTML)
), patch("src.clients.content_extractor.trafilatura") as mock_traf:
mock_traf.bare_extraction = slow_parse
result = await fast_extractor.extract(test_url)
assert result.success is False
assert "timed out" in result.error.lower()
@pytest.mark.asyncio
async def test_extract_from_html(self, content_extractor):
"""Test extraction from raw HTML."""
test_html = "Article content here."
with patch("src.clients.content_extractor.trafilatura") as mock_traf:
mock_traf.bare_extraction.return_value = _doc(
"Article content here.", title="Test"
)
result = await content_extractor.extract_from_html(
test_html, url="https://example.com"
)
assert result.success is True
assert result.content == "Article content here."
class TestFetchStreamingCap:
"""The 5MB cap must abort the DOWNLOAD, not just truncate after it."""
def _extractor_with_transport(self, handler):
extractor = ContentExtractor(timeout=5, max_length=2000)
extractor._http = httpx.AsyncClient(
transport=httpx.MockTransport(handler)
)
return extractor
@pytest.mark.asyncio
async def test_download_aborts_past_cap(self):
from src.clients.content_extractor import MAX_RESPONSE_BYTES
chunk = b"x" * (1024 * 1024) # 1MB per chunk
chunks_produced = []
async def body():
for i in range(100): # 100MB on offer
chunks_produced.append(i)
yield chunk
def handler(request):
return httpx.Response(
200,
content=body(),
headers={"Content-Type": "text/html; charset=utf-8"},
)
extractor = self._extractor_with_transport(handler)
try:
text = await extractor._fetch("https://example.com/huge")
finally:
await extractor.close()
assert text is not None
assert len(text.encode()) == MAX_RESPONSE_BYTES
# Streaming stopped at the cap instead of consuming all 100 chunks
assert len(chunks_produced) <= (MAX_RESPONSE_BYTES // len(chunk)) + 1
@pytest.mark.asyncio
async def test_small_response_returned_whole(self):
def handler(request):
return httpx.Response(
200,
content=TEST_HTML.encode(),
headers={"Content-Type": "text/html; charset=utf-8"},
)
extractor = self._extractor_with_transport(handler)
try:
text = await extractor._fetch("https://example.com/small")
finally:
await extractor.close()
assert text == TEST_HTML
@pytest.mark.asyncio
async def test_non_200_returns_none(self):
def handler(request):
return httpx.Response(404, content=b"not found")
extractor = self._extractor_with_transport(handler)
try:
text = await extractor._fetch("https://example.com/missing")
finally:
await extractor.close()
assert text is None
@pytest.mark.asyncio
async def test_empty_body_returns_none(self):
def handler(request):
return httpx.Response(200, content=b"")
extractor = self._extractor_with_transport(handler)
try:
text = await extractor._fetch("https://example.com/empty")
finally:
await extractor.close()
assert text is None
class TestContentExtractionResult:
"""Tests for ContentExtractionResult model."""
def test_success_result(self):
"""Test creating a successful result."""
result = ContentExtractionResult(
url="https://example.com",
title="Test Article",
content="Article content",
author="John Doe",
date="2024-01-15",
language="en",
success=True,
error=None
)
assert result.url == "https://example.com"
assert result.success is True
assert result.error is None
def test_failure_result(self):
"""Test creating a failure result."""
result = ContentExtractionResult(
url="https://example.com/error",
content="",
success=False,
error="Failed to fetch URL"
)
assert result.url == "https://example.com/error"
assert result.success is False
assert result.error == "Failed to fetch URL"