jpmschweitzerandClaude Opus 4.5
61863ff597
Build and Push / build (release) Successful in 1m2s
- Add /rag/search endpoint for web, news, and image search via SearXNG - Add /content/extract and /content/extract/batch endpoints - Add ContentExtractor client using Trafilatura for content extraction - Enhance HybridRAG web search with full content extraction - Add Redis caching for search results - Add new configuration options for search and extraction timeouts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
133 lines
3.9 KiB
Python
133 lines
3.9 KiB
Python
"""
|
|
Content extraction router for Library Desk API.
|
|
|
|
Endpoints for extracting main content from web URLs using Trafilatura.
|
|
"""
|
|
|
|
import time
|
|
from fastapi import APIRouter, HTTPException, Depends
|
|
import logging
|
|
|
|
from src.models.content import (
|
|
ContentExtractionRequest,
|
|
ContentExtractionResponse,
|
|
BatchContentExtractionRequest,
|
|
BatchContentExtractionResponse,
|
|
)
|
|
from src.clients.content_extractor import ContentExtractor
|
|
from src.core.dependencies import verify_api_key
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter(prefix="/content", tags=["Content Extraction"])
|
|
|
|
|
|
# Lazy import to avoid circular dependency
|
|
def get_content_extractor() -> ContentExtractor:
|
|
"""Get content extractor instance."""
|
|
from src.core.dependencies import get_content_extractor as _get_extractor
|
|
return _get_extractor()
|
|
|
|
|
|
@router.post("/extract", response_model=ContentExtractionResponse)
|
|
async def extract_content(
|
|
request: ContentExtractionRequest,
|
|
api_key: str = Depends(verify_api_key)
|
|
):
|
|
"""
|
|
Extract main content from a single URL.
|
|
|
|
Uses Trafilatura to fetch the URL and extract the main text content,
|
|
removing navigation, ads, and other boilerplate.
|
|
|
|
**Example Request:**
|
|
```json
|
|
{
|
|
"url": "https://example.com/article",
|
|
"include_metadata": true,
|
|
"max_length": 2000
|
|
}
|
|
```
|
|
|
|
**Returns:** Extracted content with optional metadata (title, author, date)
|
|
"""
|
|
start_time = time.time()
|
|
|
|
try:
|
|
extractor = get_content_extractor()
|
|
result = await extractor.extract(
|
|
url=request.url,
|
|
include_metadata=request.include_metadata,
|
|
max_length=request.max_length
|
|
)
|
|
|
|
extraction_time_ms = int((time.time() - start_time) * 1000)
|
|
|
|
return ContentExtractionResponse(
|
|
result=result,
|
|
extraction_time_ms=extraction_time_ms
|
|
)
|
|
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
logger.error(f"Content extraction failed: {e}", exc_info=True)
|
|
raise HTTPException(status_code=500, detail="Content extraction failed")
|
|
|
|
|
|
@router.post("/extract/batch", response_model=BatchContentExtractionResponse)
|
|
async def extract_content_batch(
|
|
request: BatchContentExtractionRequest,
|
|
api_key: str = Depends(verify_api_key)
|
|
):
|
|
"""
|
|
Extract content from multiple URLs in parallel.
|
|
|
|
Processes up to 20 URLs concurrently with per-URL timeouts.
|
|
Failed extractions are included in results with success=false.
|
|
|
|
**Example Request:**
|
|
```json
|
|
{
|
|
"urls": [
|
|
"https://example.com/article1",
|
|
"https://example.com/article2"
|
|
],
|
|
"include_metadata": true,
|
|
"max_length": 2000
|
|
}
|
|
```
|
|
|
|
**Returns:** List of extraction results with success/failure counts
|
|
"""
|
|
start_time = time.time()
|
|
|
|
if not request.urls:
|
|
raise HTTPException(status_code=400, detail="URLs list cannot be empty")
|
|
|
|
try:
|
|
extractor = get_content_extractor()
|
|
results = await extractor.extract_batch(
|
|
urls=request.urls,
|
|
include_metadata=request.include_metadata,
|
|
max_length=request.max_length
|
|
)
|
|
|
|
extraction_time_ms = int((time.time() - start_time) * 1000)
|
|
successful = sum(1 for r in results if r.success)
|
|
failed = len(results) - successful
|
|
|
|
return BatchContentExtractionResponse(
|
|
results=results,
|
|
total_urls=len(request.urls),
|
|
successful=successful,
|
|
failed=failed,
|
|
extraction_time_ms=extraction_time_ms
|
|
)
|
|
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
except Exception as e:
|
|
logger.error(f"Batch content extraction failed: {e}", exc_info=True)
|
|
raise HTTPException(status_code=500, detail="Batch extraction failed")
|