""" Content extraction router for Library Desk API. Endpoints for extracting main content from web URLs using Trafilatura. """ import time from fastapi import APIRouter, HTTPException, Depends import logging from src.models.content import ( ContentExtractionRequest, ContentExtractionResponse, BatchContentExtractionRequest, BatchContentExtractionResponse, ) from src.clients.content_extractor import ContentExtractor from src.core.dependencies import verify_api_key logger = logging.getLogger(__name__) router = APIRouter(prefix="/content", tags=["Content Extraction"]) # Lazy import to avoid circular dependency def get_content_extractor() -> ContentExtractor: """Get content extractor instance.""" from src.core.dependencies import get_content_extractor as _get_extractor return _get_extractor() @router.post("/extract", response_model=ContentExtractionResponse) async def extract_content( request: ContentExtractionRequest, api_key: str = Depends(verify_api_key) ): """ Extract main content from a single URL. Uses Trafilatura to fetch the URL and extract the main text content, removing navigation, ads, and other boilerplate. **Example Request:** ```json { "url": "https://example.com/article", "include_metadata": true, "max_length": 2000 } ``` **Returns:** Extracted content with optional metadata (title, author, date) """ start_time = time.time() try: extractor = get_content_extractor() result = await extractor.extract( url=request.url, include_metadata=request.include_metadata, max_length=request.max_length ) extraction_time_ms = int((time.time() - start_time) * 1000) return ContentExtractionResponse( result=result, extraction_time_ms=extraction_time_ms ) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Content extraction failed: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Content extraction failed") @router.post("/extract/batch", response_model=BatchContentExtractionResponse) async def extract_content_batch( request: BatchContentExtractionRequest, api_key: str = Depends(verify_api_key) ): """ Extract content from multiple URLs in parallel. Processes up to 20 URLs concurrently with per-URL timeouts. Failed extractions are included in results with success=false. **Example Request:** ```json { "urls": [ "https://example.com/article1", "https://example.com/article2" ], "include_metadata": true, "max_length": 2000 } ``` **Returns:** List of extraction results with success/failure counts """ start_time = time.time() if not request.urls: raise HTTPException(status_code=400, detail="URLs list cannot be empty") try: extractor = get_content_extractor() results = await extractor.extract_batch( urls=request.urls, include_metadata=request.include_metadata, max_length=request.max_length ) extraction_time_ms = int((time.time() - start_time) * 1000) successful = sum(1 for r in results if r.success) failed = len(results) - successful return BatchContentExtractionResponse( results=results, total_urls=len(request.urls), successful=successful, failed=failed, extraction_time_ms=extraction_time_ms ) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Batch content extraction failed: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Batch extraction failed")