feat: add RAG search endpoint with content extraction

- Add /rag/search endpoint for web, news, and image search via SearXNG
- Add /content/extract and /content/extract/batch endpoints
- Add ContentExtractor client using Trafilatura for content extraction
- Enhance HybridRAG web search with full content extraction
- Add Redis caching for search results
- Add new configuration options for search and extraction timeouts

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/models/content.py

@@ -0,0 +1,69 @@
+"""
+Content extraction models for Library Desk.
+
+Pydantic models for content extraction requests and responses.
+"""
+
+from typing import Optional, List
+from pydantic import BaseModel, Field
+
+
+class ContentExtractionResult(BaseModel):
+    """Result of extracting content from a single URL."""
+
+    url: str = Field(..., description="The URL that was processed")
+    title: Optional[str] = Field(None, description="Page title if extracted")
+    content: str = Field("", description="Extracted main text content")
+    author: Optional[str] = Field(None, description="Author if available")
+    date: Optional[str] = Field(None, description="Publication date if available (ISO format)")
+    language: Optional[str] = Field(None, description="Detected language code")
+    success: bool = Field(..., description="Whether extraction succeeded")
+    error: Optional[str] = Field(None, description="Error message if extraction failed")
+
+
+class ContentExtractionRequest(BaseModel):
+    """Request to extract content from a single URL."""
+
+    url: str = Field(..., min_length=1, description="URL to extract content from")
+    include_metadata: bool = Field(default=True, description="Include title, author, date metadata")
+    max_length: Optional[int] = Field(
+        None,
+        ge=100,
+        le=50000,
+        description="Override default max content length"
+    )
+
+
+class ContentExtractionResponse(BaseModel):
+    """Response for single URL extraction."""
+
+    result: ContentExtractionResult
+    extraction_time_ms: int = Field(..., ge=0, description="Time taken to extract content")
+
+
+class BatchContentExtractionRequest(BaseModel):
+    """Request to extract content from multiple URLs."""
+
+    urls: List[str] = Field(
+        ...,
+        min_length=1,
+        max_length=20,
+        description="URLs to extract content from (max 20)"
+    )
+    include_metadata: bool = Field(default=True, description="Include title, author, date metadata")
+    max_length: Optional[int] = Field(
+        None,
+        ge=100,
+        le=50000,
+        description="Override default max content length"
+    )
+
+
+class BatchContentExtractionResponse(BaseModel):
+    """Response for batch URL extraction."""
+
+    results: List[ContentExtractionResult]
+    total_urls: int = Field(..., ge=0, description="Total number of URLs processed")
+    successful: int = Field(..., ge=0, description="Number of successful extractions")
+    failed: int = Field(..., ge=0, description="Number of failed extractions")
+    extraction_time_ms: int = Field(..., ge=0, description="Total time for batch extraction")

src/models/rag_search.py

@@ -0,0 +1,88 @@
+"""
+RAG search models for Library Desk.
+
+Pydantic models for web/news/image search requests and responses.
+"""
+
+from enum import Enum
+from typing import Optional, List
+from pydantic import BaseModel, Field
+
+from src.core.multi_tenancy import DEFAULT_USER
+
+
+class SearchType(str, Enum):
+    """Supported search types."""
+    WEB = "web"
+    NEWS = "news"
+    IMAGES = "images"
+
+
+class RAGSearchRequest(BaseModel):
+    """Request for RAG search endpoint."""
+
+    query: str = Field(
+        ...,
+        min_length=1,
+        max_length=500,
+        description="The search query"
+    )
+    search_type: SearchType = Field(
+        default=SearchType.WEB,
+        description="Type of search: web, news, or images"
+    )
+    limit: int = Field(
+        default=10,
+        ge=1,
+        le=20,
+        description="Maximum number of results (1-20)"
+    )
+    user: str = Field(
+        default=DEFAULT_USER,
+        description="User identifier for rate limiting/personalization"
+    )
+
+
+class RAGSearchResult(BaseModel):
+    """A single search result with extracted content."""
+
+    title: str = Field(..., description="Title of the result")
+    url: str = Field(..., description="URL of the source")
+    content: str = Field(
+        "",
+        description="Full extracted text via Trafilatura (max ~2000 chars)"
+    )
+    snippet: str = Field(
+        "",
+        description="Original search engine snippet (150-300 chars)"
+    )
+    source: str = Field(..., description="Domain name of the source")
+    published_date: Optional[str] = Field(
+        None,
+        description="Publication date in ISO format if available"
+    )
+
+
+class RAGSearchResponse(BaseModel):
+    """Response from RAG search endpoint."""
+
+    query: str = Field(..., description="Echo of the original query")
+    search_type: SearchType = Field(..., description="Type of search performed")
+    results: List[RAGSearchResult] = Field(
+        default_factory=list,
+        description="List of search results with extracted content"
+    )
+    total_results: int = Field(
+        ...,
+        ge=0,
+        description="Number of results returned"
+    )
+    search_time_ms: int = Field(
+        ...,
+        ge=0,
+        description="Total time for search and content extraction"
+    )
+    sources_summary: str = Field(
+        "",
+        description="Markdown-formatted list of all source URLs"
+    )