feat: add RAG search endpoint with content extraction
Build and Push / build (release) Successful in 1m2s

- Add /rag/search endpoint for web, news, and image search via SearXNG
- Add /content/extract and /content/extract/batch endpoints
- Add ContentExtractor client using Trafilatura for content extraction
- Enhance HybridRAG web search with full content extraction
- Add Redis caching for search results
- Add new configuration options for search and extraction timeouts

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2025-12-15 15:50:48 +01:00
co-authored by Claude Opus 4.5
parent 16b86a7c1b
commit 61863ff597
16 changed files with 1642 additions and 4 deletions
+69
View File
@@ -0,0 +1,69 @@
"""
Content extraction models for Library Desk.
Pydantic models for content extraction requests and responses.
"""
from typing import Optional, List
from pydantic import BaseModel, Field
class ContentExtractionResult(BaseModel):
"""Result of extracting content from a single URL."""
url: str = Field(..., description="The URL that was processed")
title: Optional[str] = Field(None, description="Page title if extracted")
content: str = Field("", description="Extracted main text content")
author: Optional[str] = Field(None, description="Author if available")
date: Optional[str] = Field(None, description="Publication date if available (ISO format)")
language: Optional[str] = Field(None, description="Detected language code")
success: bool = Field(..., description="Whether extraction succeeded")
error: Optional[str] = Field(None, description="Error message if extraction failed")
class ContentExtractionRequest(BaseModel):
"""Request to extract content from a single URL."""
url: str = Field(..., min_length=1, description="URL to extract content from")
include_metadata: bool = Field(default=True, description="Include title, author, date metadata")
max_length: Optional[int] = Field(
None,
ge=100,
le=50000,
description="Override default max content length"
)
class ContentExtractionResponse(BaseModel):
"""Response for single URL extraction."""
result: ContentExtractionResult
extraction_time_ms: int = Field(..., ge=0, description="Time taken to extract content")
class BatchContentExtractionRequest(BaseModel):
"""Request to extract content from multiple URLs."""
urls: List[str] = Field(
...,
min_length=1,
max_length=20,
description="URLs to extract content from (max 20)"
)
include_metadata: bool = Field(default=True, description="Include title, author, date metadata")
max_length: Optional[int] = Field(
None,
ge=100,
le=50000,
description="Override default max content length"
)
class BatchContentExtractionResponse(BaseModel):
"""Response for batch URL extraction."""
results: List[ContentExtractionResult]
total_urls: int = Field(..., ge=0, description="Total number of URLs processed")
successful: int = Field(..., ge=0, description="Number of successful extractions")
failed: int = Field(..., ge=0, description="Number of failed extractions")
extraction_time_ms: int = Field(..., ge=0, description="Total time for batch extraction")