feat: add RAG search endpoint with content extraction
Build and Push / build (release) Successful in 1m2s
Build and Push / build (release) Successful in 1m2s
- Add /rag/search endpoint for web, news, and image search via SearXNG - Add /content/extract and /content/extract/batch endpoints - Add ContentExtractor client using Trafilatura for content extraction - Enhance HybridRAG web search with full content extraction - Add Redis caching for search results - Add new configuration options for search and extraction timeouts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Content extraction models for Library Desk.
|
||||
|
||||
Pydantic models for content extraction requests and responses.
|
||||
"""
|
||||
|
||||
from typing import Optional, List
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ContentExtractionResult(BaseModel):
|
||||
"""Result of extracting content from a single URL."""
|
||||
|
||||
url: str = Field(..., description="The URL that was processed")
|
||||
title: Optional[str] = Field(None, description="Page title if extracted")
|
||||
content: str = Field("", description="Extracted main text content")
|
||||
author: Optional[str] = Field(None, description="Author if available")
|
||||
date: Optional[str] = Field(None, description="Publication date if available (ISO format)")
|
||||
language: Optional[str] = Field(None, description="Detected language code")
|
||||
success: bool = Field(..., description="Whether extraction succeeded")
|
||||
error: Optional[str] = Field(None, description="Error message if extraction failed")
|
||||
|
||||
|
||||
class ContentExtractionRequest(BaseModel):
|
||||
"""Request to extract content from a single URL."""
|
||||
|
||||
url: str = Field(..., min_length=1, description="URL to extract content from")
|
||||
include_metadata: bool = Field(default=True, description="Include title, author, date metadata")
|
||||
max_length: Optional[int] = Field(
|
||||
None,
|
||||
ge=100,
|
||||
le=50000,
|
||||
description="Override default max content length"
|
||||
)
|
||||
|
||||
|
||||
class ContentExtractionResponse(BaseModel):
|
||||
"""Response for single URL extraction."""
|
||||
|
||||
result: ContentExtractionResult
|
||||
extraction_time_ms: int = Field(..., ge=0, description="Time taken to extract content")
|
||||
|
||||
|
||||
class BatchContentExtractionRequest(BaseModel):
|
||||
"""Request to extract content from multiple URLs."""
|
||||
|
||||
urls: List[str] = Field(
|
||||
...,
|
||||
min_length=1,
|
||||
max_length=20,
|
||||
description="URLs to extract content from (max 20)"
|
||||
)
|
||||
include_metadata: bool = Field(default=True, description="Include title, author, date metadata")
|
||||
max_length: Optional[int] = Field(
|
||||
None,
|
||||
ge=100,
|
||||
le=50000,
|
||||
description="Override default max content length"
|
||||
)
|
||||
|
||||
|
||||
class BatchContentExtractionResponse(BaseModel):
|
||||
"""Response for batch URL extraction."""
|
||||
|
||||
results: List[ContentExtractionResult]
|
||||
total_urls: int = Field(..., ge=0, description="Total number of URLs processed")
|
||||
successful: int = Field(..., ge=0, description="Number of successful extractions")
|
||||
failed: int = Field(..., ge=0, description="Number of failed extractions")
|
||||
extraction_time_ms: int = Field(..., ge=0, description="Total time for batch extraction")
|
||||
Reference in New Issue
Block a user