jpmschweitzerandClaude Opus 4.5
61863ff597
Build and Push / build (release) Successful in 1m2s
- Add /rag/search endpoint for web, news, and image search via SearXNG - Add /content/extract and /content/extract/batch endpoints - Add ContentExtractor client using Trafilatura for content extraction - Enhance HybridRAG web search with full content extraction - Add Redis caching for search results - Add new configuration options for search and extraction timeouts 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
70 lines
2.5 KiB
Python
70 lines
2.5 KiB
Python
"""
|
|
Content extraction models for Library Desk.
|
|
|
|
Pydantic models for content extraction requests and responses.
|
|
"""
|
|
|
|
from typing import Optional, List
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class ContentExtractionResult(BaseModel):
|
|
"""Result of extracting content from a single URL."""
|
|
|
|
url: str = Field(..., description="The URL that was processed")
|
|
title: Optional[str] = Field(None, description="Page title if extracted")
|
|
content: str = Field("", description="Extracted main text content")
|
|
author: Optional[str] = Field(None, description="Author if available")
|
|
date: Optional[str] = Field(None, description="Publication date if available (ISO format)")
|
|
language: Optional[str] = Field(None, description="Detected language code")
|
|
success: bool = Field(..., description="Whether extraction succeeded")
|
|
error: Optional[str] = Field(None, description="Error message if extraction failed")
|
|
|
|
|
|
class ContentExtractionRequest(BaseModel):
|
|
"""Request to extract content from a single URL."""
|
|
|
|
url: str = Field(..., min_length=1, description="URL to extract content from")
|
|
include_metadata: bool = Field(default=True, description="Include title, author, date metadata")
|
|
max_length: Optional[int] = Field(
|
|
None,
|
|
ge=100,
|
|
le=50000,
|
|
description="Override default max content length"
|
|
)
|
|
|
|
|
|
class ContentExtractionResponse(BaseModel):
|
|
"""Response for single URL extraction."""
|
|
|
|
result: ContentExtractionResult
|
|
extraction_time_ms: int = Field(..., ge=0, description="Time taken to extract content")
|
|
|
|
|
|
class BatchContentExtractionRequest(BaseModel):
|
|
"""Request to extract content from multiple URLs."""
|
|
|
|
urls: List[str] = Field(
|
|
...,
|
|
min_length=1,
|
|
max_length=20,
|
|
description="URLs to extract content from (max 20)"
|
|
)
|
|
include_metadata: bool = Field(default=True, description="Include title, author, date metadata")
|
|
max_length: Optional[int] = Field(
|
|
None,
|
|
ge=100,
|
|
le=50000,
|
|
description="Override default max content length"
|
|
)
|
|
|
|
|
|
class BatchContentExtractionResponse(BaseModel):
|
|
"""Response for batch URL extraction."""
|
|
|
|
results: List[ContentExtractionResult]
|
|
total_urls: int = Field(..., ge=0, description="Total number of URLs processed")
|
|
successful: int = Field(..., ge=0, description="Number of successful extractions")
|
|
failed: int = Field(..., ge=0, description="Number of failed extractions")
|
|
extraction_time_ms: int = Field(..., ge=0, description="Total time for batch extraction")
|