feat(library-desk): add ingestion models for page processing
- Add IngestionResult model for single page ingestion - Add BatchIngestionResult for batch operations - Track vector chunks, graph entities, and relationships - Include processing time metrics Used by ingestion_service for page indexing
This commit is contained in:
@@ -0,0 +1,67 @@
|
|||||||
|
"""
|
||||||
|
Pydantic models for Document Ingestion system.
|
||||||
|
"""
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from typing import Optional, List, Dict, Any
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionRequest(BaseModel):
|
||||||
|
"""Request to ingest a wiki page."""
|
||||||
|
page_id: int = Field(..., description="Wiki page ID to ingest")
|
||||||
|
user: str = Field(default="jpmschweitzer", description="User identifier")
|
||||||
|
force_refresh: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description="Force re-ingestion even if page hasn't changed"
|
||||||
|
)
|
||||||
|
skip_vectors: bool = Field(default=False, description="Skip vector embedding generation")
|
||||||
|
skip_graph: bool = Field(default=False, description="Skip graph entity extraction")
|
||||||
|
|
||||||
|
|
||||||
|
class BatchIngestionRequest(BaseModel):
|
||||||
|
"""Request to ingest multiple wiki pages."""
|
||||||
|
page_ids: List[int] = Field(..., description="List of wiki page IDs to ingest")
|
||||||
|
user: str = Field(default="jpmschweitzer", description="User identifier")
|
||||||
|
force_refresh: bool = Field(default=False)
|
||||||
|
skip_vectors: bool = Field(default=False)
|
||||||
|
skip_graph: bool = Field(default=False)
|
||||||
|
max_concurrent: int = Field(
|
||||||
|
default=3,
|
||||||
|
ge=1,
|
||||||
|
le=10,
|
||||||
|
description="Maximum concurrent ingestion tasks"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionResult(BaseModel):
|
||||||
|
"""Result of a single page ingestion."""
|
||||||
|
page_id: int
|
||||||
|
page_title: str
|
||||||
|
page_path: Optional[str] = None
|
||||||
|
success: bool
|
||||||
|
error: Optional[str] = None
|
||||||
|
vector_chunks_created: int = 0
|
||||||
|
graph_entities_extracted: int = 0
|
||||||
|
graph_relationships_created: int = 0
|
||||||
|
processing_time_ms: float
|
||||||
|
|
||||||
|
|
||||||
|
class BatchIngestionResult(BaseModel):
|
||||||
|
"""Result of batch ingestion."""
|
||||||
|
total_pages: int
|
||||||
|
successful: int
|
||||||
|
failed: int
|
||||||
|
results: List[IngestionResult]
|
||||||
|
total_processing_time_ms: float
|
||||||
|
|
||||||
|
|
||||||
|
class IngestionStatus(BaseModel):
|
||||||
|
"""Status of an ingestion job."""
|
||||||
|
job_id: str
|
||||||
|
status: str # "queued", "processing", "completed", "failed"
|
||||||
|
progress: int # 0-100
|
||||||
|
page_id: Optional[int] = None
|
||||||
|
result: Optional[IngestionResult] = None
|
||||||
|
created_at: datetime
|
||||||
|
started_at: Optional[datetime] = None
|
||||||
|
completed_at: Optional[datetime] = None
|
||||||
Reference in New Issue
Block a user