feat(library-desk): add ingestion models for page processing
- Add IngestionResult model for single page ingestion - Add BatchIngestionResult for batch operations - Track vector chunks, graph entities, and relationships - Include processing time metrics Used by ingestion_service for page indexing
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
"""
|
||||
Pydantic models for Document Ingestion system.
|
||||
"""
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional, List, Dict, Any
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class IngestionRequest(BaseModel):
|
||||
"""Request to ingest a wiki page."""
|
||||
page_id: int = Field(..., description="Wiki page ID to ingest")
|
||||
user: str = Field(default="jpmschweitzer", description="User identifier")
|
||||
force_refresh: bool = Field(
|
||||
default=False,
|
||||
description="Force re-ingestion even if page hasn't changed"
|
||||
)
|
||||
skip_vectors: bool = Field(default=False, description="Skip vector embedding generation")
|
||||
skip_graph: bool = Field(default=False, description="Skip graph entity extraction")
|
||||
|
||||
|
||||
class BatchIngestionRequest(BaseModel):
|
||||
"""Request to ingest multiple wiki pages."""
|
||||
page_ids: List[int] = Field(..., description="List of wiki page IDs to ingest")
|
||||
user: str = Field(default="jpmschweitzer", description="User identifier")
|
||||
force_refresh: bool = Field(default=False)
|
||||
skip_vectors: bool = Field(default=False)
|
||||
skip_graph: bool = Field(default=False)
|
||||
max_concurrent: int = Field(
|
||||
default=3,
|
||||
ge=1,
|
||||
le=10,
|
||||
description="Maximum concurrent ingestion tasks"
|
||||
)
|
||||
|
||||
|
||||
class IngestionResult(BaseModel):
|
||||
"""Result of a single page ingestion."""
|
||||
page_id: int
|
||||
page_title: str
|
||||
page_path: Optional[str] = None
|
||||
success: bool
|
||||
error: Optional[str] = None
|
||||
vector_chunks_created: int = 0
|
||||
graph_entities_extracted: int = 0
|
||||
graph_relationships_created: int = 0
|
||||
processing_time_ms: float
|
||||
|
||||
|
||||
class BatchIngestionResult(BaseModel):
|
||||
"""Result of batch ingestion."""
|
||||
total_pages: int
|
||||
successful: int
|
||||
failed: int
|
||||
results: List[IngestionResult]
|
||||
total_processing_time_ms: float
|
||||
|
||||
|
||||
class IngestionStatus(BaseModel):
|
||||
"""Status of an ingestion job."""
|
||||
job_id: str
|
||||
status: str # "queued", "processing", "completed", "failed"
|
||||
progress: int # 0-100
|
||||
page_id: Optional[int] = None
|
||||
result: Optional[IngestionResult] = None
|
||||
created_at: datetime
|
||||
started_at: Optional[datetime] = None
|
||||
completed_at: Optional[datetime] = None
|
||||
Reference in New Issue
Block a user