jpmschweitzerandClaude Fable 5
86051d8022
Replace the four stub endpoints with real implementations, all requiring
an explicit tenant user (Phase B rule):
- /ingest/check-updates: GraphService now records a SHA-256 content_hash
on every Document node at ingestion time; the endpoint compares those
stored hashes against current Wiki.js page content in one UNWIND Cypher
query per tenant and returns changed/new/deleted page lists (entity-stub
pages excluded, pre-hash-tracking documents flagged stored_hash_missing).
- /ingest/status/{job_id}: backed by the Redis JobManager; jobs are
tenant-scoped (foreign jobs 404). /ingest/page, /ingest/batch and
/ingest/all now create job records and return job_id.
- /ingest/repo-status/{repository}: wiki page count vs indexed Document
nodes under users/{tenant}/{repository} plus tenant job stats.
- /deduplicate/check: tenant-scoped Qdrant similarity scan; chunk pairs
above ~0.9 cosine from different pages grouped per page pair with best
score and page references (read-only).
Supporting changes: get_job_manager dependency (+ shutdown close),
scroll_all_points can return vectors, VectorService.find_duplicate_pairs,
src/core/hashing.compute_content_hash. 13 new offline unit tests.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
78 lines
2.6 KiB
Python
78 lines
2.6 KiB
Python
"""
|
|
Pydantic models for Document Ingestion system.
|
|
"""
|
|
from pydantic import BaseModel, Field
|
|
from typing import Optional, List, Dict, Any
|
|
from datetime import datetime
|
|
|
|
from src.core.multi_tenancy import RequiredUser
|
|
|
|
|
|
class IngestionRequest(BaseModel):
|
|
"""Request to ingest a wiki page."""
|
|
page_id: int = Field(..., description="Wiki page ID to ingest")
|
|
user: RequiredUser = Field(..., description="User identifier (tenant). Required — ingestion writes to this tenant's namespaces only.")
|
|
force_refresh: bool = Field(
|
|
default=False,
|
|
description="Force re-ingestion even if page hasn't changed"
|
|
)
|
|
skip_vectors: bool = Field(default=False, description="Skip vector embedding generation")
|
|
skip_graph: bool = Field(default=False, description="Skip graph entity extraction")
|
|
|
|
|
|
class BatchIngestionRequest(BaseModel):
|
|
"""Request to ingest multiple wiki pages."""
|
|
page_ids: List[int] = Field(..., description="List of wiki page IDs to ingest")
|
|
user: RequiredUser = Field(..., description="User identifier (tenant). Required — ingestion writes to this tenant's namespaces only.")
|
|
force_refresh: bool = Field(default=False)
|
|
skip_vectors: bool = Field(default=False)
|
|
skip_graph: bool = Field(default=False)
|
|
max_concurrent: int = Field(
|
|
default=3,
|
|
ge=1,
|
|
le=10,
|
|
description="Maximum concurrent ingestion tasks"
|
|
)
|
|
|
|
|
|
class IngestionResult(BaseModel):
|
|
"""Result of a single page ingestion."""
|
|
page_id: int
|
|
page_title: str
|
|
page_path: Optional[str] = None
|
|
success: bool
|
|
error: Optional[str] = None
|
|
vector_chunks_created: int = 0
|
|
graph_entities_extracted: int = 0
|
|
graph_relationships_created: int = 0
|
|
processing_time_ms: float
|
|
job_id: Optional[str] = Field(
|
|
default=None,
|
|
description="Redis job-tracking ID (query via GET /ingest/status/{job_id})"
|
|
)
|
|
|
|
|
|
class BatchIngestionResult(BaseModel):
|
|
"""Result of batch ingestion."""
|
|
total_pages: int
|
|
successful: int
|
|
failed: int
|
|
results: List[IngestionResult]
|
|
total_processing_time_ms: float
|
|
job_id: Optional[str] = Field(
|
|
default=None,
|
|
description="Redis job-tracking ID (query via GET /ingest/status/{job_id})"
|
|
)
|
|
|
|
|
|
class IngestionStatus(BaseModel):
|
|
"""Status of an ingestion job."""
|
|
job_id: str
|
|
status: str # "queued", "processing", "completed", "failed"
|
|
progress: int # 0-100
|
|
page_id: Optional[int] = None
|
|
result: Optional[IngestionResult] = None
|
|
created_at: datetime
|
|
started_at: Optional[datetime] = None
|
|
completed_at: Optional[datetime] = None
|