jpmschweitzerandClaude Fable 5
8348b4bf92
- embed_batch now issues one batched /api/embed request (the old loop made one /api/embeddings round-trip per chunk) with a per-text fallback that preserves None-for-failed semantics - update_from_page embeds all chunks in that single call and stores them in one Qdrant batch upsert (upsert_points) - reindex order reversed: upsert new points first, then prune stale ids (deterministic uuid5 ids make overwrite safe) so a mid-way failure no longer leaves the page with zero vectors - VectorUpdateSummary gains status (success/partial/failed) and chunks_skipped; all-embeddings-failed keeps old vectors and reports failure instead of success=True Measured on a 7-chunk page ingest (local server, llm_tester): ~375ms -> ~181ms median over 3 runs. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
105 lines
4.7 KiB
Python
105 lines
4.7 KiB
Python
"""
|
|
Vector models for Library Desk Qdrant operations.
|
|
|
|
Provides models for semantic search, document chunks, and embeddings.
|
|
"""
|
|
|
|
from pydantic import BaseModel, Field
|
|
from typing import List, Dict, Any, Optional
|
|
|
|
from src.core.multi_tenancy import RequiredUser
|
|
|
|
|
|
class DocumentChunk(BaseModel):
|
|
"""Document chunk with embedding."""
|
|
chunk_id: str = Field(..., description="Unique chunk ID (page_id:chunk_index)")
|
|
page_id: int = Field(..., description="Wiki page ID")
|
|
chunk_index: int = Field(..., description="Chunk index within document")
|
|
content: str = Field(..., description="Chunk text content")
|
|
metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
|
|
|
|
|
|
class SearchResult(BaseModel):
|
|
"""Semantic search result."""
|
|
chunk_id: str = Field(..., description="Chunk ID")
|
|
page_id: int = Field(..., description="Wiki page ID")
|
|
page_title: Optional[str] = Field(None, description="Page title")
|
|
page_path: Optional[str] = Field(None, description="Page path")
|
|
chunk_index: int = Field(..., description="Chunk index")
|
|
content: str = Field(..., description="Chunk content")
|
|
score: float = Field(..., description="Similarity score (0-1)")
|
|
metadata: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata")
|
|
|
|
|
|
class SearchRequest(BaseModel):
|
|
"""Semantic search request."""
|
|
query: str = Field(..., min_length=1, description="Search query")
|
|
user: RequiredUser = Field(..., description="User identifier (tenant). Required — search is scoped to this tenant's collection.")
|
|
limit: int = Field(default=10, ge=1, le=100, description="Maximum results")
|
|
score_threshold: float = Field(default=0.5, ge=0.0, le=1.0, description="Minimum similarity score")
|
|
|
|
|
|
class SearchResponse(BaseModel):
|
|
"""Semantic search response."""
|
|
query: str = Field(..., description="Search query")
|
|
results: List[SearchResult] = Field(..., description="Search results")
|
|
total: int = Field(..., description="Number of results")
|
|
user: str = Field(..., description="User filter applied")
|
|
|
|
|
|
class VectorUpdateRequest(BaseModel):
|
|
"""Request to update vectors from a wiki page."""
|
|
page_id: int = Field(..., description="Wiki page ID to process")
|
|
user: RequiredUser = Field(
|
|
...,
|
|
description="User identifier (tenant). Required — vectors are written to this tenant's collection."
|
|
)
|
|
force_refresh: bool = Field(
|
|
default=False,
|
|
description="Force re-embedding even if page hasn't changed"
|
|
)
|
|
|
|
|
|
class VectorUpdateSummary(BaseModel):
|
|
"""Summary of vector update operation."""
|
|
page_id: int = Field(..., description="Page ID processed")
|
|
page_title: str = Field(..., description="Page title")
|
|
chunks_created: int = Field(default=0, description="New chunks created")
|
|
chunks_updated: int = Field(default=0, description="Existing chunks updated")
|
|
chunks_deleted: int = Field(default=0, description="Old chunks deleted")
|
|
chunks_skipped: int = Field(default=0, description="Chunks skipped (embedding failed)")
|
|
status: str = Field(default="success", description="'success', 'partial' (some chunks skipped), or 'failed'")
|
|
total_chunks: int = Field(default=0, description="Total chunks for this page")
|
|
embedding_dim: int = Field(default=768, description="Embedding dimensionality")
|
|
processing_time_ms: float = Field(..., description="Processing time in milliseconds")
|
|
success: bool = Field(default=True, description="Whether update succeeded")
|
|
error_message: Optional[str] = Field(default=None, description="Error message if failed")
|
|
|
|
|
|
class CollectionInfo(BaseModel):
|
|
"""Qdrant collection information."""
|
|
name: str = Field(..., description="Collection name")
|
|
vectors_count: int = Field(..., description="Number of vectors")
|
|
points_count: int = Field(..., description="Number of points")
|
|
segments_count: int = Field(..., description="Number of segments")
|
|
|
|
|
|
class CollectionListResponse(BaseModel):
|
|
"""List of Qdrant collections."""
|
|
collections: List[CollectionInfo] = Field(..., description="List of collections")
|
|
total: int = Field(..., description="Total number of collections")
|
|
|
|
|
|
class DeletePageChunksRequest(BaseModel):
|
|
"""Request to delete all chunks for a page."""
|
|
page_id: int = Field(..., description="Wiki page ID")
|
|
user: RequiredUser = Field(..., description="User identifier (tenant). Required — deletion is scoped to this tenant's collection.")
|
|
|
|
|
|
class DeletePageChunksResponse(BaseModel):
|
|
"""Response from deleting page chunks."""
|
|
page_id: int = Field(..., description="Page ID")
|
|
chunks_deleted: int = Field(..., description="Number of chunks deleted")
|
|
success: bool = Field(..., description="Whether deletion succeeded")
|
|
message: str = Field(..., description="Result message")
|