feat: add maintenance system with index reconciliation

Complete maintenance subsystem for index health and cleanup:

Endpoints:
- GET /maintenance/health - lightweight (or detailed) health check
- POST /maintenance/cleanup/all - full orphan cleanup
- POST /maintenance/cleanup/vectors - purge orphan vector chunks
- POST /maintenance/cleanup/graph - purge orphan graph nodes
- POST /maintenance/reconcile-index - cleanup + reindex missing pages

Bidirectional orphan detection:
- find_documents_without_vectors() in GraphService
- find_chunks_without_graph_nodes() in VectorService

Redis integration:
- Tracks last_cleanup timestamp for scheduler visibility

Config additions:
- Document store, volatile cache, and maintenance settings
- VectorServiceDep and GraphServiceDep type aliases

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2025-12-24 16:22:39 +01:00
co-authored by Claude Opus 4.5
parent 0e6c3619eb
commit f756ca9490
7 changed files with 1851 additions and 6 deletions
+186
View File
@@ -356,3 +356,189 @@ class VectorService:
collections=[],
total=0
)
# ========== Cleanup Methods ==========
async def delete_document_chunks(
self,
document_id: str,
user: str
) -> int:
"""
Delete all chunks for a document (Document Store).
Args:
document_id: Document UUID
user: User identifier
Returns:
Number of chunks deleted
"""
collection_name = get_qdrant_collection_name(user)
try:
deleted_count = await self.qdrant.delete_by_filter(
collection_name=collection_name,
filter_conditions={"document_id": document_id}
)
logger.info(f"Deleted chunks for document {document_id}")
return deleted_count
except Exception as e:
logger.error(f"Failed to delete chunks for document {document_id}: {e}", exc_info=True)
return 0
async def delete_collection_chunks(
self,
collection_id: str,
user: str
) -> int:
"""
Delete all chunks for a document collection.
Args:
collection_id: Collection UUID
user: User identifier
Returns:
Number of chunks deleted
"""
collection_name = get_qdrant_collection_name(user)
try:
deleted_count = await self.qdrant.delete_by_filter(
collection_name=collection_name,
filter_conditions={"collection_id": collection_id}
)
logger.info(f"Deleted chunks for collection {collection_id}")
return deleted_count
except Exception as e:
logger.error(f"Failed to delete chunks for collection {collection_id}: {e}", exc_info=True)
return 0
async def get_all_chunk_references(
self,
user: str
) -> List[Dict[str, Any]]:
"""
Get all chunk references for orphan detection.
Returns list of {id, page_id, document_id} for all chunks.
Args:
user: User identifier
Returns:
List of chunk references
"""
collection_name = get_qdrant_collection_name(user)
try:
# Check if collection exists
exists = await self.qdrant.collection_exists(collection_name)
if not exists:
return []
all_points = await self.qdrant.scroll_all_points(
collection_name=collection_name,
batch_size=100,
with_payload=True
)
references = []
for point in all_points:
payload = point.get("payload", {})
references.append({
"chunk_id": point["id"],
"page_id": payload.get("page_id"),
"document_id": payload.get("document_id"),
"collection_id": payload.get("collection_id"),
"doc_type": payload.get("doc_type", "wiki")
})
logger.info(f"Found {len(references)} chunks for user {user}")
return references
except Exception as e:
logger.error(f"Failed to get chunk references: {e}", exc_info=True)
return []
async def purge_chunks_by_ids(
self,
user: str,
chunk_ids: List[str]
) -> int:
"""
Delete specific chunks by their IDs.
Args:
user: User identifier
chunk_ids: List of chunk IDs to delete
Returns:
Number of chunks deleted
"""
if not chunk_ids:
return 0
collection_name = get_qdrant_collection_name(user)
try:
deleted_count = await self.qdrant.delete_by_ids(
collection_name=collection_name,
point_ids=chunk_ids
)
logger.info(f"Purged {deleted_count} orphan chunks for user {user}")
return deleted_count
except Exception as e:
logger.error(f"Failed to purge chunks: {e}", exc_info=True)
return 0
def find_chunks_without_graph_nodes(
self,
chunk_references: List[Dict[str, Any]],
graph_references: List[Dict[str, Any]]
) -> List[str]:
"""
Find vector chunks that have no corresponding graph Document node.
Used for bidirectional orphan detection - vectors without graph representation.
Args:
chunk_references: List from get_all_chunk_references()
graph_references: List from GraphService.get_all_document_references()
Returns:
List of orphan chunk IDs
"""
# Build sets of IDs that have graph nodes
graph_page_ids = {
ref.get("page_id") for ref in graph_references
if ref.get("doc_type") == "wiki" and ref.get("page_id")
}
graph_doc_ids = {
ref.get("document_id") for ref in graph_references
if ref.get("doc_type") != "wiki" and ref.get("document_id")
}
# Find chunks with no graph node
orphan_ids = []
for chunk in chunk_references:
doc_type = chunk.get("doc_type", "wiki")
if doc_type == "wiki":
page_id = chunk.get("page_id")
if page_id and page_id not in graph_page_ids:
orphan_ids.append(chunk["chunk_id"])
else:
document_id = chunk.get("document_id")
if document_id and document_id not in graph_doc_ids:
orphan_ids.append(chunk["chunk_id"])
logger.info(f"Found {len(orphan_ids)} vector chunks without graph nodes")
return orphan_ids