feat: add maintenance system with index reconciliation
Complete maintenance subsystem for index health and cleanup: Endpoints: - GET /maintenance/health - lightweight (or detailed) health check - POST /maintenance/cleanup/all - full orphan cleanup - POST /maintenance/cleanup/vectors - purge orphan vector chunks - POST /maintenance/cleanup/graph - purge orphan graph nodes - POST /maintenance/reconcile-index - cleanup + reindex missing pages Bidirectional orphan detection: - find_documents_without_vectors() in GraphService - find_chunks_without_graph_nodes() in VectorService Redis integration: - Tracks last_cleanup timestamp for scheduler visibility Config additions: - Document store, volatile cache, and maintenance settings - VectorServiceDep and GraphServiceDep type aliases 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -356,3 +356,189 @@ class VectorService:
|
||||
collections=[],
|
||||
total=0
|
||||
)
|
||||
|
||||
# ========== Cleanup Methods ==========
|
||||
|
||||
async def delete_document_chunks(
|
||||
self,
|
||||
document_id: str,
|
||||
user: str
|
||||
) -> int:
|
||||
"""
|
||||
Delete all chunks for a document (Document Store).
|
||||
|
||||
Args:
|
||||
document_id: Document UUID
|
||||
user: User identifier
|
||||
|
||||
Returns:
|
||||
Number of chunks deleted
|
||||
"""
|
||||
collection_name = get_qdrant_collection_name(user)
|
||||
|
||||
try:
|
||||
deleted_count = await self.qdrant.delete_by_filter(
|
||||
collection_name=collection_name,
|
||||
filter_conditions={"document_id": document_id}
|
||||
)
|
||||
|
||||
logger.info(f"Deleted chunks for document {document_id}")
|
||||
return deleted_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete chunks for document {document_id}: {e}", exc_info=True)
|
||||
return 0
|
||||
|
||||
async def delete_collection_chunks(
|
||||
self,
|
||||
collection_id: str,
|
||||
user: str
|
||||
) -> int:
|
||||
"""
|
||||
Delete all chunks for a document collection.
|
||||
|
||||
Args:
|
||||
collection_id: Collection UUID
|
||||
user: User identifier
|
||||
|
||||
Returns:
|
||||
Number of chunks deleted
|
||||
"""
|
||||
collection_name = get_qdrant_collection_name(user)
|
||||
|
||||
try:
|
||||
deleted_count = await self.qdrant.delete_by_filter(
|
||||
collection_name=collection_name,
|
||||
filter_conditions={"collection_id": collection_id}
|
||||
)
|
||||
|
||||
logger.info(f"Deleted chunks for collection {collection_id}")
|
||||
return deleted_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete chunks for collection {collection_id}: {e}", exc_info=True)
|
||||
return 0
|
||||
|
||||
async def get_all_chunk_references(
|
||||
self,
|
||||
user: str
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get all chunk references for orphan detection.
|
||||
|
||||
Returns list of {id, page_id, document_id} for all chunks.
|
||||
|
||||
Args:
|
||||
user: User identifier
|
||||
|
||||
Returns:
|
||||
List of chunk references
|
||||
"""
|
||||
collection_name = get_qdrant_collection_name(user)
|
||||
|
||||
try:
|
||||
# Check if collection exists
|
||||
exists = await self.qdrant.collection_exists(collection_name)
|
||||
if not exists:
|
||||
return []
|
||||
|
||||
all_points = await self.qdrant.scroll_all_points(
|
||||
collection_name=collection_name,
|
||||
batch_size=100,
|
||||
with_payload=True
|
||||
)
|
||||
|
||||
references = []
|
||||
for point in all_points:
|
||||
payload = point.get("payload", {})
|
||||
references.append({
|
||||
"chunk_id": point["id"],
|
||||
"page_id": payload.get("page_id"),
|
||||
"document_id": payload.get("document_id"),
|
||||
"collection_id": payload.get("collection_id"),
|
||||
"doc_type": payload.get("doc_type", "wiki")
|
||||
})
|
||||
|
||||
logger.info(f"Found {len(references)} chunks for user {user}")
|
||||
return references
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get chunk references: {e}", exc_info=True)
|
||||
return []
|
||||
|
||||
async def purge_chunks_by_ids(
|
||||
self,
|
||||
user: str,
|
||||
chunk_ids: List[str]
|
||||
) -> int:
|
||||
"""
|
||||
Delete specific chunks by their IDs.
|
||||
|
||||
Args:
|
||||
user: User identifier
|
||||
chunk_ids: List of chunk IDs to delete
|
||||
|
||||
Returns:
|
||||
Number of chunks deleted
|
||||
"""
|
||||
if not chunk_ids:
|
||||
return 0
|
||||
|
||||
collection_name = get_qdrant_collection_name(user)
|
||||
|
||||
try:
|
||||
deleted_count = await self.qdrant.delete_by_ids(
|
||||
collection_name=collection_name,
|
||||
point_ids=chunk_ids
|
||||
)
|
||||
|
||||
logger.info(f"Purged {deleted_count} orphan chunks for user {user}")
|
||||
return deleted_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to purge chunks: {e}", exc_info=True)
|
||||
return 0
|
||||
|
||||
def find_chunks_without_graph_nodes(
|
||||
self,
|
||||
chunk_references: List[Dict[str, Any]],
|
||||
graph_references: List[Dict[str, Any]]
|
||||
) -> List[str]:
|
||||
"""
|
||||
Find vector chunks that have no corresponding graph Document node.
|
||||
|
||||
Used for bidirectional orphan detection - vectors without graph representation.
|
||||
|
||||
Args:
|
||||
chunk_references: List from get_all_chunk_references()
|
||||
graph_references: List from GraphService.get_all_document_references()
|
||||
|
||||
Returns:
|
||||
List of orphan chunk IDs
|
||||
"""
|
||||
# Build sets of IDs that have graph nodes
|
||||
graph_page_ids = {
|
||||
ref.get("page_id") for ref in graph_references
|
||||
if ref.get("doc_type") == "wiki" and ref.get("page_id")
|
||||
}
|
||||
graph_doc_ids = {
|
||||
ref.get("document_id") for ref in graph_references
|
||||
if ref.get("doc_type") != "wiki" and ref.get("document_id")
|
||||
}
|
||||
|
||||
# Find chunks with no graph node
|
||||
orphan_ids = []
|
||||
for chunk in chunk_references:
|
||||
doc_type = chunk.get("doc_type", "wiki")
|
||||
|
||||
if doc_type == "wiki":
|
||||
page_id = chunk.get("page_id")
|
||||
if page_id and page_id not in graph_page_ids:
|
||||
orphan_ids.append(chunk["chunk_id"])
|
||||
else:
|
||||
document_id = chunk.get("document_id")
|
||||
if document_id and document_id not in graph_doc_ids:
|
||||
orphan_ids.append(chunk["chunk_id"])
|
||||
|
||||
logger.info(f"Found {len(orphan_ids)} vector chunks without graph nodes")
|
||||
return orphan_ids
|
||||
|
||||
Reference in New Issue
Block a user