jpmschweitzerandClaude Fable 5
8348b4bf92
- embed_batch now issues one batched /api/embed request (the old loop made one /api/embeddings round-trip per chunk) with a per-text fallback that preserves None-for-failed semantics - update_from_page embeds all chunks in that single call and stores them in one Qdrant batch upsert (upsert_points) - reindex order reversed: upsert new points first, then prune stale ids (deterministic uuid5 ids make overwrite safe) so a mid-way failure no longer leaves the page with zero vectors - VectorUpdateSummary gains status (success/partial/failed) and chunks_skipped; all-embeddings-failed keeps old vectors and reports failure instead of success=True Measured on a 7-chunk page ingest (local server, llm_tester): ~375ms -> ~181ms median over 3 runs. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
747 lines
25 KiB
Python
747 lines
25 KiB
Python
"""
|
|
Vector service for Library Desk Qdrant operations.
|
|
|
|
Handles semantic search, document chunking, and embeddings.
|
|
"""
|
|
|
|
import re
|
|
import time
|
|
import hashlib
|
|
import uuid
|
|
from typing import List, Dict, Any, Optional
|
|
import logging
|
|
|
|
from src.clients.qdrant_client import QdrantClientWrapper
|
|
from src.clients.wikijs_client import WikiJSClient
|
|
from src.clients.ollama_client import OllamaClient
|
|
from src.core.multi_tenancy import get_qdrant_collection_name, is_path_in_user_namespace
|
|
from src.models.vector import (
|
|
SearchResult, SearchResponse, VectorUpdateSummary,
|
|
DocumentChunk, CollectionInfo, CollectionListResponse
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class VectorService:
|
|
"""
|
|
Service for Qdrant vector operations.
|
|
|
|
Responsibilities:
|
|
- Document chunking
|
|
- Embedding generation
|
|
- Semantic search
|
|
- Vector CRUD operations
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
qdrant_client: QdrantClientWrapper,
|
|
wikijs_client: WikiJSClient,
|
|
ollama_client: OllamaClient,
|
|
chunk_size: int = 500,
|
|
chunk_overlap: int = 50
|
|
):
|
|
"""
|
|
Initialize vector service.
|
|
|
|
Args:
|
|
qdrant_client: Qdrant database client
|
|
wikijs_client: Wiki.js client for fetching pages
|
|
ollama_client: Ollama client for embeddings
|
|
chunk_size: Target chunk size in tokens (approximate)
|
|
chunk_overlap: Overlap between chunks in tokens
|
|
"""
|
|
self.qdrant = qdrant_client
|
|
self.wiki = wikijs_client
|
|
self.ollama = ollama_client
|
|
self.chunk_size = chunk_size
|
|
self.chunk_overlap = chunk_overlap
|
|
|
|
def _chunk_text(self, text: str) -> List[str]:
|
|
"""
|
|
Chunk text into overlapping segments.
|
|
|
|
Simple word-based chunking for now.
|
|
TODO: Use tiktoken or similar for token-accurate chunking.
|
|
|
|
Args:
|
|
text: Text to chunk
|
|
|
|
Returns:
|
|
List of text chunks
|
|
"""
|
|
# Remove extra whitespace
|
|
text = re.sub(r'\s+', ' ', text).strip()
|
|
|
|
# Split into words (approximates tokens)
|
|
words = text.split()
|
|
|
|
if len(words) <= self.chunk_size:
|
|
return [text]
|
|
|
|
chunks = []
|
|
start = 0
|
|
|
|
while start < len(words):
|
|
end = start + self.chunk_size
|
|
chunk_words = words[start:end]
|
|
chunks.append(' '.join(chunk_words))
|
|
|
|
# Move start forward with overlap
|
|
start = end - self.chunk_overlap
|
|
|
|
return chunks
|
|
|
|
async def update_from_page(
|
|
self,
|
|
page_id: int,
|
|
user: str,
|
|
force_refresh: bool = False
|
|
) -> VectorUpdateSummary:
|
|
"""
|
|
Update vector embeddings from a wiki page.
|
|
|
|
Chunks the page content, generates embeddings, and upserts to Qdrant.
|
|
|
|
Args:
|
|
page_id: Wiki page ID
|
|
user: User identifier
|
|
force_refresh: Force re-embedding even if unchanged
|
|
|
|
Returns:
|
|
Summary of update operation
|
|
"""
|
|
start_time = time.time()
|
|
|
|
try:
|
|
# Fetch page from Wiki.js
|
|
page = await self.wiki.get_page(page_id)
|
|
if not page:
|
|
raise ValueError(f"Page {page_id} not found")
|
|
|
|
# TENANT ISOLATION: only pages inside the user's own wiki
|
|
# namespace may be embedded into that user's collection.
|
|
# Without this check any tenant could ingest (and then read)
|
|
# another tenant's wiki content.
|
|
if not is_path_in_user_namespace(page.get("path", ""), user):
|
|
raise ValueError(
|
|
f"Page {page_id} (path: {page.get('path')!r}) is outside "
|
|
f"user '{user}' namespace - refusing cross-tenant ingestion"
|
|
)
|
|
|
|
# Get collection name for user
|
|
collection_name = get_qdrant_collection_name(user)
|
|
|
|
# Ensure collection exists
|
|
await self.qdrant.ensure_collection(collection_name)
|
|
|
|
# Extract content
|
|
content = page.get("content", "")
|
|
title = page.get("title", "")
|
|
path = page.get("path", "")
|
|
|
|
if not content:
|
|
logger.warning(f"Page {page_id} has no content, skipping vector update")
|
|
return VectorUpdateSummary(
|
|
page_id=page_id,
|
|
page_title=title,
|
|
processing_time_ms=(time.time() - start_time) * 1000,
|
|
success=True
|
|
)
|
|
|
|
# Chunk the content
|
|
chunks = self._chunk_text(content)
|
|
logger.info(f"Split page {page_id} into {len(chunks)} chunks")
|
|
|
|
# Generate ALL embeddings in one batched /api/embed call
|
|
# (previously one sequential Ollama round-trip per chunk)
|
|
embeddings = await self.ollama.embed_batch(chunks)
|
|
|
|
# Build points; deterministic uuid5 IDs mean re-upserting the
|
|
# same page overwrites its previous chunks in place.
|
|
points = []
|
|
chunks_skipped = 0
|
|
embedding_dim = 768
|
|
for idx, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)):
|
|
if not embedding:
|
|
chunks_skipped += 1
|
|
logger.error(f"Failed to generate embedding for page {page_id} chunk {idx}")
|
|
continue
|
|
|
|
embedding_dim = len(embedding)
|
|
chunk_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, f"page_{page_id}_chunk_{idx}"))
|
|
points.append({
|
|
"id": chunk_id,
|
|
"vector": embedding,
|
|
"payload": {
|
|
"page_id": page_id,
|
|
"page_title": title,
|
|
"page_path": path,
|
|
"chunk_index": idx,
|
|
"chunk_text": chunk_text,
|
|
"user": user
|
|
}
|
|
})
|
|
|
|
# Upsert BEFORE deleting stale points. The old order (delete all,
|
|
# then embed+upsert one by one) left the page with ZERO vectors if
|
|
# anything failed mid-way; now old vectors survive until the new
|
|
# ones are safely stored.
|
|
chunks_created = 0
|
|
if points:
|
|
chunks_created = await self.qdrant.upsert_points(
|
|
collection_name=collection_name,
|
|
points=points
|
|
)
|
|
|
|
# Prune stale points from a previous version of the page (chunk
|
|
# indexes beyond the new count, or indexes whose new embedding
|
|
# failed). Only prune if the new upsert actually stored points —
|
|
# a fully failed embedding pass must not wipe the old vectors.
|
|
deleted_count = 0
|
|
if points:
|
|
new_ids = {p["id"] for p in points}
|
|
existing = await self.qdrant.scroll_all_points(
|
|
collection_name=collection_name,
|
|
filter_conditions={"page_id": page_id},
|
|
with_payload=False
|
|
)
|
|
stale_ids = [pt["id"] for pt in existing if pt["id"] not in new_ids]
|
|
if stale_ids:
|
|
deleted_count = await self.qdrant.delete_by_ids(
|
|
collection_name=collection_name,
|
|
point_ids=stale_ids
|
|
)
|
|
|
|
processing_time_ms = (time.time() - start_time) * 1000
|
|
|
|
if chunks_created == 0:
|
|
status = "failed"
|
|
elif chunks_skipped > 0:
|
|
status = "partial"
|
|
else:
|
|
status = "success"
|
|
|
|
logger.info(
|
|
f"Updated vectors for page {page_id}: "
|
|
f"{chunks_created} chunks created, {chunks_skipped} skipped, "
|
|
f"{deleted_count} stale chunks deleted ({status})"
|
|
)
|
|
|
|
return VectorUpdateSummary(
|
|
page_id=page_id,
|
|
page_title=title,
|
|
chunks_created=chunks_created,
|
|
chunks_deleted=deleted_count,
|
|
chunks_skipped=chunks_skipped,
|
|
total_chunks=chunks_created,
|
|
embedding_dim=embedding_dim,
|
|
processing_time_ms=processing_time_ms,
|
|
success=chunks_created > 0,
|
|
status=status,
|
|
error_message=(
|
|
f"{chunks_skipped}/{len(chunks)} chunk embeddings failed"
|
|
if chunks_skipped else None
|
|
)
|
|
)
|
|
|
|
except Exception as e:
|
|
processing_time_ms = (time.time() - start_time) * 1000
|
|
logger.error(f"Failed to update vectors for page {page_id}: {e}", exc_info=True)
|
|
|
|
return VectorUpdateSummary(
|
|
page_id=page_id,
|
|
page_title="Unknown",
|
|
processing_time_ms=processing_time_ms,
|
|
success=False,
|
|
error_message=str(e)
|
|
)
|
|
|
|
async def search(
|
|
self,
|
|
query: str,
|
|
user: str,
|
|
limit: int = 10,
|
|
score_threshold: float = 0.5
|
|
) -> SearchResponse:
|
|
"""
|
|
Perform semantic search across user's documents.
|
|
|
|
Args:
|
|
query: Search query text
|
|
user: User identifier
|
|
limit: Maximum results to return
|
|
score_threshold: Minimum similarity score (0-1)
|
|
|
|
Returns:
|
|
Search results with similarity scores
|
|
"""
|
|
start_time = time.time()
|
|
|
|
try:
|
|
# Get collection name
|
|
collection_name = get_qdrant_collection_name(user)
|
|
|
|
# Check if collection exists
|
|
exists = await self.qdrant.collection_exists(collection_name)
|
|
if not exists:
|
|
logger.info(f"Collection {collection_name} doesn't exist, returning empty results")
|
|
return SearchResponse(
|
|
query=query,
|
|
results=[],
|
|
total=0,
|
|
user=user
|
|
)
|
|
|
|
# Generate query embedding
|
|
query_embedding = await self.ollama.embed(query)
|
|
if not query_embedding:
|
|
raise ValueError("Failed to generate query embedding")
|
|
|
|
# Search in Qdrant
|
|
search_results = await self.qdrant.search_vectors(
|
|
collection_name=collection_name,
|
|
query_vector=query_embedding,
|
|
limit=limit,
|
|
score_threshold=score_threshold
|
|
)
|
|
|
|
# Convert to SearchResult models
|
|
results = []
|
|
for result in search_results:
|
|
payload = result.get("payload", {})
|
|
results.append(SearchResult(
|
|
chunk_id=result["id"],
|
|
page_id=payload.get("page_id", 0),
|
|
page_title=payload.get("page_title"),
|
|
page_path=payload.get("page_path"),
|
|
chunk_index=payload.get("chunk_index", 0),
|
|
content=payload.get("chunk_text", ""),
|
|
score=result["score"],
|
|
metadata=payload
|
|
))
|
|
|
|
query_time_ms = (time.time() - start_time) * 1000
|
|
logger.info(f"Semantic search completed in {query_time_ms:.2f}ms: {len(results)} results")
|
|
|
|
return SearchResponse(
|
|
query=query,
|
|
results=results,
|
|
total=len(results),
|
|
user=user
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Semantic search failed: {e}", exc_info=True)
|
|
return SearchResponse(
|
|
query=query,
|
|
results=[],
|
|
total=0,
|
|
user=user
|
|
)
|
|
|
|
async def delete_page_chunks(
|
|
self,
|
|
page_id: int,
|
|
user: str
|
|
) -> int:
|
|
"""
|
|
Delete all chunks for a wiki page.
|
|
|
|
Args:
|
|
page_id: Wiki page ID
|
|
user: User identifier
|
|
|
|
Returns:
|
|
Number of chunks deleted
|
|
"""
|
|
collection_name = get_qdrant_collection_name(user)
|
|
|
|
try:
|
|
deleted_count = await self.qdrant.delete_by_filter(
|
|
collection_name=collection_name,
|
|
filter_conditions={"page_id": page_id}
|
|
)
|
|
|
|
logger.info(f"Deleted {deleted_count} chunks for page {page_id}")
|
|
return deleted_count
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete chunks for page {page_id}: {e}", exc_info=True)
|
|
return 0
|
|
|
|
async def list_collections(self) -> CollectionListResponse:
|
|
"""
|
|
List all Qdrant collections.
|
|
|
|
Returns:
|
|
List of collections with stats
|
|
"""
|
|
try:
|
|
collections_data = await self.qdrant.list_collections()
|
|
|
|
collections = []
|
|
for coll in collections_data:
|
|
collections.append(CollectionInfo(
|
|
name=coll["name"],
|
|
vectors_count=coll.get("vectors_count", 0),
|
|
points_count=coll.get("points_count", 0),
|
|
segments_count=coll.get("segments_count", 0)
|
|
))
|
|
|
|
return CollectionListResponse(
|
|
collections=collections,
|
|
total=len(collections)
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to list collections: {e}", exc_info=True)
|
|
return CollectionListResponse(
|
|
collections=[],
|
|
total=0
|
|
)
|
|
|
|
# ========== Cleanup Methods ==========
|
|
|
|
async def delete_document_chunks(
|
|
self,
|
|
document_id: str,
|
|
user: str
|
|
) -> int:
|
|
"""
|
|
Delete all chunks for a document (Document Store).
|
|
|
|
Args:
|
|
document_id: Document UUID
|
|
user: User identifier
|
|
|
|
Returns:
|
|
Number of chunks deleted
|
|
"""
|
|
collection_name = get_qdrant_collection_name(user)
|
|
|
|
try:
|
|
deleted_count = await self.qdrant.delete_by_filter(
|
|
collection_name=collection_name,
|
|
filter_conditions={"document_id": document_id}
|
|
)
|
|
|
|
logger.info(f"Deleted chunks for document {document_id}")
|
|
return deleted_count
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete chunks for document {document_id}: {e}", exc_info=True)
|
|
return 0
|
|
|
|
async def delete_paperless_document_chunks(
|
|
self,
|
|
paperless_id: int,
|
|
user: str
|
|
) -> int:
|
|
"""
|
|
Delete all chunks for a Paperless document.
|
|
|
|
Args:
|
|
paperless_id: Paperless-ngx document ID
|
|
user: User identifier
|
|
|
|
Returns:
|
|
Number of chunks deleted
|
|
"""
|
|
collection_name = get_qdrant_collection_name(user)
|
|
|
|
try:
|
|
deleted_count = await self.qdrant.delete_by_filter(
|
|
collection_name=collection_name,
|
|
filter_conditions={
|
|
"doc_type": "document",
|
|
"paperless_id": paperless_id
|
|
}
|
|
)
|
|
|
|
logger.info(f"Deleted chunks for Paperless document {paperless_id}")
|
|
return deleted_count
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete chunks for Paperless document {paperless_id}: {e}", exc_info=True)
|
|
return 0
|
|
|
|
async def delete_collection_chunks(
|
|
self,
|
|
collection_id: str,
|
|
user: str
|
|
) -> int:
|
|
"""
|
|
Delete all chunks for a document collection.
|
|
|
|
Args:
|
|
collection_id: Collection UUID
|
|
user: User identifier
|
|
|
|
Returns:
|
|
Number of chunks deleted
|
|
"""
|
|
collection_name = get_qdrant_collection_name(user)
|
|
|
|
try:
|
|
deleted_count = await self.qdrant.delete_by_filter(
|
|
collection_name=collection_name,
|
|
filter_conditions={"collection_id": collection_id}
|
|
)
|
|
|
|
logger.info(f"Deleted chunks for collection {collection_id}")
|
|
return deleted_count
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete chunks for collection {collection_id}: {e}", exc_info=True)
|
|
return 0
|
|
|
|
async def get_all_chunk_references(
|
|
self,
|
|
user: str
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Get all chunk references for orphan detection.
|
|
|
|
Returns list of {id, page_id, document_id} for all chunks.
|
|
|
|
Args:
|
|
user: User identifier
|
|
|
|
Returns:
|
|
List of chunk references
|
|
"""
|
|
collection_name = get_qdrant_collection_name(user)
|
|
|
|
try:
|
|
# Check if collection exists
|
|
exists = await self.qdrant.collection_exists(collection_name)
|
|
if not exists:
|
|
return []
|
|
|
|
all_points = await self.qdrant.scroll_all_points(
|
|
collection_name=collection_name,
|
|
batch_size=100,
|
|
with_payload=True
|
|
)
|
|
|
|
references = []
|
|
for point in all_points:
|
|
payload = point.get("payload", {})
|
|
references.append({
|
|
"chunk_id": point["id"],
|
|
"page_id": payload.get("page_id"),
|
|
"document_id": payload.get("document_id"),
|
|
"paperless_id": payload.get("paperless_id"), # For Paperless documents
|
|
"collection_id": payload.get("collection_id"),
|
|
"doc_type": payload.get("doc_type", "wiki")
|
|
})
|
|
|
|
logger.info(f"Found {len(references)} chunks for user {user}")
|
|
return references
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to get chunk references: {e}", exc_info=True)
|
|
return []
|
|
|
|
async def purge_chunks_by_ids(
|
|
self,
|
|
user: str,
|
|
chunk_ids: List[str]
|
|
) -> int:
|
|
"""
|
|
Delete specific chunks by their IDs.
|
|
|
|
Args:
|
|
user: User identifier
|
|
chunk_ids: List of chunk IDs to delete
|
|
|
|
Returns:
|
|
Number of chunks deleted
|
|
"""
|
|
if not chunk_ids:
|
|
return 0
|
|
|
|
collection_name = get_qdrant_collection_name(user)
|
|
|
|
try:
|
|
deleted_count = await self.qdrant.delete_by_ids(
|
|
collection_name=collection_name,
|
|
point_ids=chunk_ids
|
|
)
|
|
|
|
logger.info(f"Purged {deleted_count} orphan chunks for user {user}")
|
|
return deleted_count
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to purge chunks: {e}", exc_info=True)
|
|
return 0
|
|
|
|
async def find_duplicate_pairs(
|
|
self,
|
|
user: str,
|
|
similarity_threshold: float = 0.9,
|
|
max_chunks_scanned: int = 2000,
|
|
max_pairs: int = 100
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Tenant-scoped similarity scan for near-duplicate wiki pages.
|
|
|
|
Scrolls the tenant's own Qdrant collection (never another tenant's),
|
|
then queries each chunk's vector against the same collection. Chunk
|
|
pairs from DIFFERENT pages scoring above the threshold are grouped
|
|
per page pair with the best score and the number of matching chunk
|
|
pairs. Read-only: nothing is modified.
|
|
|
|
Args:
|
|
user: Tenant user identifier
|
|
similarity_threshold: Minimum cosine similarity (default 0.9)
|
|
max_chunks_scanned: Safety cap on chunks used as probes
|
|
max_pairs: Maximum page pairs returned (highest score first)
|
|
|
|
Returns:
|
|
{
|
|
"chunks_scanned": int,
|
|
"duplicate_groups": [
|
|
{
|
|
"pages": [{page_id, path, title}, {page_id, path, title}],
|
|
"max_similarity": float,
|
|
"matching_chunk_pairs": int
|
|
}, ...
|
|
]
|
|
}
|
|
"""
|
|
collection_name = get_qdrant_collection_name(user)
|
|
|
|
exists = await self.qdrant.collection_exists(collection_name)
|
|
if not exists:
|
|
return {"chunks_scanned": 0, "duplicate_groups": []}
|
|
|
|
points = await self.qdrant.scroll_all_points(
|
|
collection_name=collection_name,
|
|
batch_size=100,
|
|
with_payload=True,
|
|
with_vectors=True
|
|
)
|
|
|
|
# Only wiki chunks participate (documents have their own dedup story)
|
|
wiki_points = [
|
|
p for p in points
|
|
if p.get("vector") is not None
|
|
and (p.get("payload") or {}).get("doc_type", "wiki") == "wiki"
|
|
and (p.get("payload") or {}).get("page_id")
|
|
][:max_chunks_scanned]
|
|
|
|
page_meta: Dict[int, Dict[str, Any]] = {}
|
|
pair_stats: Dict[tuple, Dict[str, Any]] = {}
|
|
seen_chunk_pairs = set()
|
|
|
|
for point in wiki_points:
|
|
payload = point.get("payload") or {}
|
|
page_id = payload.get("page_id")
|
|
page_meta.setdefault(page_id, {
|
|
"page_id": page_id,
|
|
"path": payload.get("page_path", ""),
|
|
"title": payload.get("page_title", "")
|
|
})
|
|
|
|
hits = await self.qdrant.search_vectors(
|
|
collection_name=collection_name,
|
|
query_vector=point["vector"],
|
|
limit=10,
|
|
score_threshold=similarity_threshold
|
|
)
|
|
|
|
for hit in hits:
|
|
hit_payload = hit.get("payload") or {}
|
|
hit_page_id = hit_payload.get("page_id")
|
|
if not hit_page_id or hit_page_id == page_id:
|
|
continue
|
|
if hit_payload.get("doc_type", "wiki") != "wiki":
|
|
continue
|
|
|
|
# Deduplicate the A->B / B->A chunk pair directions
|
|
chunk_pair = tuple(sorted((point["id"], hit["id"])))
|
|
if chunk_pair in seen_chunk_pairs:
|
|
continue
|
|
seen_chunk_pairs.add(chunk_pair)
|
|
|
|
page_meta.setdefault(hit_page_id, {
|
|
"page_id": hit_page_id,
|
|
"path": hit_payload.get("page_path", ""),
|
|
"title": hit_payload.get("page_title", "")
|
|
})
|
|
|
|
page_pair = tuple(sorted((page_id, hit_page_id)))
|
|
stats = pair_stats.setdefault(page_pair, {
|
|
"max_similarity": 0.0,
|
|
"matching_chunk_pairs": 0
|
|
})
|
|
stats["max_similarity"] = max(stats["max_similarity"], hit["score"])
|
|
stats["matching_chunk_pairs"] += 1
|
|
|
|
groups = [
|
|
{
|
|
"pages": [page_meta[a], page_meta[b]],
|
|
"max_similarity": stats["max_similarity"],
|
|
"matching_chunk_pairs": stats["matching_chunk_pairs"]
|
|
}
|
|
for (a, b), stats in pair_stats.items()
|
|
]
|
|
groups.sort(key=lambda g: g["max_similarity"], reverse=True)
|
|
|
|
logger.info(
|
|
f"Duplicate scan for {user}: {len(wiki_points)} chunks scanned, "
|
|
f"{len(groups)} page pairs above {similarity_threshold}"
|
|
)
|
|
|
|
return {
|
|
"chunks_scanned": len(wiki_points),
|
|
"duplicate_groups": groups[:max_pairs]
|
|
}
|
|
|
|
def find_chunks_without_graph_nodes(
|
|
self,
|
|
chunk_references: List[Dict[str, Any]],
|
|
graph_references: List[Dict[str, Any]]
|
|
) -> List[str]:
|
|
"""
|
|
Find vector chunks that have no corresponding graph Document node.
|
|
|
|
Used for bidirectional orphan detection - vectors without graph representation.
|
|
|
|
Args:
|
|
chunk_references: List from get_all_chunk_references()
|
|
graph_references: List from GraphService.get_all_document_references()
|
|
|
|
Returns:
|
|
List of orphan chunk IDs
|
|
"""
|
|
# Build sets of IDs that have graph nodes
|
|
graph_page_ids = {
|
|
ref.get("page_id") for ref in graph_references
|
|
if ref.get("doc_type") == "wiki" and ref.get("page_id")
|
|
}
|
|
graph_doc_ids = {
|
|
ref.get("document_id") for ref in graph_references
|
|
if ref.get("doc_type") != "wiki" and ref.get("document_id")
|
|
}
|
|
|
|
# Find chunks with no graph node
|
|
orphan_ids = []
|
|
for chunk in chunk_references:
|
|
doc_type = chunk.get("doc_type", "wiki")
|
|
|
|
if doc_type == "wiki":
|
|
page_id = chunk.get("page_id")
|
|
if page_id and page_id not in graph_page_ids:
|
|
orphan_ids.append(chunk["chunk_id"])
|
|
else:
|
|
document_id = chunk.get("document_id")
|
|
if document_id and document_id not in graph_doc_ids:
|
|
orphan_ids.append(chunk["chunk_id"])
|
|
|
|
logger.info(f"Found {len(orphan_ids)} vector chunks without graph nodes")
|
|
return orphan_ids
|