perf: batch page embeddings via /api/embed and upsert before pruning stale points
- embed_batch now issues one batched /api/embed request (the old loop made one /api/embeddings round-trip per chunk) with a per-text fallback that preserves None-for-failed semantics - update_from_page embeds all chunks in that single call and stores them in one Qdrant batch upsert (upsert_points) - reindex order reversed: upsert new points first, then prune stale ids (deterministic uuid5 ids make overwrite safe) so a mid-way failure no longer leaves the page with zero vectors - VectorUpdateSummary gains status (success/partial/failed) and chunks_skipped; all-embeddings-failed keeps old vectors and reports failure instead of success=True Measured on a 7-chunk page ingest (local server, llm_tester): ~375ms -> ~181ms median over 3 runs. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
This commit is contained in:
Claude Fable 5
@@ -154,50 +154,79 @@ class VectorService:
|
||||
chunks = self._chunk_text(content)
|
||||
logger.info(f"Split page {page_id} into {len(chunks)} chunks")
|
||||
|
||||
# Delete existing chunks for this page
|
||||
deleted_count = await self.qdrant.delete_by_filter(
|
||||
collection_name=collection_name,
|
||||
filter_conditions={"page_id": page_id}
|
||||
)
|
||||
# Generate ALL embeddings in one batched /api/embed call
|
||||
# (previously one sequential Ollama round-trip per chunk)
|
||||
embeddings = await self.ollama.embed_batch(chunks)
|
||||
|
||||
# Generate embeddings and upsert chunks
|
||||
chunks_created = 0
|
||||
for idx, chunk_text in enumerate(chunks):
|
||||
# Generate deterministic UUID from page_id and chunk_index
|
||||
chunk_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, f"page_{page_id}_chunk_{idx}"))
|
||||
|
||||
# Generate embedding
|
||||
embedding = await self.ollama.embed(chunk_text)
|
||||
# Build points; deterministic uuid5 IDs mean re-upserting the
|
||||
# same page overwrites its previous chunks in place.
|
||||
points = []
|
||||
chunks_skipped = 0
|
||||
embedding_dim = 768
|
||||
for idx, (chunk_text, embedding) in enumerate(zip(chunks, embeddings)):
|
||||
if not embedding:
|
||||
logger.error(f"Failed to generate embedding for chunk {chunk_id}")
|
||||
chunks_skipped += 1
|
||||
logger.error(f"Failed to generate embedding for page {page_id} chunk {idx}")
|
||||
continue
|
||||
|
||||
# Prepare metadata
|
||||
metadata = {
|
||||
"page_id": page_id,
|
||||
"page_title": title,
|
||||
"page_path": path,
|
||||
"chunk_index": idx,
|
||||
"chunk_text": chunk_text,
|
||||
"user": user
|
||||
}
|
||||
embedding_dim = len(embedding)
|
||||
chunk_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, f"page_{page_id}_chunk_{idx}"))
|
||||
points.append({
|
||||
"id": chunk_id,
|
||||
"vector": embedding,
|
||||
"payload": {
|
||||
"page_id": page_id,
|
||||
"page_title": title,
|
||||
"page_path": path,
|
||||
"chunk_index": idx,
|
||||
"chunk_text": chunk_text,
|
||||
"user": user
|
||||
}
|
||||
})
|
||||
|
||||
# Upsert to Qdrant
|
||||
success = await self.qdrant.upsert_vector(
|
||||
# Upsert BEFORE deleting stale points. The old order (delete all,
|
||||
# then embed+upsert one by one) left the page with ZERO vectors if
|
||||
# anything failed mid-way; now old vectors survive until the new
|
||||
# ones are safely stored.
|
||||
chunks_created = 0
|
||||
if points:
|
||||
chunks_created = await self.qdrant.upsert_points(
|
||||
collection_name=collection_name,
|
||||
vector_id=chunk_id,
|
||||
vector=embedding,
|
||||
payload=metadata
|
||||
points=points
|
||||
)
|
||||
|
||||
if success:
|
||||
chunks_created += 1
|
||||
# Prune stale points from a previous version of the page (chunk
|
||||
# indexes beyond the new count, or indexes whose new embedding
|
||||
# failed). Only prune if the new upsert actually stored points —
|
||||
# a fully failed embedding pass must not wipe the old vectors.
|
||||
deleted_count = 0
|
||||
if points:
|
||||
new_ids = {p["id"] for p in points}
|
||||
existing = await self.qdrant.scroll_all_points(
|
||||
collection_name=collection_name,
|
||||
filter_conditions={"page_id": page_id},
|
||||
with_payload=False
|
||||
)
|
||||
stale_ids = [pt["id"] for pt in existing if pt["id"] not in new_ids]
|
||||
if stale_ids:
|
||||
deleted_count = await self.qdrant.delete_by_ids(
|
||||
collection_name=collection_name,
|
||||
point_ids=stale_ids
|
||||
)
|
||||
|
||||
processing_time_ms = (time.time() - start_time) * 1000
|
||||
|
||||
if chunks_created == 0:
|
||||
status = "failed"
|
||||
elif chunks_skipped > 0:
|
||||
status = "partial"
|
||||
else:
|
||||
status = "success"
|
||||
|
||||
logger.info(
|
||||
f"Updated vectors for page {page_id}: "
|
||||
f"{chunks_created} chunks created, {deleted_count} old chunks deleted"
|
||||
f"{chunks_created} chunks created, {chunks_skipped} skipped, "
|
||||
f"{deleted_count} stale chunks deleted ({status})"
|
||||
)
|
||||
|
||||
return VectorUpdateSummary(
|
||||
@@ -205,10 +234,16 @@ class VectorService:
|
||||
page_title=title,
|
||||
chunks_created=chunks_created,
|
||||
chunks_deleted=deleted_count,
|
||||
chunks_skipped=chunks_skipped,
|
||||
total_chunks=chunks_created,
|
||||
embedding_dim=len(embedding) if embedding else 768,
|
||||
embedding_dim=embedding_dim,
|
||||
processing_time_ms=processing_time_ms,
|
||||
success=True
|
||||
success=chunks_created > 0,
|
||||
status=status,
|
||||
error_message=(
|
||||
f"{chunks_skipped}/{len(chunks)} chunk embeddings failed"
|
||||
if chunks_skipped else None
|
||||
)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user