fix: await ensure_collection and survive partial embed failures in document sync

- ensure_collection was called without await, so the coroutine never ran
  and fresh tenants had no collection when the upsert hit Qdrant
- a single None entry from embed_batch poisoned the point batch and
  aborted the whole document upsert; failed chunks are now skipped with
  a warning (all-failed raises and the IndexResult reports failure)
- raw client.delete/client.upsert calls now go through the async wrapper
  (delete_by_filter and the new batch upsert_points method)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
This commit is contained in:
2026-07-14 13:00:59 +02:00
co-authored by Claude Fable 5
parent 406143e7ae
commit c35d3c1fa9
4 changed files with 203 additions and 14 deletions
+30 -14
View File
@@ -194,19 +194,15 @@ class DocumentSyncService:
) -> int:
"""Create vector embeddings for document content."""
collection = get_qdrant_collection_name(user)
self.qdrant.ensure_collection(collection)
await self.qdrant.ensure_collection(collection)
# Delete existing chunks for this document
# Delete existing chunks for this document (via the async wrapper)
try:
await self.qdrant.client.delete(
await self.qdrant.delete_by_filter(
collection_name=collection,
points_selector={
"filter": {
"must": [
{"key": "doc_type", "match": {"value": "document"}},
{"key": "paperless_id", "match": {"value": document_id}},
]
}
filter_conditions={
"doc_type": "document",
"paperless_id": document_id,
}
)
except Exception as e:
@@ -217,12 +213,22 @@ class DocumentSyncService:
if not chunks:
return 0
# Generate embeddings
# Generate embeddings (embed_batch returns None for failed chunks)
embeddings = await self.ollama.embed_batch(chunks)
# Build points
# Build points, skipping chunks whose embedding failed. Previously a
# single None embedding poisoned the batch and aborted the whole
# document upsert.
points = []
skipped = 0
for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)):
if embedding is None:
skipped += 1
logger.warning(
f"Skipping chunk {i} of document {document_id}: embedding failed"
)
continue
point_id = str(uuid.uuid4())
content_hash = hashlib.md5(chunk.encode()).hexdigest()
@@ -240,9 +246,19 @@ class DocumentSyncService:
}
})
# Upsert to Qdrant
if skipped and not points:
raise RuntimeError(
f"All {skipped} chunk embeddings failed for document {document_id}"
)
if skipped:
logger.warning(
f"Document {document_id}: {skipped}/{len(chunks)} chunks skipped "
f"(embedding failures); indexing the remaining {len(points)}"
)
# Upsert to Qdrant in one batch via the async wrapper
if points:
await self.qdrant.client.upsert(
await self.qdrant.upsert_points(
collection_name=collection,
points=points
)