feat: add Paperless-ngx document storage integration
- Add /documents router with webhook, upload, search, health endpoints - Create DocumentSyncService for indexing documents to vectors/graph - Add PaperlessClient for REST API integration - Configure dependency injection for Paperless client - Add document models for webhook payloads and responses - Event-driven architecture via Paperless workflow webhooks 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.5
@@ -0,0 +1,274 @@
|
||||
"""
|
||||
Document sync service for Library Desk.
|
||||
|
||||
Handles indexing of Paperless-ngx documents into vectors and graph.
|
||||
Called by webhook when Paperless completes document processing.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import hashlib
|
||||
import uuid
|
||||
from typing import Optional, List
|
||||
from dataclasses import dataclass
|
||||
|
||||
from src.clients.paperless_client import PaperlessClient
|
||||
from src.clients.qdrant_client import QdrantClientWrapper
|
||||
from src.clients.ollama_client import OllamaClient
|
||||
from src.clients.neo4j_client import Neo4jClient
|
||||
from src.clients.wikijs_client import WikiJSClient
|
||||
from src.core.multi_tenancy import get_qdrant_collection_name
|
||||
from src.config import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class IndexResult:
|
||||
"""Result of indexing a single document."""
|
||||
success: bool
|
||||
document_id: int
|
||||
title: str = ""
|
||||
chunks_created: int = 0
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class DocumentSyncService:
|
||||
"""
|
||||
Service for syncing Paperless documents to Library Desk indexes.
|
||||
|
||||
Handles:
|
||||
- Fetching document content from Paperless API
|
||||
- Chunking and embedding into Qdrant
|
||||
- Creating graph nodes in Neo4j
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
paperless_client: PaperlessClient,
|
||||
qdrant_client: QdrantClientWrapper,
|
||||
ollama_client: OllamaClient,
|
||||
neo4j_client: Neo4jClient,
|
||||
wiki_client: WikiJSClient,
|
||||
settings: Settings,
|
||||
chunk_size: int = 500,
|
||||
chunk_overlap: int = 50
|
||||
):
|
||||
self.paperless = paperless_client
|
||||
self.qdrant = qdrant_client
|
||||
self.ollama = ollama_client
|
||||
self.neo4j = neo4j_client
|
||||
self.wiki = wiki_client
|
||||
self.settings = settings
|
||||
self.chunk_size = chunk_size
|
||||
self.chunk_overlap = chunk_overlap
|
||||
|
||||
def _chunk_text(self, text: str) -> List[str]:
|
||||
"""Chunk text into overlapping segments."""
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
words = text.split()
|
||||
|
||||
if len(words) <= self.chunk_size:
|
||||
return [text] if text else []
|
||||
|
||||
chunks = []
|
||||
start = 0
|
||||
|
||||
while start < len(words):
|
||||
end = start + self.chunk_size
|
||||
chunk_words = words[start:end]
|
||||
chunks.append(' '.join(chunk_words))
|
||||
start = end - self.chunk_overlap
|
||||
|
||||
return chunks
|
||||
|
||||
async def index_document(
|
||||
self,
|
||||
document_id: int,
|
||||
user: str,
|
||||
) -> IndexResult:
|
||||
"""
|
||||
Index a single document from Paperless into vectors and graph.
|
||||
|
||||
Args:
|
||||
document_id: Paperless document ID
|
||||
user: User identifier for multi-tenancy
|
||||
|
||||
Returns:
|
||||
IndexResult with success status and details
|
||||
"""
|
||||
logger.info(f"Indexing document {document_id} for user {user}")
|
||||
|
||||
try:
|
||||
# Fetch document from Paperless
|
||||
doc = await self.paperless.get_document(document_id)
|
||||
if not doc:
|
||||
return IndexResult(
|
||||
success=False,
|
||||
document_id=document_id,
|
||||
error="Document not found in Paperless"
|
||||
)
|
||||
|
||||
title = doc.title
|
||||
content = doc.content or ""
|
||||
|
||||
if not content.strip():
|
||||
logger.warning(f"Document {document_id} has no text content")
|
||||
return IndexResult(
|
||||
success=True,
|
||||
document_id=document_id,
|
||||
title=title,
|
||||
chunks_created=0,
|
||||
error="No text content (possibly image/video only)"
|
||||
)
|
||||
|
||||
# Index vectors
|
||||
chunks_created = await self._index_vectors(
|
||||
document_id=document_id,
|
||||
title=title,
|
||||
content=content,
|
||||
user=user,
|
||||
metadata={
|
||||
"paperless_id": document_id,
|
||||
"original_filename": doc.original_file_name,
|
||||
"correspondent": doc.correspondent,
|
||||
"document_type": doc.document_type,
|
||||
"tags": doc.tags,
|
||||
}
|
||||
)
|
||||
|
||||
# Index graph node
|
||||
await self._index_graph(
|
||||
document_id=document_id,
|
||||
title=title,
|
||||
content=content,
|
||||
user=user,
|
||||
)
|
||||
|
||||
# Mark as indexed in Paperless (optional - if custom field exists)
|
||||
try:
|
||||
await self._mark_indexed(document_id)
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not mark document as indexed: {e}")
|
||||
|
||||
logger.info(f"Successfully indexed document {document_id}: {chunks_created} chunks")
|
||||
|
||||
return IndexResult(
|
||||
success=True,
|
||||
document_id=document_id,
|
||||
title=title,
|
||||
chunks_created=chunks_created
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to index document {document_id}: {e}", exc_info=True)
|
||||
return IndexResult(
|
||||
success=False,
|
||||
document_id=document_id,
|
||||
error=str(e)
|
||||
)
|
||||
|
||||
async def _index_vectors(
|
||||
self,
|
||||
document_id: int,
|
||||
title: str,
|
||||
content: str,
|
||||
user: str,
|
||||
metadata: dict,
|
||||
) -> int:
|
||||
"""Create vector embeddings for document content."""
|
||||
collection = get_qdrant_collection_name(user)
|
||||
self.qdrant.ensure_collection(collection)
|
||||
|
||||
# Delete existing chunks for this document
|
||||
try:
|
||||
self.qdrant.client.delete(
|
||||
collection_name=collection,
|
||||
points_selector={
|
||||
"filter": {
|
||||
"must": [
|
||||
{"key": "doc_type", "match": {"value": "document"}},
|
||||
{"key": "paperless_id", "match": {"value": document_id}},
|
||||
]
|
||||
}
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"No existing chunks to delete: {e}")
|
||||
|
||||
# Chunk content
|
||||
chunks = self._chunk_text(content)
|
||||
if not chunks:
|
||||
return 0
|
||||
|
||||
# Generate embeddings
|
||||
embeddings = await self.ollama.embed_batch(chunks)
|
||||
|
||||
# Build points
|
||||
points = []
|
||||
for i, (chunk, embedding) in enumerate(zip(chunks, embeddings)):
|
||||
point_id = str(uuid.uuid4())
|
||||
content_hash = hashlib.md5(chunk.encode()).hexdigest()
|
||||
|
||||
points.append({
|
||||
"id": point_id,
|
||||
"vector": embedding,
|
||||
"payload": {
|
||||
"doc_type": "document",
|
||||
"paperless_id": document_id,
|
||||
"title": title,
|
||||
"chunk_text": chunk,
|
||||
"chunk_index": i,
|
||||
"content_hash": content_hash,
|
||||
**metadata
|
||||
}
|
||||
})
|
||||
|
||||
# Upsert to Qdrant
|
||||
if points:
|
||||
self.qdrant.client.upsert(
|
||||
collection_name=collection,
|
||||
points=points
|
||||
)
|
||||
|
||||
return len(points)
|
||||
|
||||
async def _index_graph(
|
||||
self,
|
||||
document_id: int,
|
||||
title: str,
|
||||
content: str,
|
||||
user: str,
|
||||
):
|
||||
"""Create graph node for document."""
|
||||
# Create Document node in Neo4j
|
||||
query = """
|
||||
MERGE (d:Document {paperless_id: $paperless_id, user: $user})
|
||||
SET d.title = $title,
|
||||
d.doc_type = 'document',
|
||||
d.updated_at = datetime()
|
||||
RETURN d
|
||||
"""
|
||||
await self.neo4j.execute_query(
|
||||
query,
|
||||
{
|
||||
"paperless_id": document_id,
|
||||
"user": user,
|
||||
"title": title,
|
||||
}
|
||||
)
|
||||
|
||||
# TODO: Extract entities from content and create relationships
|
||||
# This could use the same entity extraction as wiki pages
|
||||
|
||||
async def _mark_indexed(self, document_id: int):
|
||||
"""Mark document as indexed in Paperless custom field."""
|
||||
# Try to update library_indexed custom field if it exists
|
||||
try:
|
||||
await self.paperless.update_document(
|
||||
document_id=document_id,
|
||||
custom_fields=[{"field": "library_indexed", "value": True}]
|
||||
)
|
||||
except Exception:
|
||||
# Field might not exist, that's OK
|
||||
pass
|
||||
Reference in New Issue
Block a user