""" Document storage router for Library Desk API. Event-driven integration with Paperless-ngx: - Webhook receiver triggers indexing after Paperless virus scan passes - Upload endpoint sends files to Paperless for processing - Search across indexed documents """ from fastapi import APIRouter, HTTPException, Depends, Query, UploadFile, File, Request from typing import Optional import logging import time from src.models.document import ( PaperlessWebhookPayload, WebhookResponse, DocumentUploadRequest, DocumentUploadResponse, DocumentSearchRequest, DocumentSearchResponse, DocumentStoreHealth, ) from src.core.dependencies import ( verify_api_key, PaperlessDep, QdrantDep, OllamaDep, Neo4jDep, WikiJSDep, ) from src.core.dependencies import RequiredUserQuery from src.config import get_settings logger = logging.getLogger(__name__) router = APIRouter(prefix="/documents", tags=["Documents"]) # ============================================================================= # Webhook Endpoint (primary integration - event-driven) # ============================================================================= @router.post("/webhook", response_model=WebhookResponse) async def receive_webhook( payload: PaperlessWebhookPayload, paperless: PaperlessDep, qdrant: QdrantDep, ollama: OllamaDep, neo4j: Neo4jDep, wiki: WikiJSDep, user: RequiredUserQuery, ): """ Receive webhook events from Paperless-ngx. This is the primary integration point. Configure Paperless workflow: 1. Trigger: Document Added (after consumption completes) 2. Condition: Document passed virus scan (ClamAV in Paperless) 3. Action: Webhook POST to this endpoint Library Desk indexes the document into vectors and graph. """ from src.services.document_sync_service import DocumentSyncService doc_id = payload.document_id logger.info(f"Webhook received: document_id={doc_id}, title={payload.title}") settings = get_settings() if not settings.document_store_enabled: return WebhookResponse( document_id=doc_id, status="skipped", indexed=False, message="Document store is disabled" ) try: sync_service = DocumentSyncService( paperless_client=paperless, qdrant_client=qdrant, ollama_client=ollama, neo4j_client=neo4j, wiki_client=wiki, settings=settings ) # Fetch content from Paperless (template only provides doc_url and title) result = await sync_service.index_document( document_id=doc_id, user=user, ) return WebhookResponse( document_id=doc_id, status="indexed" if result.success else "failed", indexed=result.success, message=result.error if not result.success else f"Indexed: {result.title}" ) except Exception as e: logger.error(f"Webhook processing failed for document {doc_id}: {e}", exc_info=True) return WebhookResponse( document_id=doc_id, status="error", indexed=False, message=str(e) ) # ============================================================================= # Debug Capture Endpoint # ============================================================================= @router.post("/webhook-capture") async def capture_webhook(request: Request): """Capture raw webhook payload for debugging.""" import json from pathlib import Path from datetime import datetime # Get raw body body = await request.body() headers = dict(request.headers) query_params = dict(request.query_params) # Build capture data capture = { "timestamp": datetime.now().isoformat(), "method": request.method, "url": str(request.url), "query_params": query_params, "headers": headers, "content_type": headers.get("content-type", "unknown"), "body_raw": body.decode("utf-8", errors="replace"), } # Try to parse as JSON try: capture["body_json"] = json.loads(body) except: capture["body_json"] = None # Write to file capture_file = Path("logs/webhook_capture.json") capture_file.parent.mkdir(exist_ok=True) with open(capture_file, "w") as f: json.dump(capture, f, indent=2, default=str) logger.info(f"Captured webhook: {capture['body_raw'][:200]}") return {"status": "captured", "file": str(capture_file)} # ============================================================================= # Simple Webhook (URL parameters only) # ============================================================================= @router.post("/webhook-simple", response_model=WebhookResponse) async def receive_webhook_simple( user: RequiredUserQuery, doc_url: str = Query(..., description="Paperless document URL containing ID"), title: str = Query(default="", description="Document title"), paperless: PaperlessDep = None, qdrant: QdrantDep = None, ollama: OllamaDep = None, neo4j: Neo4jDep = None, wiki: WikiJSDep = None, ): """ Simple webhook endpoint accepting URL parameters. Used when Paperless Jinja templates don't work with JSON body. URL format: /webhook-simple?doc_url=http://...&title=...&user=... """ from src.services.document_sync_service import DocumentSyncService import re # Extract document ID from URL match = re.search(r'/documents/(\d+)/?', doc_url) if not match: return WebhookResponse( document_id=0, status="error", indexed=False, message=f"Cannot extract document ID from URL: {doc_url}" ) doc_id = int(match.group(1)) logger.info(f"Webhook-simple received: document_id={doc_id}, title={title}") settings = get_settings() if not settings.document_store_enabled: return WebhookResponse( document_id=doc_id, status="skipped", indexed=False, message="Document store is disabled" ) try: sync_service = DocumentSyncService( paperless_client=paperless, qdrant_client=qdrant, ollama_client=ollama, neo4j_client=neo4j, wiki_client=wiki, settings=settings ) result = await sync_service.index_document( document_id=doc_id, user=user, ) return WebhookResponse( document_id=doc_id, status="indexed" if result.success else "failed", indexed=result.success, message=result.error if not result.success else f"Indexed: {result.title}" ) except Exception as e: logger.error(f"Webhook-simple failed for document {doc_id}: {e}", exc_info=True) return WebhookResponse( document_id=doc_id, status="error", indexed=False, message=str(e) ) # ============================================================================= # Upload Endpoints # ============================================================================= @router.post("/upload", response_model=DocumentUploadResponse) async def upload_document( file: UploadFile = File(...), title: Optional[str] = Query(None, description="Document title"), collection: Optional[str] = Query(None, description="Collection name"), paperless: PaperlessDep = None, api_key: str = Depends(verify_api_key), ): """ Upload a document to Paperless-ngx. Paperless handles virus scanning. If clean, Paperless webhook triggers indexing back to Library Desk. """ settings = get_settings() if not settings.document_store_enabled: raise HTTPException(status_code=503, detail="Document store is disabled") content = await file.read() filename = file.filename or "document" custom_fields = [] if collection: custom_fields.append({"field": "collection", "value": collection}) try: task_id = await paperless.upload_document( file_content=content, filename=filename, title=title, custom_fields=custom_fields if custom_fields else None, ) return DocumentUploadResponse( task_id=task_id, filename=filename, message=f"Uploaded to Paperless, task {task_id}. Indexing via webhook after scan." ) except Exception as e: logger.error(f"Upload failed for '{filename}': {e}") raise HTTPException(status_code=500, detail=f"Upload failed: {e}") @router.post("/upload-url", response_model=DocumentUploadResponse) async def upload_from_url( request: DocumentUploadRequest, paperless: PaperlessDep = None, api_key: str = Depends(verify_api_key), ): """ Download document from URL and upload to Paperless-ngx. Used by HybridRAG to save discovered PDFs. Paperless scans and webhooks back for indexing. """ import httpx settings = get_settings() if not settings.document_store_enabled: raise HTTPException(status_code=503, detail="Document store is disabled") if not request.url: raise HTTPException(status_code=400, detail="URL is required") try: async with httpx.AsyncClient(timeout=60.0) as client: response = await client.get(request.url, follow_redirects=True) response.raise_for_status() content = response.content filename = request.url.split("/")[-1].split("?")[0] or "document" except Exception as e: logger.error(f"Download failed from {request.url}: {e}") raise HTTPException(status_code=400, detail=f"Download failed: {e}") try: custom_fields = [{"field": "source_url", "value": request.url}] if request.collection: custom_fields.append({"field": "collection", "value": request.collection}) task_id = await paperless.upload_document( file_content=content, filename=filename, title=request.title, custom_fields=custom_fields, ) return DocumentUploadResponse( task_id=task_id, filename=filename, message=f"Uploaded from URL, task {task_id}. Indexing via webhook after scan." ) except Exception as e: logger.error(f"Upload failed for URL '{request.url}': {e}") raise HTTPException(status_code=500, detail=f"Upload failed: {e}") # ============================================================================= # Search # ============================================================================= @router.post("/search", response_model=DocumentSearchResponse) async def search_documents( request: DocumentSearchRequest, qdrant: QdrantDep, ollama: OllamaDep, user: RequiredUserQuery, api_key: str = Depends(verify_api_key), ): """ Semantic search across indexed documents. """ from src.services.vector_service import VectorService from src.core.dependencies import get_wikijs_client from src.models.document import DocumentSearchHit settings = get_settings() if not settings.document_store_enabled: raise HTTPException(status_code=503, detail="Document store is disabled") start_time = time.time() try: wiki = get_wikijs_client() vector_service = VectorService(qdrant, wiki, ollama) results = await vector_service.search( query=request.query, user=user, limit=request.limit, score_threshold=0.5, doc_type="document" ) hits = [] for result in results.get("results", []): hits.append(DocumentSearchHit( paperless_id=result.get("metadata", {}).get("paperless_id", 0), title=result.get("title", ""), score=result.get("score", 0.0), highlights=result.get("chunk_text", "")[:200] if request.include_content else None, collection=result.get("metadata", {}).get("collection"), document_type=result.get("metadata", {}).get("document_type"), content_preview=result.get("chunk_text", "")[:500] if request.include_content else None, )) return DocumentSearchResponse( query=request.query, hits=hits, total=len(hits), duration_ms=int((time.time() - start_time) * 1000) ) except Exception as e: logger.error(f"Document search failed: {e}", exc_info=True) raise HTTPException(status_code=500, detail=str(e)) # ============================================================================= # Health # ============================================================================= @router.get("/health", response_model=DocumentStoreHealth) async def document_store_health(paperless: PaperlessDep): """Check Paperless-ngx connectivity.""" settings = get_settings() paperless_healthy = False if settings.paperless_token: try: paperless_healthy = await paperless.health_check() except Exception as e: logger.error(f"Paperless health check failed: {e}") return DocumentStoreHealth( paperless_healthy=paperless_healthy, paperless_version="connected" if paperless_healthy else None, total_documents=None, indexed_documents=None )