From 867de65354b2ca32f685100ece039d137f3d80cb Mon Sep 17 00:00:00 2001 From: Jeroen Schweitzer Date: Thu, 25 Dec 2025 16:37:26 +0100 Subject: [PATCH] fix: use field ID for Paperless custom field updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paperless API requires field ID (integer) not field name (string) when updating custom fields. Now looks up field ID by name before updating library_indexed custom field. Also includes webhook debugging endpoint for development. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .env.example | 2 +- src/models/document.py | 31 +++--- src/routers/documents.py | 142 ++++++++++++++++++++++++-- src/services/document_sync_service.py | 11 +- 4 files changed, 159 insertions(+), 27 deletions(-) diff --git a/.env.example b/.env.example index 4f75923..ffc0a6f 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,7 @@ QDRANT_PORT=6333 OLLAMA_URL=http://192.168.86.149:11434 SEARXNG_URL=http://192.168.86.149:8080 REDIS_HOST=192.168.86.149 -PAPERLESS_URL=http://192.168.86.149:8000 +PAPERLESS_URL=http://192.168.86.149:8091 OLLAMA_MODEL=mistral-nemo-large:latest OLLAMA_EMBEDDING_MODEL=nomic-embed-text diff --git a/src/models/document.py b/src/models/document.py index 23816e7..6aeae7e 100644 --- a/src/models/document.py +++ b/src/models/document.py @@ -86,22 +86,27 @@ class DocumentUploadResponse(BaseModel): class PaperlessWebhookPayload(BaseModel): - """Payload from Paperless-ngx webhook (include_document=true format).""" - id: int = Field(..., description="Paperless document ID") + """ + Payload from Paperless-ngx webhook. + + Supports Jinja template format: + - doc_url: Contains document ID in URL path (e.g., http://paperless:8000/documents/123/) + - title: Document title from {{ doc_title }} + """ + doc_url: str = Field(..., description="Paperless document URL containing ID") title: Optional[str] = Field(None, description="Document title") - content: Optional[str] = Field(None, description="Document text content") - correspondent: Optional[int] = Field(None, description="Correspondent ID") - document_type: Optional[int] = Field(None, description="Document type ID") - tags: List[int] = Field(default_factory=list, description="Tag IDs") - created: Optional[str] = Field(None, description="Created date") - modified: Optional[str] = Field(None, description="Modified datetime") - added: Optional[str] = Field(None, description="Added datetime") - original_file_name: Optional[str] = Field(None, description="Original filename") - owner: Optional[int] = Field(None, description="Owner user ID") - custom_fields: List[Dict[str, Any]] = Field(default_factory=list, description="Custom fields") class Config: - extra = "ignore" # Ignore extra fields from Paperless + extra = "ignore" # Ignore extra fields + + @property + def document_id(self) -> int: + """Extract document ID from doc_url.""" + import re + match = re.search(r'/documents/(\d+)/?', self.doc_url) + if match: + return int(match.group(1)) + raise ValueError(f"Cannot extract document ID from URL: {self.doc_url}") class WebhookResponse(BaseModel): diff --git a/src/routers/documents.py b/src/routers/documents.py index db65357..55ddf85 100644 --- a/src/routers/documents.py +++ b/src/routers/documents.py @@ -7,7 +7,7 @@ Event-driven integration with Paperless-ngx: - Search across indexed documents """ -from fastapi import APIRouter, HTTPException, Depends, Query, UploadFile, File +from fastapi import APIRouter, HTTPException, Depends, Query, UploadFile, File, Request from typing import Optional import logging import time @@ -64,12 +64,138 @@ async def receive_webhook( """ from src.services.document_sync_service import DocumentSyncService - logger.info(f"Webhook received: document_id={payload.id}, title={payload.title}") + doc_id = payload.document_id + logger.info(f"Webhook received: document_id={doc_id}, title={payload.title}") settings = get_settings() if not settings.document_store_enabled: return WebhookResponse( - document_id=payload.id, + document_id=doc_id, + status="skipped", + indexed=False, + message="Document store is disabled" + ) + + try: + sync_service = DocumentSyncService( + paperless_client=paperless, + qdrant_client=qdrant, + ollama_client=ollama, + neo4j_client=neo4j, + wiki_client=wiki, + settings=settings + ) + + # Fetch content from Paperless (template only provides doc_url and title) + result = await sync_service.index_document( + document_id=doc_id, + user=user, + ) + + return WebhookResponse( + document_id=doc_id, + status="indexed" if result.success else "failed", + indexed=result.success, + message=result.error if not result.success else f"Indexed: {result.title}" + ) + + except Exception as e: + logger.error(f"Webhook processing failed for document {doc_id}: {e}", exc_info=True) + return WebhookResponse( + document_id=doc_id, + status="error", + indexed=False, + message=str(e) + ) + + +# ============================================================================= +# Debug Capture Endpoint +# ============================================================================= + + +@router.post("/webhook-capture") +async def capture_webhook(request: Request): + """Capture raw webhook payload for debugging.""" + import json + from pathlib import Path + from datetime import datetime + + # Get raw body + body = await request.body() + headers = dict(request.headers) + query_params = dict(request.query_params) + + # Build capture data + capture = { + "timestamp": datetime.now().isoformat(), + "method": request.method, + "url": str(request.url), + "query_params": query_params, + "headers": headers, + "content_type": headers.get("content-type", "unknown"), + "body_raw": body.decode("utf-8", errors="replace"), + } + + # Try to parse as JSON + try: + capture["body_json"] = json.loads(body) + except: + capture["body_json"] = None + + # Write to file + capture_file = Path("logs/webhook_capture.json") + capture_file.parent.mkdir(exist_ok=True) + with open(capture_file, "w") as f: + json.dump(capture, f, indent=2, default=str) + + logger.info(f"Captured webhook: {capture['body_raw'][:200]}") + + return {"status": "captured", "file": str(capture_file)} + + +# ============================================================================= +# Simple Webhook (URL parameters only) +# ============================================================================= + + +@router.post("/webhook-simple", response_model=WebhookResponse) +async def receive_webhook_simple( + doc_url: str = Query(..., description="Paperless document URL containing ID"), + title: str = Query(default="", description="Document title"), + user: str = Query(default=DEFAULT_USER, description="User identifier"), + paperless: PaperlessDep = None, + qdrant: QdrantDep = None, + ollama: OllamaDep = None, + neo4j: Neo4jDep = None, + wiki: WikiJSDep = None, +): + """ + Simple webhook endpoint accepting URL parameters. + + Used when Paperless Jinja templates don't work with JSON body. + URL format: /webhook-simple?doc_url=http://...&title=...&user=... + """ + from src.services.document_sync_service import DocumentSyncService + import re + + # Extract document ID from URL + match = re.search(r'/documents/(\d+)/?', doc_url) + if not match: + return WebhookResponse( + document_id=0, + status="error", + indexed=False, + message=f"Cannot extract document ID from URL: {doc_url}" + ) + doc_id = int(match.group(1)) + + logger.info(f"Webhook-simple received: document_id={doc_id}, title={title}") + + settings = get_settings() + if not settings.document_store_enabled: + return WebhookResponse( + document_id=doc_id, status="skipped", indexed=False, message="Document store is disabled" @@ -86,23 +212,21 @@ async def receive_webhook( ) result = await sync_service.index_document( - document_id=payload.id, + document_id=doc_id, user=user, - content=payload.content, # Use content from webhook payload - title=payload.title, ) return WebhookResponse( - document_id=payload.id, + document_id=doc_id, status="indexed" if result.success else "failed", indexed=result.success, message=result.error if not result.success else f"Indexed: {result.title}" ) except Exception as e: - logger.error(f"Webhook processing failed for document {payload.id}: {e}", exc_info=True) + logger.error(f"Webhook-simple failed for document {doc_id}: {e}", exc_info=True) return WebhookResponse( - document_id=payload.id, + document_id=doc_id, status="error", indexed=False, message=str(e) diff --git a/src/services/document_sync_service.py b/src/services/document_sync_service.py index 7eb2aa8..88fb1e9 100644 --- a/src/services/document_sync_service.py +++ b/src/services/document_sync_service.py @@ -281,10 +281,13 @@ class DocumentSyncService: """Mark document as indexed in Paperless custom field.""" # Try to update library_indexed custom field if it exists try: - await self.paperless.update_document( - document_id=document_id, - custom_fields=[{"field": "library_indexed", "value": True}] - ) + # Look up field ID by name (Paperless requires ID, not name) + field = await self.paperless.get_custom_field_by_name("library_indexed") + if field: + await self.paperless.update_document( + document_id=document_id, + custom_fields=[{"field": field["id"], "value": True}] + ) except Exception: # Field might not exist, that's OK pass