release: v1.4.8 - Paperless orphan cleanup

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to Library Desk will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.8] - 2025-12-25
+
+### Added
+
+- **Paperless Orphan Cleanup** - `POST /maintenance/cleanup/paperless` endpoint
+  - Detects documents deleted from Paperless but still indexed in Library Desk
+  - Removes orphaned vectors and graph nodes
+  - Supports `dry_run=true` for preview mode
+
 ## [1.4.7] - 2025-12-25
 
 ### Fixed

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "library-desk"
-version = "1.4.7"
+version = "1.4.8"
 description = "Coordination service for The Library system - HybridRAG queries, document ingestion, entity extraction, and knowledge consolidation"
 readme = "README.md"
 requires-python = ">=3.12"

src/routers/maintenance.py

@@ -19,7 +19,7 @@ from src.services.graph_service import GraphService
 from src.services.volatile_service import VolatileCacheService
 from src.core.dependencies import (
     VectorServiceDep, GraphServiceDep, WikiJSDep, RedisDep,
-    QdrantDep, OllamaDep, verify_api_key
+    QdrantDep, OllamaDep, PaperlessDep, verify_api_key
 )
 from src.config import get_settings
 from src.core.multi_tenancy import DEFAULT_USER
@@ -232,6 +232,18 @@ class TestDataCleanupResponse(BaseModel):
     duration_ms: float
 
 
+class PaperlessCleanupResponse(BaseModel):
+    """Response from Paperless orphan cleanup operation."""
+    success: bool
+    dry_run: bool
+    paperless_ids_checked: int = Field(description="Total Paperless IDs found in indexes")
+    orphans_found: int = Field(description="Documents deleted from Paperless but still indexed")
+    orphan_ids: List[int] = Field(default_factory=list, description="Paperless IDs that are orphans")
+    vector_chunks_deleted: int = Field(description="Vector chunks removed")
+    graph_nodes_deleted: int = Field(description="Graph Document nodes removed")
+    duration_ms: float
+
+
 # Test data path patterns - restricted to test user namespace only
 # These are the only paths that can be cleaned up for safety
 TEST_USER_PATH_PREFIXES = [
@@ -675,6 +687,106 @@ async def cleanup_test_data(
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@router.post("/cleanup/paperless", response_model=PaperlessCleanupResponse)
+async def cleanup_paperless_orphans(
+    user: str = Query(..., description="User identifier"),
+    dry_run: bool = Query(default=True, description="Preview only, don't delete"),
+    vector_service: VectorServiceDep = None,
+    graph_service: GraphServiceDep = None,
+    paperless: PaperlessDep = None,
+    api_key: str = Depends(verify_api_key)
+):
+    """
+    Find and clean up Paperless document orphans.
+
+    Detects documents that were indexed in Library Desk but have since been
+    deleted from Paperless-ngx. Removes orphaned vectors and graph nodes.
+
+    **Use dry_run=true (default) to preview what would be deleted.**
+
+    **Scheduler Integration:**
+    ```json
+    {
+      "task_name": "paperless_orphan_cleanup",
+      "schedule": "0 5 * * *",
+      "endpoint": "POST /maintenance/cleanup/paperless?user=jpmschweitzer&dry_run=false",
+      "description": "Daily cleanup of orphaned Paperless documents"
+    }
+    ```
+    """
+    start_time = time.time()
+
+    try:
+        settings = get_settings()
+        if not settings.paperless_token:
+            raise HTTPException(status_code=503, detail="Paperless not configured")
+
+        # Get all document chunks from vectors with doc_type="document"
+        chunk_refs = await vector_service.get_all_chunk_references(user)
+        doc_chunks = [ref for ref in chunk_refs if ref.get("doc_type") == "document"]
+
+        # Extract unique paperless_ids
+        paperless_ids = list(set(
+            ref.get("paperless_id") for ref in doc_chunks
+            if ref.get("paperless_id")
+        ))
+
+        logger.info(f"Found {len(paperless_ids)} unique Paperless IDs in indexes")
+
+        # Check each against Paperless API
+        orphan_ids = []
+        for pid in paperless_ids:
+            try:
+                doc = await paperless.get_document(pid)
+                if doc is None:
+                    orphan_ids.append(pid)
+            except Exception as e:
+                # Document not found or API error - treat as orphan
+                logger.debug(f"Paperless document {pid} not found: {e}")
+                orphan_ids.append(pid)
+
+        logger.info(f"Found {len(orphan_ids)} orphaned Paperless documents")
+
+        # Delete orphans if not dry run
+        vectors_deleted = 0
+        graph_deleted = 0
+
+        if not dry_run and orphan_ids:
+            for pid in orphan_ids:
+                try:
+                    # Delete vector chunks for this paperless_id
+                    chunks_removed = await vector_service.delete_paperless_document_chunks(pid, user)
+                    vectors_deleted += chunks_removed
+
+                    # Delete graph node for this paperless_id
+                    graph_removed = await graph_service.delete_paperless_document(pid, user)
+                    graph_deleted += graph_removed
+
+                    logger.info(f"Cleaned up orphaned Paperless document {pid}: {chunks_removed} chunks, {graph_removed} nodes")
+
+                except Exception as e:
+                    logger.error(f"Failed to cleanup Paperless document {pid}: {e}")
+
+        duration_ms = (time.time() - start_time) * 1000
+
+        return PaperlessCleanupResponse(
+            success=True,
+            dry_run=dry_run,
+            paperless_ids_checked=len(paperless_ids),
+            orphans_found=len(orphan_ids),
+            orphan_ids=orphan_ids,
+            vector_chunks_deleted=vectors_deleted,
+            graph_nodes_deleted=graph_deleted,
+            duration_ms=duration_ms
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Paperless orphan cleanup failed: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.get("/health", response_model=HealthCheckResponse)
 async def maintenance_health(
     user: str = Query(..., description="User identifier"),

src/services/graph_service.py

@@ -1306,6 +1306,48 @@ Feel free to expand it with more details!
             logger.error(f"Failed to delete document {document_id} from graph: {e}", exc_info=True)
             return 0
 
+    async def delete_paperless_document(
+        self,
+        paperless_id: int,
+        user: str
+    ) -> int:
+        """
+        Delete a Paperless document node and all its relationships.
+
+        Args:
+            paperless_id: Paperless-ngx document ID
+            user: User identifier
+
+        Returns:
+            Number of nodes deleted (1 if successful, 0 if not found)
+        """
+        user_doc_label = get_neo4j_user_label(user)
+
+        delete_query = f"""
+        MATCH (d:{user_doc_label}:Document {{paperless_id: $paperless_id}})
+        DETACH DELETE d
+        RETURN count(d) as deleted_count
+        """
+
+        try:
+            result = await self.neo4j.execute_query(
+                delete_query,
+                {"paperless_id": paperless_id}
+            )
+
+            deleted_count = result[0]["deleted_count"] if result else 0
+
+            if deleted_count > 0:
+                logger.info(f"Deleted Document node for Paperless document {paperless_id}")
+            else:
+                logger.debug(f"No Document node found for Paperless document {paperless_id}")
+
+            return deleted_count
+
+        except Exception as e:
+            logger.error(f"Failed to delete Paperless document {paperless_id} from graph: {e}", exc_info=True)
+            return 0
+
     async def delete_collection_node(
         self,
         collection_id: str,

src/services/vector_service.py

@@ -389,6 +389,39 @@ class VectorService:
             logger.error(f"Failed to delete chunks for document {document_id}: {e}", exc_info=True)
             return 0
 
+    async def delete_paperless_document_chunks(
+        self,
+        paperless_id: int,
+        user: str
+    ) -> int:
+        """
+        Delete all chunks for a Paperless document.
+
+        Args:
+            paperless_id: Paperless-ngx document ID
+            user: User identifier
+
+        Returns:
+            Number of chunks deleted
+        """
+        collection_name = get_qdrant_collection_name(user)
+
+        try:
+            deleted_count = await self.qdrant.delete_by_filter(
+                collection_name=collection_name,
+                filter_conditions={
+                    "doc_type": "document",
+                    "paperless_id": paperless_id
+                }
+            )
+
+            logger.info(f"Deleted chunks for Paperless document {paperless_id}")
+            return deleted_count
+
+        except Exception as e:
+            logger.error(f"Failed to delete chunks for Paperless document {paperless_id}: {e}", exc_info=True)
+            return 0
+
     async def delete_collection_chunks(
         self,
         collection_id: str,