diff --git a/src/routers/maintenance.py b/src/routers/maintenance.py index e8855bb..ba1a9e1 100644 --- a/src/routers/maintenance.py +++ b/src/routers/maintenance.py @@ -19,7 +19,7 @@ from src.services.graph_service import GraphService from src.services.volatile_service import VolatileCacheService from src.core.dependencies import ( VectorServiceDep, GraphServiceDep, WikiJSDep, RedisDep, - QdrantDep, OllamaDep, verify_api_key + QdrantDep, OllamaDep, PaperlessDep, verify_api_key ) from src.config import get_settings from src.core.multi_tenancy import DEFAULT_USER @@ -232,6 +232,18 @@ class TestDataCleanupResponse(BaseModel): duration_ms: float +class PaperlessCleanupResponse(BaseModel): + """Response from Paperless orphan cleanup operation.""" + success: bool + dry_run: bool + paperless_ids_checked: int = Field(description="Total Paperless IDs found in indexes") + orphans_found: int = Field(description="Documents deleted from Paperless but still indexed") + orphan_ids: List[int] = Field(default_factory=list, description="Paperless IDs that are orphans") + vector_chunks_deleted: int = Field(description="Vector chunks removed") + graph_nodes_deleted: int = Field(description="Graph Document nodes removed") + duration_ms: float + + # Test data path patterns - restricted to test user namespace only # These are the only paths that can be cleaned up for safety TEST_USER_PATH_PREFIXES = [ @@ -675,6 +687,106 @@ async def cleanup_test_data( raise HTTPException(status_code=500, detail=str(e)) +@router.post("/cleanup/paperless", response_model=PaperlessCleanupResponse) +async def cleanup_paperless_orphans( + user: str = Query(..., description="User identifier"), + dry_run: bool = Query(default=True, description="Preview only, don't delete"), + vector_service: VectorServiceDep = None, + graph_service: GraphServiceDep = None, + paperless: PaperlessDep = None, + api_key: str = Depends(verify_api_key) +): + """ + Find and clean up Paperless document orphans. + + Detects documents that were indexed in Library Desk but have since been + deleted from Paperless-ngx. Removes orphaned vectors and graph nodes. + + **Use dry_run=true (default) to preview what would be deleted.** + + **Scheduler Integration:** + ```json + { + "task_name": "paperless_orphan_cleanup", + "schedule": "0 5 * * *", + "endpoint": "POST /maintenance/cleanup/paperless?user=jpmschweitzer&dry_run=false", + "description": "Daily cleanup of orphaned Paperless documents" + } + ``` + """ + start_time = time.time() + + try: + settings = get_settings() + if not settings.paperless_token: + raise HTTPException(status_code=503, detail="Paperless not configured") + + # Get all document chunks from vectors with doc_type="document" + chunk_refs = await vector_service.get_all_chunk_references(user) + doc_chunks = [ref for ref in chunk_refs if ref.get("doc_type") == "document"] + + # Extract unique paperless_ids + paperless_ids = list(set( + ref.get("paperless_id") for ref in doc_chunks + if ref.get("paperless_id") + )) + + logger.info(f"Found {len(paperless_ids)} unique Paperless IDs in indexes") + + # Check each against Paperless API + orphan_ids = [] + for pid in paperless_ids: + try: + doc = await paperless.get_document(pid) + if doc is None: + orphan_ids.append(pid) + except Exception as e: + # Document not found or API error - treat as orphan + logger.debug(f"Paperless document {pid} not found: {e}") + orphan_ids.append(pid) + + logger.info(f"Found {len(orphan_ids)} orphaned Paperless documents") + + # Delete orphans if not dry run + vectors_deleted = 0 + graph_deleted = 0 + + if not dry_run and orphan_ids: + for pid in orphan_ids: + try: + # Delete vector chunks for this paperless_id + chunks_removed = await vector_service.delete_paperless_document_chunks(pid, user) + vectors_deleted += chunks_removed + + # Delete graph node for this paperless_id + graph_removed = await graph_service.delete_paperless_document(pid, user) + graph_deleted += graph_removed + + logger.info(f"Cleaned up orphaned Paperless document {pid}: {chunks_removed} chunks, {graph_removed} nodes") + + except Exception as e: + logger.error(f"Failed to cleanup Paperless document {pid}: {e}") + + duration_ms = (time.time() - start_time) * 1000 + + return PaperlessCleanupResponse( + success=True, + dry_run=dry_run, + paperless_ids_checked=len(paperless_ids), + orphans_found=len(orphan_ids), + orphan_ids=orphan_ids, + vector_chunks_deleted=vectors_deleted, + graph_nodes_deleted=graph_deleted, + duration_ms=duration_ms + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Paperless orphan cleanup failed: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + @router.get("/health", response_model=HealthCheckResponse) async def maintenance_health( user: str = Query(..., description="User identifier"), diff --git a/src/services/graph_service.py b/src/services/graph_service.py index 15378c6..e0c1087 100644 --- a/src/services/graph_service.py +++ b/src/services/graph_service.py @@ -1306,6 +1306,48 @@ Feel free to expand it with more details! logger.error(f"Failed to delete document {document_id} from graph: {e}", exc_info=True) return 0 + async def delete_paperless_document( + self, + paperless_id: int, + user: str + ) -> int: + """ + Delete a Paperless document node and all its relationships. + + Args: + paperless_id: Paperless-ngx document ID + user: User identifier + + Returns: + Number of nodes deleted (1 if successful, 0 if not found) + """ + user_doc_label = get_neo4j_user_label(user) + + delete_query = f""" + MATCH (d:{user_doc_label}:Document {{paperless_id: $paperless_id}}) + DETACH DELETE d + RETURN count(d) as deleted_count + """ + + try: + result = await self.neo4j.execute_query( + delete_query, + {"paperless_id": paperless_id} + ) + + deleted_count = result[0]["deleted_count"] if result else 0 + + if deleted_count > 0: + logger.info(f"Deleted Document node for Paperless document {paperless_id}") + else: + logger.debug(f"No Document node found for Paperless document {paperless_id}") + + return deleted_count + + except Exception as e: + logger.error(f"Failed to delete Paperless document {paperless_id} from graph: {e}", exc_info=True) + return 0 + async def delete_collection_node( self, collection_id: str, diff --git a/src/services/vector_service.py b/src/services/vector_service.py index f7413b1..f78019f 100644 --- a/src/services/vector_service.py +++ b/src/services/vector_service.py @@ -389,6 +389,39 @@ class VectorService: logger.error(f"Failed to delete chunks for document {document_id}: {e}", exc_info=True) return 0 + async def delete_paperless_document_chunks( + self, + paperless_id: int, + user: str + ) -> int: + """ + Delete all chunks for a Paperless document. + + Args: + paperless_id: Paperless-ngx document ID + user: User identifier + + Returns: + Number of chunks deleted + """ + collection_name = get_qdrant_collection_name(user) + + try: + deleted_count = await self.qdrant.delete_by_filter( + collection_name=collection_name, + filter_conditions={ + "doc_type": "document", + "paperless_id": paperless_id + } + ) + + logger.info(f"Deleted chunks for Paperless document {paperless_id}") + return deleted_count + + except Exception as e: + logger.error(f"Failed to delete chunks for Paperless document {paperless_id}: {e}", exc_info=True) + return 0 + async def delete_collection_chunks( self, collection_id: str,