library-desk/src/routers/documents.py

"""
Document storage router for Library Desk API.

Event-driven integration with Paperless-ngx:
- Webhook receiver triggers indexing after Paperless virus scan passes
- Upload endpoint sends files to Paperless for processing
- Search across indexed documents
"""

from fastapi import APIRouter, HTTPException, Depends, Query, UploadFile, File, Request
from typing import Optional
import logging
import time

from src.models.document import (
    PaperlessWebhookPayload,
    WebhookResponse,
    DocumentUploadRequest,
    DocumentUploadResponse,
    DocumentSearchRequest,
    DocumentSearchResponse,
    DocumentStoreHealth,
)
from src.core.dependencies import (
    verify_api_key,
    PaperlessDep,
    QdrantDep,
    OllamaDep,
    Neo4jDep,
    WikiJSDep,
)
from src.core.multi_tenancy import DEFAULT_USER
from src.config import get_settings

logger = logging.getLogger(__name__)

router = APIRouter(prefix="/documents", tags=["Documents"])


# =============================================================================
# Webhook Endpoint (primary integration - event-driven)
# =============================================================================


@router.post("/webhook", response_model=WebhookResponse)
async def receive_webhook(
    payload: PaperlessWebhookPayload,
    paperless: PaperlessDep,
    qdrant: QdrantDep,
    ollama: OllamaDep,
    neo4j: Neo4jDep,
    wiki: WikiJSDep,
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
):
    """
    Receive webhook events from Paperless-ngx.

    This is the primary integration point. Configure Paperless workflow:
    1. Trigger: Document Added (after consumption completes)
    2. Condition: Document passed virus scan (ClamAV in Paperless)
    3. Action: Webhook POST to this endpoint

    Library Desk indexes the document into vectors and graph.
    """
    from src.services.document_sync_service import DocumentSyncService

    doc_id = payload.document_id
    logger.info(f"Webhook received: document_id={doc_id}, title={payload.title}")

    settings = get_settings()
    if not settings.document_store_enabled:
        return WebhookResponse(
            document_id=doc_id,
            status="skipped",
            indexed=False,
            message="Document store is disabled"
        )

    try:
        sync_service = DocumentSyncService(
            paperless_client=paperless,
            qdrant_client=qdrant,
            ollama_client=ollama,
            neo4j_client=neo4j,
            wiki_client=wiki,
            settings=settings
        )

        # Fetch content from Paperless (template only provides doc_url and title)
        result = await sync_service.index_document(
            document_id=doc_id,
            user=user,
        )

        return WebhookResponse(
            document_id=doc_id,
            status="indexed" if result.success else "failed",
            indexed=result.success,
            message=result.error if not result.success else f"Indexed: {result.title}"
        )

    except Exception as e:
        logger.error(f"Webhook processing failed for document {doc_id}: {e}", exc_info=True)
        return WebhookResponse(
            document_id=doc_id,
            status="error",
            indexed=False,
            message=str(e)
        )


# =============================================================================
# Debug Capture Endpoint
# =============================================================================


@router.post("/webhook-capture")
async def capture_webhook(request: Request):
    """Capture raw webhook payload for debugging."""
    import json
    from pathlib import Path
    from datetime import datetime

    # Get raw body
    body = await request.body()
    headers = dict(request.headers)
    query_params = dict(request.query_params)

    # Build capture data
    capture = {
        "timestamp": datetime.now().isoformat(),
        "method": request.method,
        "url": str(request.url),
        "query_params": query_params,
        "headers": headers,
        "content_type": headers.get("content-type", "unknown"),
        "body_raw": body.decode("utf-8", errors="replace"),
    }

    # Try to parse as JSON
    try:
        capture["body_json"] = json.loads(body)
    except:
        capture["body_json"] = None

    # Write to file
    capture_file = Path("logs/webhook_capture.json")
    capture_file.parent.mkdir(exist_ok=True)
    with open(capture_file, "w") as f:
        json.dump(capture, f, indent=2, default=str)

    logger.info(f"Captured webhook: {capture['body_raw'][:200]}")

    return {"status": "captured", "file": str(capture_file)}


# =============================================================================
# Simple Webhook (URL parameters only)
# =============================================================================


@router.post("/webhook-simple", response_model=WebhookResponse)
async def receive_webhook_simple(
    doc_url: str = Query(..., description="Paperless document URL containing ID"),
    title: str = Query(default="", description="Document title"),
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
    paperless: PaperlessDep = None,
    qdrant: QdrantDep = None,
    ollama: OllamaDep = None,
    neo4j: Neo4jDep = None,
    wiki: WikiJSDep = None,
):
    """
    Simple webhook endpoint accepting URL parameters.

    Used when Paperless Jinja templates don't work with JSON body.
    URL format: /webhook-simple?doc_url=http://...&title=...&user=...
    """
    from src.services.document_sync_service import DocumentSyncService
    import re

    # Extract document ID from URL
    match = re.search(r'/documents/(\d+)/?', doc_url)
    if not match:
        return WebhookResponse(
            document_id=0,
            status="error",
            indexed=False,
            message=f"Cannot extract document ID from URL: {doc_url}"
        )
    doc_id = int(match.group(1))

    logger.info(f"Webhook-simple received: document_id={doc_id}, title={title}")

    settings = get_settings()
    if not settings.document_store_enabled:
        return WebhookResponse(
            document_id=doc_id,
            status="skipped",
            indexed=False,
            message="Document store is disabled"
        )

    try:
        sync_service = DocumentSyncService(
            paperless_client=paperless,
            qdrant_client=qdrant,
            ollama_client=ollama,
            neo4j_client=neo4j,
            wiki_client=wiki,
            settings=settings
        )

        result = await sync_service.index_document(
            document_id=doc_id,
            user=user,
        )

        return WebhookResponse(
            document_id=doc_id,
            status="indexed" if result.success else "failed",
            indexed=result.success,
            message=result.error if not result.success else f"Indexed: {result.title}"
        )

    except Exception as e:
        logger.error(f"Webhook-simple failed for document {doc_id}: {e}", exc_info=True)
        return WebhookResponse(
            document_id=doc_id,
            status="error",
            indexed=False,
            message=str(e)
        )


# =============================================================================
# Upload Endpoints
# =============================================================================


@router.post("/upload", response_model=DocumentUploadResponse)
async def upload_document(
    file: UploadFile = File(...),
    title: Optional[str] = Query(None, description="Document title"),
    collection: Optional[str] = Query(None, description="Collection name"),
    paperless: PaperlessDep = None,
    api_key: str = Depends(verify_api_key),
):
    """
    Upload a document to Paperless-ngx.

    Paperless handles virus scanning. If clean, Paperless webhook
    triggers indexing back to Library Desk.
    """
    settings = get_settings()
    if not settings.document_store_enabled:
        raise HTTPException(status_code=503, detail="Document store is disabled")

    content = await file.read()
    filename = file.filename or "document"

    custom_fields = []
    if collection:
        custom_fields.append({"field": "collection", "value": collection})

    try:
        task_id = await paperless.upload_document(
            file_content=content,
            filename=filename,
            title=title,
            custom_fields=custom_fields if custom_fields else None,
        )

        return DocumentUploadResponse(
            task_id=task_id,
            filename=filename,
            message=f"Uploaded to Paperless, task {task_id}. Indexing via webhook after scan."
        )

    except Exception as e:
        logger.error(f"Upload failed for '{filename}': {e}")
        raise HTTPException(status_code=500, detail=f"Upload failed: {e}")


@router.post("/upload-url", response_model=DocumentUploadResponse)
async def upload_from_url(
    request: DocumentUploadRequest,
    paperless: PaperlessDep = None,
    api_key: str = Depends(verify_api_key),
):
    """
    Download document from URL and upload to Paperless-ngx.

    Used by HybridRAG to save discovered PDFs. Paperless scans and
    webhooks back for indexing.
    """
    import httpx

    settings = get_settings()
    if not settings.document_store_enabled:
        raise HTTPException(status_code=503, detail="Document store is disabled")

    if not request.url:
        raise HTTPException(status_code=400, detail="URL is required")

    try:
        async with httpx.AsyncClient(timeout=60.0) as client:
            response = await client.get(request.url, follow_redirects=True)
            response.raise_for_status()
            content = response.content
            filename = request.url.split("/")[-1].split("?")[0] or "document"

    except Exception as e:
        logger.error(f"Download failed from {request.url}: {e}")
        raise HTTPException(status_code=400, detail=f"Download failed: {e}")

    try:
        custom_fields = [{"field": "source_url", "value": request.url}]
        if request.collection:
            custom_fields.append({"field": "collection", "value": request.collection})

        task_id = await paperless.upload_document(
            file_content=content,
            filename=filename,
            title=request.title,
            custom_fields=custom_fields,
        )

        return DocumentUploadResponse(
            task_id=task_id,
            filename=filename,
            message=f"Uploaded from URL, task {task_id}. Indexing via webhook after scan."
        )

    except Exception as e:
        logger.error(f"Upload failed for URL '{request.url}': {e}")
        raise HTTPException(status_code=500, detail=f"Upload failed: {e}")


# =============================================================================
# Search
# =============================================================================


@router.post("/search", response_model=DocumentSearchResponse)
async def search_documents(
    request: DocumentSearchRequest,
    qdrant: QdrantDep,
    ollama: OllamaDep,
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
    api_key: str = Depends(verify_api_key),
):
    """
    Semantic search across indexed documents.
    """
    from src.services.vector_service import VectorService
    from src.core.dependencies import get_wikijs_client
    from src.models.document import DocumentSearchHit

    settings = get_settings()
    if not settings.document_store_enabled:
        raise HTTPException(status_code=503, detail="Document store is disabled")

    start_time = time.time()

    try:
        wiki = get_wikijs_client()
        vector_service = VectorService(qdrant, wiki, ollama)

        results = await vector_service.search(
            query=request.query,
            user=user,
            limit=request.limit,
            score_threshold=0.5,
            doc_type="document"
        )

        hits = []
        for result in results.get("results", []):
            hits.append(DocumentSearchHit(
                paperless_id=result.get("metadata", {}).get("paperless_id", 0),
                title=result.get("title", ""),
                score=result.get("score", 0.0),
                highlights=result.get("chunk_text", "")[:200] if request.include_content else None,
                collection=result.get("metadata", {}).get("collection"),
                document_type=result.get("metadata", {}).get("document_type"),
                content_preview=result.get("chunk_text", "")[:500] if request.include_content else None,
            ))

        return DocumentSearchResponse(
            query=request.query,
            hits=hits,
            total=len(hits),
            duration_ms=int((time.time() - start_time) * 1000)
        )

    except Exception as e:
        logger.error(f"Document search failed: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail=str(e))


# =============================================================================
# Health
# =============================================================================


@router.get("/health", response_model=DocumentStoreHealth)
async def document_store_health(paperless: PaperlessDep):
    """Check Paperless-ngx connectivity."""
    settings = get_settings()

    paperless_healthy = False
    if settings.paperless_token:
        try:
            paperless_healthy = await paperless.health_check()
        except Exception as e:
            logger.error(f"Paperless health check failed: {e}")

    return DocumentStoreHealth(
        paperless_healthy=paperless_healthy,
        paperless_version="connected" if paperless_healthy else None,
        total_documents=None,
        indexed_documents=None
    )