library-desk/src/main.py

"""
Library Desk - Main FastAPI Application

Following best practices:
- Async routes for I/O operations
- Dependency injection for configuration
- Proper error handling
- OpenAPI documentation
"""

from fastapi import FastAPI, HTTPException, Depends, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel
from typing import Dict, Any
import logging
from pathlib import Path

from src.config import Settings, get_settings, __version__
from src.core.dependencies import (
    verify_api_key, QdrantDep, WikiJSDep, OllamaDep, Neo4jDep, PaperlessDep
)
from src.core.multi_tenancy import DEFAULT_USER

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Initialize FastAPI app
app = FastAPI(
    title="Library Desk API",
    description="Coordination service for The Library system - HybridRAG queries, document ingestion, entity extraction, and mind map generation",
    version=__version__,
    docs_url="/docs",
    redoc_url="/redoc",
)

# CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],  # Configure appropriately for production
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Register routers
from src.routers import (
    wiki, tools, graph, vector, hybrid_rag, consolidation,
    ingestion, entity_linking, webhooks, rag_search, content,
    maintenance, volatile, documents
)

app.include_router(wiki.router)
app.include_router(tools.router)
app.include_router(graph.router)
app.include_router(vector.router)
app.include_router(hybrid_rag.router)
app.include_router(consolidation.router)
app.include_router(ingestion.router)
app.include_router(entity_linking.router)
app.include_router(webhooks.router)
app.include_router(rag_search.router)
app.include_router(content.router)
app.include_router(maintenance.router)
app.include_router(volatile.router)
app.include_router(documents.router)

# Mount static files directory for Wiki.js integration scripts
static_dir = Path(__file__).parent.parent / "static"
if static_dir.exists():
    app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
    logger.info(f"Mounted static files from {static_dir}")


# Response Models
class HealthResponse(BaseModel):
    """Health check response model."""
    status: str
    app_name: str
    version: str
    services: Dict[str, Any]


class StatsResponse(BaseModel):
    """System statistics response model."""
    neo4j: Dict[str, int]
    qdrant: Dict[str, Any]
    wiki_pages: int
    paperless: Dict[str, Any]


# Routes
@app.get("/", tags=["Root"])
async def root() -> Dict[str, str]:
    """Root endpoint."""
    return {
        "message": "Library Desk API",
        "docs": "/docs",
        "health": "/health"
    }


@app.get("/health", response_model=HealthResponse, tags=["System"])
async def health(settings: Settings = Depends(get_settings)) -> HealthResponse:
    """
    Health check endpoint.
    Returns status of all connected services.
    """
    from src.core.dependencies import check_service_health

    # Check service connectivity
    service_health = await check_service_health()

    # Overall status is healthy if at least Neo4j and Qdrant are up
    all_healthy = service_health.get("neo4j", False) and service_health.get("qdrant", False)
    overall_status = "healthy" if all_healthy else "degraded"

    return HealthResponse(
        status=overall_status,
        app_name=settings.app_name,
        version=settings.app_version,
        services={
            "neo4j": {
                "url": settings.neo4j_uri,
                "healthy": service_health.get("neo4j", False)
            },
            "qdrant": {
                "url": settings.qdrant_url,
                "healthy": service_health.get("qdrant", False)
            },
            "wikijs": {
                "url": settings.wikijs_url,
                "healthy": service_health.get("wikijs", False)
            },
            "searxng": {
                "url": settings.searxng_url,
                "healthy": service_health.get("searxng", False)
            },
            "ollama": {
                "url": settings.ollama_url,
                "model": settings.ollama_model,
                "healthy": service_health.get("ollama", False)
            }
        }
    )


@app.get("/stats", response_model=StatsResponse, tags=["System"])
async def stats(
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
    neo4j: Neo4jDep = None,
    qdrant: QdrantDep = None,
    wikijs: WikiJSDep = None,
    paperless: PaperlessDep = None,
    api_key: str = Depends(verify_api_key)
) -> StatsResponse:
    """
    Get system statistics.

    Returns counts for:
    - Neo4j: nodes by type (Document, Entity, Collection, Search)
    - Qdrant: vectors per collection
    - Wiki.js: total page count
    - Paperless: documents, tags, correspondents, document types
    """
    # Neo4j node counts by label
    neo4j_stats = {}
    try:
        for label in ["Document", "Entity", "Collection", "Search"]:
            result = await neo4j.execute_query(
                f"MATCH (n:{label}) RETURN count(n) as count"
            )
            neo4j_stats[label.lower() + "_nodes"] = result[0]["count"] if result else 0
    except Exception as e:
        logger.error(f"Failed to get Neo4j stats: {e}")
        neo4j_stats = {"error": str(e)}

    # Qdrant collection stats
    qdrant_stats = {}
    try:
        collections = await qdrant.list_collections()
        qdrant_stats["collections"] = len(collections)
        qdrant_stats["total_vectors"] = sum(c.get("vectors_count", 0) for c in collections)
        qdrant_stats["by_collection"] = {
            c["name"]: c["vectors_count"] for c in collections
        }
    except Exception as e:
        logger.error(f"Failed to get Qdrant stats: {e}")
        qdrant_stats = {"error": str(e)}

    # Wiki.js page count
    wiki_pages = 0
    try:
        pages = await wikijs.list_all_pages(user)
        wiki_pages = len(pages)
    except Exception as e:
        logger.warning(f"Failed to get Wiki.js stats: {e}")

    # Paperless-ngx document stats
    paperless_stats = {}
    try:
        # Get document count (page_size=1 for efficiency, we just need the count)
        docs_result = await paperless.list_documents(page_size=1)
        paperless_stats["documents"] = docs_result.get("count", 0)

        # Get metadata counts
        tags = await paperless.list_tags()
        paperless_stats["tags"] = len(tags)

        correspondents = await paperless.list_correspondents()
        paperless_stats["correspondents"] = len(correspondents)

        doc_types = await paperless.list_document_types()
        paperless_stats["document_types"] = len(doc_types)
    except Exception as e:
        logger.warning(f"Failed to get Paperless stats: {e}")
        paperless_stats = {"error": str(e)}

    return StatsResponse(
        neo4j=neo4j_stats,
        qdrant=qdrant_stats,
        wiki_pages=wiki_pages,
        paperless=paperless_stats
    )


@app.post("/ingest/check-updates", tags=["Ingestion"])
async def check_updates(
    documents: Dict[str, Any],
    api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
    """
    Check which documents need updating based on content hashes.
    Used by Scheduler to determine what changed since last sync.

    TODO: Implement update detection:
    1. Query existing documents by path
    2. Compare content hashes
    3. Return list of updates needed
    """
    return {
        "message": "Update checking not yet implemented",
        "updates_needed": [],
        "up_to_date": [],
        "new_documents": []
    }


@app.get("/ingest/status/{document_id}", tags=["Ingestion"])
async def get_ingestion_status(
    document_id: str,
    api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
    """
    Get processing status for a document.

    TODO: Implement status tracking
    """
    return {
        "message": "Status tracking not yet implemented",
        "document_id": document_id,
        "status": "unknown"
    }


@app.get("/ingest/repo-status/{repository}", tags=["Ingestion"])
async def get_repo_status(
    repository: str,
    api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
    """
    Get indexing status for an entire repository.

    TODO: Implement repository-level statistics
    """
    return {
        "message": "Repository status not yet implemented",
        "repository": repository,
        "total_documents": 0,
        "indexed_documents": 0
    }


# Query endpoints
# NOTE: /query/hybrid is implemented in routers/hybrid_rag.py

@app.post("/query/semantic", tags=["Query"])
async def semantic_query(
    query: str = Query(..., min_length=1, description="Search query text"),
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
    limit: int = Query(default=10, ge=1, le=100, description="Maximum results"),
    score_threshold: float = Query(default=0.5, ge=0.0, le=1.0, description="Minimum similarity score"),
    qdrant_client: QdrantDep = None,
    wiki_client: WikiJSDep = None,
    ollama_client: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
):
    """
    Semantic search via Qdrant vector similarity.

    Searches document chunks using embedding similarity. Returns matching
    chunks with relevance scores, page titles, and paths.

    **Example:**
    ```
    POST /query/semantic?query=docker%20configuration&user=jpmschweitzer&limit=10
    ```

    **Returns:** List of matching chunks with similarity scores (0-1)
    """
    from src.services.vector_service import VectorService

    vector_service = VectorService(qdrant_client, wiki_client, ollama_client)
    try:
        return await vector_service.search(
            query=query,
            user=user,
            limit=limit,
            score_threshold=score_threshold
        )
    except ValueError as e:
        raise HTTPException(status_code=400, detail=str(e))
    except Exception as e:
        logger.error(f"Semantic search failed: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail="Search failed")


@app.post("/query/graph", tags=["Query"])
async def graph_query(
    query: str = Query(..., description="Cypher query to execute"),
    user: str = Query(default=DEFAULT_USER, description="User for scoping (auto-filters results)"),
    neo4j_client: Neo4jDep = None,
    wiki_client: WikiJSDep = None,
    api_key: str = Depends(verify_api_key)
):
    """
    Execute a Cypher query against the Neo4j knowledge graph.

    Queries are automatically scoped to the user's data for security.
    Use this for custom graph traversals beyond what /graph/nodes provides.

    **Example:**
    ```
    POST /query/graph?query=MATCH%20(d:Document)-[:MENTIONS]->(p:Person)%20RETURN%20d,p&user=jpmschweitzer
    ```

    **Security:** All queries are user-scoped to prevent cross-user data access.
    """
    from src.services.graph_service import GraphService

    graph_service = GraphService(neo4j_client, wiki_client)
    try:
        return await graph_service.execute_query(
            query=query,
            parameters={},
            user=user
        )
    except ValueError as e:
        raise HTTPException(status_code=400, detail=str(e))
    except Exception as e:
        logger.error(f"Graph query failed: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail="Query execution failed")


# Deduplication endpoints
@app.post("/deduplicate/check", tags=["Deduplication"])
async def check_duplicates(
    request: Dict[str, Any],
    api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
    """
    Check for duplicate or highly similar documents.
    Uses vector similarity and graph analysis.

    Expected fields:
    - document_id: str
    - similarity_threshold: float (default 0.85)

    TODO: Implement deduplication:
    1. Get document embedding from Qdrant
    2. Find similar vectors above threshold
    3. Check graph relationships
    4. Return candidates with similarity scores
    """
    document_id = request.get("document_id")
    threshold = request.get("similarity_threshold", 0.85)

    return {
        "message": "Deduplication not yet implemented",
        "document_id": document_id,
        "threshold": threshold,
        "duplicates": [],
        "suggestions": None
    }


# Application lifecycle
@app.on_event("startup")
async def startup_event():
    """Initialize connections and resources on startup."""
    from src.core.dependencies import startup_clients
    from src.services.wiki_change_listener import WikiChangeListener

    settings = get_settings()
    logger.info(f"Starting {settings.app_name} v{settings.app_version}")
    logger.info(f"Neo4j: {settings.neo4j_uri}")
    logger.info(f"Qdrant: {settings.qdrant_url}")
    logger.info(f"Wiki.js: {settings.wikijs_url}")
    logger.info(f"SearXNG: {settings.searxng_url}")
    logger.info(f"Ollama: {settings.ollama_url}")

    # Initialize all service clients
    await startup_clients()

    # Start Wiki.js change listener (PostgreSQL NOTIFY/LISTEN)
    # This enables automatic processing of user-edited pages
    try:
        wiki_listener = WikiChangeListener()
        await wiki_listener.start()
        # Store reference for shutdown
        app.state.wiki_listener = wiki_listener
        logger.info("Wiki.js change listener started successfully")
    except Exception as e:
        logger.error(f"Failed to start Wiki.js change listener: {e}", exc_info=True)
        logger.warning("Continuing without change listener - manual page updates will not be auto-processed")


@app.on_event("shutdown")
async def shutdown_event():
    """Clean up resources on shutdown."""
    from src.core.dependencies import shutdown_clients

    logger.info("Shutting down Library Desk API")

    # Stop Wiki.js change listener if running
    if hasattr(app.state, "wiki_listener"):
        try:
            await app.state.wiki_listener.stop()
            logger.info("Wiki.js change listener stopped")
        except Exception as e:
            logger.error(f"Error stopping Wiki.js change listener: {e}")

    # Close all service clients
    await shutdown_clients()