library-desk/src/main.py

"""
Library Desk - Main FastAPI Application

Following best practices:
- Async routes for I/O operations
- Dependency injection for configuration
- Proper error handling
- OpenAPI documentation
"""

from fastapi import FastAPI, HTTPException, Depends, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field
from typing import Dict, Any
import logging
from pathlib import Path

from src.config import Settings, get_settings, __version__
from src.core.dependencies import (
    verify_api_key, QdrantDep, WikiJSDep, OllamaDep, Neo4jDep, PaperlessDep,
    RequiredUserQuery, JobManagerDep
)
from src.core.multi_tenancy import RequiredUser

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)

# Initialize FastAPI app
app = FastAPI(
    title="Library Desk API",
    description="Coordination service for The Library system - HybridRAG queries, document ingestion, entity extraction, and mind map generation",
    version=__version__,
    docs_url="/docs",
    redoc_url="/redoc",
)

# CORS middleware.
# allow_credentials is deliberately False: combined with a wildcard origin it
# would tell browsers to attach cookies/credentials for ANY site, which is the
# classic CORS misconfiguration. All real callers (tatlock, the Scheduler) are
# server-to-server and use the Authorization header, which wildcard-origin
# CORS without credentials still permits. Origins can be restricted via the
# CORS_ALLOW_ORIGINS env (comma-separated) once a cross-origin browser UI
# exists; the bundled static UI is served same-origin and needs no CORS.
app.add_middleware(
    CORSMiddleware,
    allow_origins=get_settings().cors_allow_origins_list,
    allow_credentials=False,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Register routers
from src.routers import (
    wiki, tools, graph, vector, hybrid_rag, consolidation,
    ingestion, entity_linking, webhooks, rag_search, content,
    maintenance, volatile, documents
)

app.include_router(wiki.router)
app.include_router(tools.router)
app.include_router(graph.router)
app.include_router(vector.router)
app.include_router(hybrid_rag.router)
app.include_router(consolidation.router)
app.include_router(ingestion.router)
app.include_router(entity_linking.router)
app.include_router(webhooks.router)
app.include_router(rag_search.router)
app.include_router(content.router)
app.include_router(maintenance.router)
app.include_router(volatile.router)
app.include_router(documents.router)

# Mount static files directory for Wiki.js integration scripts
static_dir = Path(__file__).parent.parent / "static"
if static_dir.exists():
    app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
    logger.info(f"Mounted static files from {static_dir}")


# Response Models
class HealthResponse(BaseModel):
    """Health check response model."""
    status: str
    app_name: str
    version: str
    services: Dict[str, Any]


class StatsResponse(BaseModel):
    """System statistics response model."""
    neo4j: Dict[str, int]
    qdrant: Dict[str, Any]
    wiki_pages: int
    paperless: Dict[str, Any]


# Routes
@app.get("/", tags=["Root"])
async def root() -> Dict[str, str]:
    """Root endpoint."""
    return {
        "message": "Library Desk API",
        "docs": "/docs",
        "health": "/health"
    }


@app.get("/health", response_model=HealthResponse, tags=["System"])
async def health(settings: Settings = Depends(get_settings)) -> HealthResponse:
    """
    Health check endpoint.
    Returns status of all connected services.
    """
    from src.core.dependencies import check_service_health

    # Check service connectivity
    service_health = await check_service_health()

    # Overall status is healthy if at least Neo4j and Qdrant are up
    all_healthy = service_health.get("neo4j", False) and service_health.get("qdrant", False)
    overall_status = "healthy" if all_healthy else "degraded"

    return HealthResponse(
        status=overall_status,
        app_name=settings.app_name,
        version=settings.app_version,
        services={
            "neo4j": {
                "url": settings.neo4j_uri,
                "healthy": service_health.get("neo4j", False)
            },
            "qdrant": {
                "url": settings.qdrant_url,
                "healthy": service_health.get("qdrant", False)
            },
            "wikijs": {
                "url": settings.wikijs_url,
                "healthy": service_health.get("wikijs", False)
            },
            "searxng": {
                "url": settings.searxng_url,
                "healthy": service_health.get("searxng", False)
            },
            "ollama": {
                "url": settings.ollama_url,
                "model": settings.ollama_llm_model,
                "healthy": service_health.get("ollama", False)
            }
        }
    )


@app.get("/stats", response_model=StatsResponse, tags=["System"])
async def stats(
    user: RequiredUserQuery,
    neo4j: Neo4jDep = None,
    qdrant: QdrantDep = None,
    wikijs: WikiJSDep = None,
    paperless: PaperlessDep = None,
    api_key: str = Depends(verify_api_key)
) -> StatsResponse:
    """
    Get system statistics.

    Returns counts for:
    - Neo4j: nodes by type (Document, Entity, Collection, Search)
    - Qdrant: vectors per collection
    - Wiki.js: total page count
    - Paperless: documents, tags, correspondents, document types
    """
    # Neo4j node counts by label
    neo4j_stats = {}
    try:
        for label in ["Document", "Entity", "Collection", "Search"]:
            result = await neo4j.execute_query(
                f"MATCH (n:{label}) RETURN count(n) as count"
            )
            neo4j_stats[label.lower() + "_nodes"] = result[0]["count"] if result else 0
    except Exception as e:
        logger.error(f"Failed to get Neo4j stats: {e}")
        neo4j_stats = {"error": str(e)}

    # Qdrant collection stats
    qdrant_stats = {}
    try:
        collections = await qdrant.list_collections()
        qdrant_stats["collections"] = len(collections)
        qdrant_stats["total_vectors"] = sum(c.get("vectors_count", 0) for c in collections)
        qdrant_stats["by_collection"] = {
            c["name"]: c["vectors_count"] for c in collections
        }
    except Exception as e:
        logger.error(f"Failed to get Qdrant stats: {e}")
        qdrant_stats = {"error": str(e)}

    # Wiki.js page count (pages live under the user namespace, e.g. "users/jpmschweitzer/...")
    wiki_pages = 0
    try:
        pages = await wikijs.list_all_pages(path_prefix=f"users/{user}")
        wiki_pages = len(pages)
    except Exception as e:
        logger.warning(f"Failed to get Wiki.js stats: {e}")

    # Paperless-ngx document stats
    paperless_stats = {}
    try:
        # Get document count (page_size=1 for efficiency, we just need the count)
        docs_result = await paperless.list_documents(page_size=1)
        paperless_stats["documents"] = docs_result.get("count", 0)

        # Get metadata counts
        tags = await paperless.list_tags()
        paperless_stats["tags"] = len(tags)

        correspondents = await paperless.list_correspondents()
        paperless_stats["correspondents"] = len(correspondents)

        doc_types = await paperless.list_document_types()
        paperless_stats["document_types"] = len(doc_types)
    except Exception as e:
        logger.warning(f"Failed to get Paperless stats: {e}")
        paperless_stats = {"error": str(e)}

    return StatsResponse(
        neo4j=neo4j_stats,
        qdrant=qdrant_stats,
        wiki_pages=wiki_pages,
        paperless=paperless_stats
    )


class CheckUpdatesRequest(BaseModel):
    """Request body for /ingest/check-updates."""
    user: RequiredUser = Field(
        ...,
        description="User identifier (tenant). Required — only this tenant's namespace is compared."
    )
    path_prefix: str | None = Field(
        default=None,
        description="Optional sub-path inside the tenant namespace (e.g. 'technology')"
    )


@app.post("/ingest/check-updates", tags=["Ingestion"])
async def check_updates(
    request: CheckUpdatesRequest,
    neo4j: Neo4jDep = None,
    wikijs: WikiJSDep = None,
    api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
    """
    Check which wiki pages need (re-)ingestion based on content hashes.

    Compares the `content_hash` stored on the tenant's Neo4j Document nodes
    (recorded at ingestion time) against the SHA-256 of the current Wiki.js
    page content, in a single UNWIND Cypher query. Used by the Scheduler to
    determine what changed since the last sync. Read-only.

    Returns per-tenant lists:
    - `changed`: page exists in wiki AND graph, but hashes differ (or the
      stored hash predates hash tracking — flagged `stored_hash_missing`)
    - `new`: wiki page with no Document node yet
    - `deleted`: Document node whose wiki page no longer exists
    """
    import time as _time
    from src.core.hashing import compute_content_hash
    from src.core.multi_tenancy import get_neo4j_user_label, sanitize_user_id

    start_time = _time.time()
    user = request.user
    tenant_prefix = f"users/{sanitize_user_id(user)}"
    if request.path_prefix:
        tenant_prefix = f"{tenant_prefix}/{request.path_prefix.strip('/')}"

    try:
        pages = await wikijs.list_all_pages(path_prefix=tenant_prefix)

        # Auto-generated entity stubs are intentionally never ingested into
        # the graph (see GraphService.update_from_page), so they would show
        # up as perpetually "new". Exclude them.
        pages = [
            p for p in pages
            if not ({"entity-stub", "auto-generated"} & set(p.get("tags") or []))
        ]

        page_hashes = []
        for p in pages:
            full_page = await wikijs.get_page(p["id"])
            content = (full_page or {}).get("content", "")
            page_hashes.append({
                "page_id": p["id"],
                "path": p.get("path", ""),
                "title": p.get("title", ""),
                "hash": compute_content_hash(content)
            })

        user_doc_label = get_neo4j_user_label(user)

        if page_hashes:
            # Single UNWIND query: compare every current page hash against the
            # stored Document hash AND collect stale Document nodes whose wiki
            # page is gone.
            cypher = f"""
            UNWIND $pages AS p
            OPTIONAL MATCH (d:{user_doc_label}:Document {{page_id: p.page_id}})
            WITH collect({{
                    page_id: p.page_id,
                    path: p.path,
                    title: p.title,
                    is_new: d IS NULL,
                    changed: d IS NOT NULL AND (d.content_hash IS NULL OR d.content_hash <> p.hash),
                    stored_hash_missing: d IS NOT NULL AND d.content_hash IS NULL
                }}) AS checked,
                collect(p.page_id) AS current_ids
            OPTIONAL MATCH (stale:{user_doc_label}:Document)
            WHERE stale.page_id IS NOT NULL AND NOT stale.page_id IN current_ids
            RETURN checked,
                   collect(CASE WHEN stale IS NULL THEN NULL ELSE {{
                       page_id: stale.page_id, path: stale.path, title: stale.title
                   }} END) AS deleted
            """
            rows = await neo4j.execute_query(cypher, {"pages": page_hashes})
            checked = rows[0]["checked"] if rows else []
            deleted = rows[0]["deleted"] if rows else []
        else:
            # No wiki pages under the prefix: every Document node is stale.
            cypher = f"""
            MATCH (stale:{user_doc_label}:Document)
            WHERE stale.page_id IS NOT NULL
            RETURN collect({{page_id: stale.page_id, path: stale.path, title: stale.title}}) AS deleted
            """
            rows = await neo4j.execute_query(cypher, {})
            checked = []
            deleted = rows[0]["deleted"] if rows else []

        # Deleted detection is namespace-wide only for full-tenant scans; a
        # sub-path scan must not flag documents outside its prefix.
        if request.path_prefix:
            deleted = [
                d for d in deleted
                if str(d.get("path", "")).lstrip("/").startswith(tenant_prefix)
            ]

        new_pages = [c for c in checked if c["is_new"]]
        changed_pages = [c for c in checked if c["changed"]]
        up_to_date = len(checked) - len(new_pages) - len(changed_pages)

        duration_ms = (_time.time() - start_time) * 1000

        return {
            "user": user,
            "path_prefix": tenant_prefix,
            "total_wiki_pages": len(page_hashes),
            "changed": [
                {k: c[k] for k in ("page_id", "path", "title", "stored_hash_missing")}
                for c in changed_pages
            ],
            "new": [
                {k: c[k] for k in ("page_id", "path", "title")} for c in new_pages
            ],
            "deleted": deleted,
            "counts": {
                "changed": len(changed_pages),
                "new": len(new_pages),
                "deleted": len(deleted),
                "up_to_date": up_to_date
            },
            "duration_ms": duration_ms
        }

    except Exception as e:
        logger.error(f"check-updates failed for {user}: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail="Update check failed")


@app.get("/ingest/status/{job_id}", tags=["Ingestion"])
async def get_ingestion_status(
    job_id: str,
    user: RequiredUserQuery,
    job_manager: JobManagerDep = None,
    api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
    """
    Get processing status for an ingestion job.

    Backed by the Redis job store (`library:job:{job_id}`, 24h TTL). Job IDs
    are returned by /ingest/page, /ingest/batch and /ingest/all. Jobs are
    tenant-scoped: requesting another tenant's job returns 404.
    """
    job = await job_manager.get_job(job_id)
    if not job or job.get("user") != user:
        raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
    return job


@app.get("/ingest/repo-status/{repository}", tags=["Ingestion"])
async def get_repo_status(
    repository: str,
    user: RequiredUserQuery,
    neo4j: Neo4jDep = None,
    wikijs: WikiJSDep = None,
    job_manager: JobManagerDep = None,
    api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
    """
    Get indexing status for a repository (a sub-path of the tenant namespace).

    `repository` is resolved as `users/{tenant}/{repository}`; use `_all` for
    the whole tenant namespace. Reports how many wiki pages exist under the
    path, how many have graph Document nodes (i.e. are indexed), and the
    tenant's recent job statistics from the Redis job store.
    """
    import time as _time
    from src.core.multi_tenancy import get_neo4j_user_label, sanitize_user_id

    start_time = _time.time()
    tenant_root = f"users/{sanitize_user_id(user)}"
    prefix = tenant_root if repository in ("_all", "all", "") else f"{tenant_root}/{repository.strip('/')}"

    try:
        pages = await wikijs.list_all_pages(path_prefix=prefix)
        page_ids = [p["id"] for p in pages if p.get("id")]

        indexed = 0
        if page_ids:
            user_doc_label = get_neo4j_user_label(user)
            rows = await neo4j.execute_query(
                f"""
                MATCH (d:{user_doc_label}:Document)
                WHERE d.page_id IN $page_ids
                RETURN count(DISTINCT d.page_id) AS indexed
                """,
                {"page_ids": page_ids}
            )
            indexed = rows[0]["indexed"] if rows else 0

        job_stats = await job_manager.get_job_stats(user=user)

        return {
            "repository": repository,
            "user": user,
            "path_prefix": prefix,
            "total_documents": len(page_ids),
            "indexed_documents": indexed,
            "unindexed_documents": len(page_ids) - indexed,
            "jobs": job_stats,
            "duration_ms": (_time.time() - start_time) * 1000
        }

    except Exception as e:
        logger.error(f"repo-status failed for {user}/{repository}: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail="Repository status failed")


# Query endpoints
# NOTE: /query/hybrid is implemented in routers/hybrid_rag.py

@app.post("/query/semantic", tags=["Query"])
async def semantic_query(
    user: RequiredUserQuery,
    query: str = Query(..., min_length=1, description="Search query text"),
    limit: int = Query(default=10, ge=1, le=100, description="Maximum results"),
    score_threshold: float = Query(default=0.5, ge=0.0, le=1.0, description="Minimum similarity score"),
    qdrant_client: QdrantDep = None,
    wiki_client: WikiJSDep = None,
    ollama_client: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
):
    """
    Semantic search via Qdrant vector similarity.

    Searches document chunks using embedding similarity. Returns matching
    chunks with relevance scores, page titles, and paths.

    **Example:**
    ```
    POST /query/semantic?query=docker%20configuration&user=jpmschweitzer&limit=10
    ```

    **Returns:** List of matching chunks with similarity scores (0-1)
    """
    from src.services.vector_service import VectorService

    vector_service = VectorService(qdrant_client, wiki_client, ollama_client)
    try:
        return await vector_service.search(
            query=query,
            user=user,
            limit=limit,
            score_threshold=score_threshold
        )
    except ValueError as e:
        raise HTTPException(status_code=400, detail=str(e))
    except Exception as e:
        logger.error(f"Semantic search failed: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail="Search failed")


@app.post("/query/graph", tags=["Query"])
async def graph_query(
    user: RequiredUserQuery,
    query: str = Query(..., description="Cypher query to execute"),
    neo4j_client: Neo4jDep = None,
    wiki_client: WikiJSDep = None,
    api_key: str = Depends(verify_api_key)
):
    """
    Execute a raw Cypher query against the Neo4j knowledge graph
    (ADMIN/DEBUG — read-only, NOT tenant-scoped).

    **Security model:**
    - Queries containing write clauses (CREATE/MERGE/DELETE/SET/REMOVE/DROP/
      DETACH/FOREACH/LOAD CSV) or any CALL are rejected with 400.
    - Execution happens in a read-only Neo4j session, so writes are refused
      by the database even if validation is bypassed.
    - Results are NOT automatically restricted to the requesting user's
      tenant: an arbitrary query can read any tenant's nodes. Scope your
      own patterns (e.g. `MATCH (d:User_<Tenant>_Document:Document) ...`).
      For tenant-scoped access use /graph/nodes instead.

    **Example:**
    ```
    POST /query/graph?query=MATCH%20(d:Document)-[:MENTIONS]->(p:Person)%20RETURN%20d,p&user=<tenant>
    ```
    """
    from src.services.graph_service import GraphService

    graph_service = GraphService(neo4j_client, wiki_client)
    try:
        return await graph_service.execute_query(
            query=query,
            parameters={},
            user=user
        )
    except ValueError as e:
        raise HTTPException(status_code=400, detail=str(e))
    except Exception as e:
        logger.error(f"Graph query failed: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail="Query execution failed")


# Deduplication endpoints
class DeduplicateCheckRequest(BaseModel):
    """Request body for /deduplicate/check."""
    user: RequiredUser = Field(
        ...,
        description="User identifier (tenant). Required — only this tenant's collection is scanned."
    )
    similarity_threshold: float = Field(
        default=0.9, ge=0.5, le=1.0,
        description="Minimum cosine similarity for a chunk pair to count as duplicate"
    )
    max_pairs: int = Field(default=100, ge=1, le=500, description="Maximum page pairs returned")


@app.post("/deduplicate/check", tags=["Deduplication"])
async def check_duplicates(
    request: DeduplicateCheckRequest,
    qdrant_client: QdrantDep = None,
    wiki_client: WikiJSDep = None,
    ollama_client: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
) -> Dict[str, Any]:
    """
    Check for duplicate or highly similar wiki pages (tenant-scoped, read-only).

    Scans the tenant's own Qdrant collection: every wiki chunk vector is
    queried against the same collection, and chunk pairs from different
    pages scoring above the threshold (default 0.9 cosine) are grouped per
    page pair with the best similarity and matching chunk-pair count.
    """
    import time as _time
    from src.services.vector_service import VectorService

    start_time = _time.time()
    vector_service = VectorService(qdrant_client, wiki_client, ollama_client)

    try:
        scan = await vector_service.find_duplicate_pairs(
            user=request.user,
            similarity_threshold=request.similarity_threshold,
            max_pairs=request.max_pairs
        )
        return {
            "user": request.user,
            "similarity_threshold": request.similarity_threshold,
            "chunks_scanned": scan["chunks_scanned"],
            "duplicate_groups": scan["duplicate_groups"],
            "duplicate_group_count": len(scan["duplicate_groups"]),
            "duration_ms": (_time.time() - start_time) * 1000
        }
    except Exception as e:
        logger.error(f"Deduplication check failed for {request.user}: {e}", exc_info=True)
        raise HTTPException(status_code=500, detail="Deduplication check failed")


# Application lifecycle
@app.on_event("startup")
async def startup_event():
    """Initialize connections and resources on startup."""
    import asyncio

    from src.core.dependencies import startup_clients, get_job_manager
    from src.jobs.job_manager import job_cleanup_loop
    from src.services.wiki_change_listener import WikiChangeListener

    settings = get_settings()
    logger.info(f"Starting {settings.app_name} v{settings.app_version}")
    logger.info(f"Neo4j: {settings.neo4j_uri}")
    logger.info(f"Qdrant: {settings.qdrant_url}")
    logger.info(f"Wiki.js: {settings.wikijs_url}")
    logger.info(f"SearXNG: {settings.searxng_url}")
    logger.info(f"Ollama: {settings.ollama_url}")
    logger.info(f"Ollama generation model: {settings.ollama_llm_model} (embedding model: {settings.ollama_embedding_model})")

    # Initialize all service clients
    await startup_clients()

    # Hourly in-process cleanup of expired Redis job-set memberships
    # (job payloads auto-expire via TTL; set memberships do not)
    app.state.job_cleanup_task = asyncio.create_task(
        job_cleanup_loop(get_job_manager(), interval_seconds=3600)
    )
    logger.info("Job cleanup loop started (hourly)")

    # Start Wiki.js change listener (PostgreSQL NOTIFY/LISTEN)
    # This enables automatic processing of user-edited pages
    try:
        wiki_listener = WikiChangeListener()
        await wiki_listener.start()
        # Store reference for shutdown
        app.state.wiki_listener = wiki_listener
        logger.info("Wiki.js change listener started successfully")
    except Exception as e:
        logger.error(f"Failed to start Wiki.js change listener: {e}", exc_info=True)
        logger.warning("Continuing without change listener - manual page updates will not be auto-processed")


@app.on_event("shutdown")
async def shutdown_event():
    """Clean up resources on shutdown."""
    from src.core.dependencies import shutdown_clients

    logger.info("Shutting down Library Desk API")

    # Stop the job cleanup loop
    if hasattr(app.state, "job_cleanup_task"):
        app.state.job_cleanup_task.cancel()
        try:
            await app.state.job_cleanup_task
        except Exception:
            pass
        logger.info("Job cleanup loop stopped")

    # Stop Wiki.js change listener if running
    if hasattr(app.state, "wiki_listener"):
        try:
            await app.state.wiki_listener.stop()
            logger.info("Wiki.js change listener stopped")
        except Exception as e:
            logger.error(f"Error stopping Wiki.js change listener: {e}")

    # Close all service clients
    await shutdown_clients()