""" Library Desk - Main FastAPI Application Following best practices: - Async routes for I/O operations - Dependency injection for configuration - Proper error handling - OpenAPI documentation """ from fastapi import FastAPI, HTTPException, Depends, Query from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field from typing import Dict, Any import logging from pathlib import Path from src.config import Settings, get_settings, __version__ from src.core.dependencies import ( verify_api_key, QdrantDep, WikiJSDep, OllamaDep, Neo4jDep, PaperlessDep, RequiredUserQuery, JobManagerDep ) from src.core.multi_tenancy import RequiredUser # Configure logging logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) logger = logging.getLogger(__name__) # Initialize FastAPI app app = FastAPI( title="Library Desk API", description="Coordination service for The Library system - HybridRAG queries, document ingestion, entity extraction, and mind map generation", version=__version__, docs_url="/docs", redoc_url="/redoc", ) # CORS middleware app.add_middleware( CORSMiddleware, allow_origins=["*"], # Configure appropriately for production allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # Register routers from src.routers import ( wiki, tools, graph, vector, hybrid_rag, consolidation, ingestion, entity_linking, webhooks, rag_search, content, maintenance, volatile, documents ) app.include_router(wiki.router) app.include_router(tools.router) app.include_router(graph.router) app.include_router(vector.router) app.include_router(hybrid_rag.router) app.include_router(consolidation.router) app.include_router(ingestion.router) app.include_router(entity_linking.router) app.include_router(webhooks.router) app.include_router(rag_search.router) app.include_router(content.router) app.include_router(maintenance.router) app.include_router(volatile.router) app.include_router(documents.router) # Mount static files directory for Wiki.js integration scripts static_dir = Path(__file__).parent.parent / "static" if static_dir.exists(): app.mount("/static", StaticFiles(directory=str(static_dir)), name="static") logger.info(f"Mounted static files from {static_dir}") # Response Models class HealthResponse(BaseModel): """Health check response model.""" status: str app_name: str version: str services: Dict[str, Any] class StatsResponse(BaseModel): """System statistics response model.""" neo4j: Dict[str, int] qdrant: Dict[str, Any] wiki_pages: int paperless: Dict[str, Any] # Routes @app.get("/", tags=["Root"]) async def root() -> Dict[str, str]: """Root endpoint.""" return { "message": "Library Desk API", "docs": "/docs", "health": "/health" } @app.get("/health", response_model=HealthResponse, tags=["System"]) async def health(settings: Settings = Depends(get_settings)) -> HealthResponse: """ Health check endpoint. Returns status of all connected services. """ from src.core.dependencies import check_service_health # Check service connectivity service_health = await check_service_health() # Overall status is healthy if at least Neo4j and Qdrant are up all_healthy = service_health.get("neo4j", False) and service_health.get("qdrant", False) overall_status = "healthy" if all_healthy else "degraded" return HealthResponse( status=overall_status, app_name=settings.app_name, version=settings.app_version, services={ "neo4j": { "url": settings.neo4j_uri, "healthy": service_health.get("neo4j", False) }, "qdrant": { "url": settings.qdrant_url, "healthy": service_health.get("qdrant", False) }, "wikijs": { "url": settings.wikijs_url, "healthy": service_health.get("wikijs", False) }, "searxng": { "url": settings.searxng_url, "healthy": service_health.get("searxng", False) }, "ollama": { "url": settings.ollama_url, "model": settings.ollama_llm_model, "healthy": service_health.get("ollama", False) } } ) @app.get("/stats", response_model=StatsResponse, tags=["System"]) async def stats( user: RequiredUserQuery, neo4j: Neo4jDep = None, qdrant: QdrantDep = None, wikijs: WikiJSDep = None, paperless: PaperlessDep = None, api_key: str = Depends(verify_api_key) ) -> StatsResponse: """ Get system statistics. Returns counts for: - Neo4j: nodes by type (Document, Entity, Collection, Search) - Qdrant: vectors per collection - Wiki.js: total page count - Paperless: documents, tags, correspondents, document types """ # Neo4j node counts by label neo4j_stats = {} try: for label in ["Document", "Entity", "Collection", "Search"]: result = await neo4j.execute_query( f"MATCH (n:{label}) RETURN count(n) as count" ) neo4j_stats[label.lower() + "_nodes"] = result[0]["count"] if result else 0 except Exception as e: logger.error(f"Failed to get Neo4j stats: {e}") neo4j_stats = {"error": str(e)} # Qdrant collection stats qdrant_stats = {} try: collections = await qdrant.list_collections() qdrant_stats["collections"] = len(collections) qdrant_stats["total_vectors"] = sum(c.get("vectors_count", 0) for c in collections) qdrant_stats["by_collection"] = { c["name"]: c["vectors_count"] for c in collections } except Exception as e: logger.error(f"Failed to get Qdrant stats: {e}") qdrant_stats = {"error": str(e)} # Wiki.js page count (pages live under the user namespace, e.g. "users/jpmschweitzer/...") wiki_pages = 0 try: pages = await wikijs.list_all_pages(path_prefix=f"users/{user}") wiki_pages = len(pages) except Exception as e: logger.warning(f"Failed to get Wiki.js stats: {e}") # Paperless-ngx document stats paperless_stats = {} try: # Get document count (page_size=1 for efficiency, we just need the count) docs_result = await paperless.list_documents(page_size=1) paperless_stats["documents"] = docs_result.get("count", 0) # Get metadata counts tags = await paperless.list_tags() paperless_stats["tags"] = len(tags) correspondents = await paperless.list_correspondents() paperless_stats["correspondents"] = len(correspondents) doc_types = await paperless.list_document_types() paperless_stats["document_types"] = len(doc_types) except Exception as e: logger.warning(f"Failed to get Paperless stats: {e}") paperless_stats = {"error": str(e)} return StatsResponse( neo4j=neo4j_stats, qdrant=qdrant_stats, wiki_pages=wiki_pages, paperless=paperless_stats ) class CheckUpdatesRequest(BaseModel): """Request body for /ingest/check-updates.""" user: RequiredUser = Field( ..., description="User identifier (tenant). Required — only this tenant's namespace is compared." ) path_prefix: str | None = Field( default=None, description="Optional sub-path inside the tenant namespace (e.g. 'technology')" ) @app.post("/ingest/check-updates", tags=["Ingestion"]) async def check_updates( request: CheckUpdatesRequest, neo4j: Neo4jDep = None, wikijs: WikiJSDep = None, api_key: str = Depends(verify_api_key) ) -> Dict[str, Any]: """ Check which wiki pages need (re-)ingestion based on content hashes. Compares the `content_hash` stored on the tenant's Neo4j Document nodes (recorded at ingestion time) against the SHA-256 of the current Wiki.js page content, in a single UNWIND Cypher query. Used by the Scheduler to determine what changed since the last sync. Read-only. Returns per-tenant lists: - `changed`: page exists in wiki AND graph, but hashes differ (or the stored hash predates hash tracking — flagged `stored_hash_missing`) - `new`: wiki page with no Document node yet - `deleted`: Document node whose wiki page no longer exists """ import time as _time from src.core.hashing import compute_content_hash from src.core.multi_tenancy import get_neo4j_user_label, sanitize_user_id start_time = _time.time() user = request.user tenant_prefix = f"users/{sanitize_user_id(user)}" if request.path_prefix: tenant_prefix = f"{tenant_prefix}/{request.path_prefix.strip('/')}" try: pages = await wikijs.list_all_pages(path_prefix=tenant_prefix) # Auto-generated entity stubs are intentionally never ingested into # the graph (see GraphService.update_from_page), so they would show # up as perpetually "new". Exclude them. pages = [ p for p in pages if not ({"entity-stub", "auto-generated"} & set(p.get("tags") or [])) ] page_hashes = [] for p in pages: full_page = await wikijs.get_page(p["id"]) content = (full_page or {}).get("content", "") page_hashes.append({ "page_id": p["id"], "path": p.get("path", ""), "title": p.get("title", ""), "hash": compute_content_hash(content) }) user_doc_label = get_neo4j_user_label(user) if page_hashes: # Single UNWIND query: compare every current page hash against the # stored Document hash AND collect stale Document nodes whose wiki # page is gone. cypher = f""" UNWIND $pages AS p OPTIONAL MATCH (d:{user_doc_label}:Document {{page_id: p.page_id}}) WITH collect({{ page_id: p.page_id, path: p.path, title: p.title, is_new: d IS NULL, changed: d IS NOT NULL AND (d.content_hash IS NULL OR d.content_hash <> p.hash), stored_hash_missing: d IS NOT NULL AND d.content_hash IS NULL }}) AS checked, collect(p.page_id) AS current_ids OPTIONAL MATCH (stale:{user_doc_label}:Document) WHERE stale.page_id IS NOT NULL AND NOT stale.page_id IN current_ids RETURN checked, collect(CASE WHEN stale IS NULL THEN NULL ELSE {{ page_id: stale.page_id, path: stale.path, title: stale.title }} END) AS deleted """ rows = await neo4j.execute_query(cypher, {"pages": page_hashes}) checked = rows[0]["checked"] if rows else [] deleted = rows[0]["deleted"] if rows else [] else: # No wiki pages under the prefix: every Document node is stale. cypher = f""" MATCH (stale:{user_doc_label}:Document) WHERE stale.page_id IS NOT NULL RETURN collect({{page_id: stale.page_id, path: stale.path, title: stale.title}}) AS deleted """ rows = await neo4j.execute_query(cypher, {}) checked = [] deleted = rows[0]["deleted"] if rows else [] # Deleted detection is namespace-wide only for full-tenant scans; a # sub-path scan must not flag documents outside its prefix. if request.path_prefix: deleted = [ d for d in deleted if str(d.get("path", "")).lstrip("/").startswith(tenant_prefix) ] new_pages = [c for c in checked if c["is_new"]] changed_pages = [c for c in checked if c["changed"]] up_to_date = len(checked) - len(new_pages) - len(changed_pages) duration_ms = (_time.time() - start_time) * 1000 return { "user": user, "path_prefix": tenant_prefix, "total_wiki_pages": len(page_hashes), "changed": [ {k: c[k] for k in ("page_id", "path", "title", "stored_hash_missing")} for c in changed_pages ], "new": [ {k: c[k] for k in ("page_id", "path", "title")} for c in new_pages ], "deleted": deleted, "counts": { "changed": len(changed_pages), "new": len(new_pages), "deleted": len(deleted), "up_to_date": up_to_date }, "duration_ms": duration_ms } except Exception as e: logger.error(f"check-updates failed for {user}: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Update check failed") @app.get("/ingest/status/{job_id}", tags=["Ingestion"]) async def get_ingestion_status( job_id: str, user: RequiredUserQuery, job_manager: JobManagerDep = None, api_key: str = Depends(verify_api_key) ) -> Dict[str, Any]: """ Get processing status for an ingestion job. Backed by the Redis job store (`library:job:{job_id}`, 24h TTL). Job IDs are returned by /ingest/page, /ingest/batch and /ingest/all. Jobs are tenant-scoped: requesting another tenant's job returns 404. """ job = await job_manager.get_job(job_id) if not job or job.get("user") != user: raise HTTPException(status_code=404, detail=f"Job {job_id} not found") return job @app.get("/ingest/repo-status/{repository}", tags=["Ingestion"]) async def get_repo_status( repository: str, user: RequiredUserQuery, neo4j: Neo4jDep = None, wikijs: WikiJSDep = None, job_manager: JobManagerDep = None, api_key: str = Depends(verify_api_key) ) -> Dict[str, Any]: """ Get indexing status for a repository (a sub-path of the tenant namespace). `repository` is resolved as `users/{tenant}/{repository}`; use `_all` for the whole tenant namespace. Reports how many wiki pages exist under the path, how many have graph Document nodes (i.e. are indexed), and the tenant's recent job statistics from the Redis job store. """ import time as _time from src.core.multi_tenancy import get_neo4j_user_label, sanitize_user_id start_time = _time.time() tenant_root = f"users/{sanitize_user_id(user)}" prefix = tenant_root if repository in ("_all", "all", "") else f"{tenant_root}/{repository.strip('/')}" try: pages = await wikijs.list_all_pages(path_prefix=prefix) page_ids = [p["id"] for p in pages if p.get("id")] indexed = 0 if page_ids: user_doc_label = get_neo4j_user_label(user) rows = await neo4j.execute_query( f""" MATCH (d:{user_doc_label}:Document) WHERE d.page_id IN $page_ids RETURN count(DISTINCT d.page_id) AS indexed """, {"page_ids": page_ids} ) indexed = rows[0]["indexed"] if rows else 0 job_stats = await job_manager.get_job_stats(user=user) return { "repository": repository, "user": user, "path_prefix": prefix, "total_documents": len(page_ids), "indexed_documents": indexed, "unindexed_documents": len(page_ids) - indexed, "jobs": job_stats, "duration_ms": (_time.time() - start_time) * 1000 } except Exception as e: logger.error(f"repo-status failed for {user}/{repository}: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Repository status failed") # Query endpoints # NOTE: /query/hybrid is implemented in routers/hybrid_rag.py @app.post("/query/semantic", tags=["Query"]) async def semantic_query( user: RequiredUserQuery, query: str = Query(..., min_length=1, description="Search query text"), limit: int = Query(default=10, ge=1, le=100, description="Maximum results"), score_threshold: float = Query(default=0.5, ge=0.0, le=1.0, description="Minimum similarity score"), qdrant_client: QdrantDep = None, wiki_client: WikiJSDep = None, ollama_client: OllamaDep = None, api_key: str = Depends(verify_api_key) ): """ Semantic search via Qdrant vector similarity. Searches document chunks using embedding similarity. Returns matching chunks with relevance scores, page titles, and paths. **Example:** ``` POST /query/semantic?query=docker%20configuration&user=jpmschweitzer&limit=10 ``` **Returns:** List of matching chunks with similarity scores (0-1) """ from src.services.vector_service import VectorService vector_service = VectorService(qdrant_client, wiki_client, ollama_client) try: return await vector_service.search( query=query, user=user, limit=limit, score_threshold=score_threshold ) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Semantic search failed: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Search failed") @app.post("/query/graph", tags=["Query"]) async def graph_query( user: RequiredUserQuery, query: str = Query(..., description="Cypher query to execute"), neo4j_client: Neo4jDep = None, wiki_client: WikiJSDep = None, api_key: str = Depends(verify_api_key) ): """ Execute a raw Cypher query against the Neo4j knowledge graph (ADMIN/DEBUG — read-only, NOT tenant-scoped). **Security model:** - Queries containing write clauses (CREATE/MERGE/DELETE/SET/REMOVE/DROP/ DETACH/FOREACH/LOAD CSV) or any CALL are rejected with 400. - Execution happens in a read-only Neo4j session, so writes are refused by the database even if validation is bypassed. - Results are NOT automatically restricted to the requesting user's tenant: an arbitrary query can read any tenant's nodes. Scope your own patterns (e.g. `MATCH (d:User__Document:Document) ...`). For tenant-scoped access use /graph/nodes instead. **Example:** ``` POST /query/graph?query=MATCH%20(d:Document)-[:MENTIONS]->(p:Person)%20RETURN%20d,p&user= ``` """ from src.services.graph_service import GraphService graph_service = GraphService(neo4j_client, wiki_client) try: return await graph_service.execute_query( query=query, parameters={}, user=user ) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: logger.error(f"Graph query failed: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Query execution failed") # Deduplication endpoints class DeduplicateCheckRequest(BaseModel): """Request body for /deduplicate/check.""" user: RequiredUser = Field( ..., description="User identifier (tenant). Required — only this tenant's collection is scanned." ) similarity_threshold: float = Field( default=0.9, ge=0.5, le=1.0, description="Minimum cosine similarity for a chunk pair to count as duplicate" ) max_pairs: int = Field(default=100, ge=1, le=500, description="Maximum page pairs returned") @app.post("/deduplicate/check", tags=["Deduplication"]) async def check_duplicates( request: DeduplicateCheckRequest, qdrant_client: QdrantDep = None, wiki_client: WikiJSDep = None, ollama_client: OllamaDep = None, api_key: str = Depends(verify_api_key) ) -> Dict[str, Any]: """ Check for duplicate or highly similar wiki pages (tenant-scoped, read-only). Scans the tenant's own Qdrant collection: every wiki chunk vector is queried against the same collection, and chunk pairs from different pages scoring above the threshold (default 0.9 cosine) are grouped per page pair with the best similarity and matching chunk-pair count. """ import time as _time from src.services.vector_service import VectorService start_time = _time.time() vector_service = VectorService(qdrant_client, wiki_client, ollama_client) try: scan = await vector_service.find_duplicate_pairs( user=request.user, similarity_threshold=request.similarity_threshold, max_pairs=request.max_pairs ) return { "user": request.user, "similarity_threshold": request.similarity_threshold, "chunks_scanned": scan["chunks_scanned"], "duplicate_groups": scan["duplicate_groups"], "duplicate_group_count": len(scan["duplicate_groups"]), "duration_ms": (_time.time() - start_time) * 1000 } except Exception as e: logger.error(f"Deduplication check failed for {request.user}: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Deduplication check failed") # Application lifecycle @app.on_event("startup") async def startup_event(): """Initialize connections and resources on startup.""" import asyncio from src.core.dependencies import startup_clients, get_job_manager from src.jobs.job_manager import job_cleanup_loop from src.services.wiki_change_listener import WikiChangeListener settings = get_settings() logger.info(f"Starting {settings.app_name} v{settings.app_version}") logger.info(f"Neo4j: {settings.neo4j_uri}") logger.info(f"Qdrant: {settings.qdrant_url}") logger.info(f"Wiki.js: {settings.wikijs_url}") logger.info(f"SearXNG: {settings.searxng_url}") logger.info(f"Ollama: {settings.ollama_url}") logger.info(f"Ollama generation model: {settings.ollama_llm_model} (embedding model: {settings.ollama_embedding_model})") # Initialize all service clients await startup_clients() # Hourly in-process cleanup of expired Redis job-set memberships # (job payloads auto-expire via TTL; set memberships do not) app.state.job_cleanup_task = asyncio.create_task( job_cleanup_loop(get_job_manager(), interval_seconds=3600) ) logger.info("Job cleanup loop started (hourly)") # Start Wiki.js change listener (PostgreSQL NOTIFY/LISTEN) # This enables automatic processing of user-edited pages try: wiki_listener = WikiChangeListener() await wiki_listener.start() # Store reference for shutdown app.state.wiki_listener = wiki_listener logger.info("Wiki.js change listener started successfully") except Exception as e: logger.error(f"Failed to start Wiki.js change listener: {e}", exc_info=True) logger.warning("Continuing without change listener - manual page updates will not be auto-processed") @app.on_event("shutdown") async def shutdown_event(): """Clean up resources on shutdown.""" from src.core.dependencies import shutdown_clients logger.info("Shutting down Library Desk API") # Stop the job cleanup loop if hasattr(app.state, "job_cleanup_task"): app.state.job_cleanup_task.cancel() try: await app.state.job_cleanup_task except Exception: pass logger.info("Job cleanup loop stopped") # Stop Wiki.js change listener if running if hasattr(app.state, "wiki_listener"): try: await app.state.wiki_listener.stop() logger.info("Wiki.js change listener stopped") except Exception as e: logger.error(f"Error stopping Wiki.js change listener: {e}") # Close all service clients await shutdown_clients()