Initial commit: library-desk service extraction from portainer-core

src/core/dependencies.py

@@ -0,0 +1,384 @@
+"""
+Dependency injection for Library Desk.
+
+Provides FastAPI dependencies for service clients with:
+- Singleton pattern via @lru_cache
+- Lazy initialization
+- Proper lifecycle management
+- Type aliases for clean endpoint signatures
+"""
+
+from functools import lru_cache
+from typing import Annotated
+from fastapi import Depends
+import logging
+
+from src.config import Settings, get_settings
+from src.clients.neo4j_client import Neo4jClient
+from src.clients.qdrant_client import QdrantClientWrapper
+from src.clients.wikijs_client import WikiJSClient
+from src.clients.searxng_client import SearXNGClient
+from src.clients.ollama_client import OllamaClient
+
+logger = logging.getLogger(__name__)
+
+
+# Settings dependency
+SettingsDep = Annotated[Settings, Depends(get_settings)]
+
+
+# Client factory functions with @lru_cache for singletons
+@lru_cache
+def get_neo4j_client() -> Neo4jClient:
+    """
+    Get Neo4j client singleton.
+
+    Returns:
+        Initialized Neo4j client (not yet connected)
+
+    Note: Call client.connect() during app startup
+    """
+    settings = get_settings()
+    client = Neo4jClient(
+        uri=settings.neo4j_uri,
+        user=settings.neo4j_user,
+        password=settings.neo4j_password
+    )
+    logger.debug("Created Neo4j client instance")
+    return client
+
+
+@lru_cache
+def get_qdrant_client() -> QdrantClientWrapper:
+    """
+    Get Qdrant client singleton.
+
+    Returns:
+        Initialized Qdrant client
+
+    Note: Collections are created lazily per-user
+    """
+    settings = get_settings()
+    client = QdrantClientWrapper(
+        url=settings.qdrant_url,
+        embedding_dim=768  # nomic-embed-text default
+    )
+    logger.debug("Created Qdrant client instance")
+    return client
+
+
+@lru_cache
+def get_wikijs_client() -> WikiJSClient:
+    """
+    Get Wiki.js client singleton.
+
+    Returns:
+        Initialized Wiki.js GraphQL client with username/password auth
+    """
+    settings = get_settings()
+    client = WikiJSClient(
+        base_url=settings.wikijs_url,
+        username=settings.wikijs_username,
+        password=settings.wikijs_password
+    )
+    logger.debug("Created Wiki.js client instance")
+    return client
+
+
+@lru_cache
+def get_searxng_client() -> SearXNGClient:
+    """
+    Get SearXNG client singleton.
+
+    Returns:
+        Initialized SearXNG search client
+    """
+    settings = get_settings()
+    client = SearXNGClient(base_url=settings.searxng_url)
+    logger.debug("Created SearXNG client instance")
+    return client
+
+
+@lru_cache
+def get_ollama_client() -> OllamaClient:
+    """
+    Get Ollama client singleton.
+
+    Returns:
+        Initialized Ollama embeddings client
+    """
+    settings = get_settings()
+    client = OllamaClient(
+        base_url=settings.ollama_url,
+        model=settings.ollama_model
+    )
+    logger.debug("Created Ollama client instance")
+    return client
+
+
+# Type aliases for FastAPI endpoint dependencies
+# Usage: def my_endpoint(neo4j: Neo4jDep):
+Neo4jDep = Annotated[Neo4jClient, Depends(get_neo4j_client)]
+QdrantDep = Annotated[QdrantClientWrapper, Depends(get_qdrant_client)]
+WikiJSDep = Annotated[WikiJSClient, Depends(get_wikijs_client)]
+SearXNGDep = Annotated[SearXNGClient, Depends(get_searxng_client)]
+OllamaDep = Annotated[OllamaClient, Depends(get_ollama_client)]
+
+
+# Lifecycle management functions
+async def startup_clients():
+    """
+    Initialize all service clients at application startup.
+
+    Should be called in FastAPI lifespan or startup event.
+
+    Performs:
+    - Neo4j connection pool initialization
+    - Neo4j connectivity verification
+    - Ollama model availability check
+    """
+    logger.info("Starting up service clients...")
+
+    # Initialize Neo4j connection pool
+    neo4j = get_neo4j_client()
+    try:
+        await neo4j.connect()
+        logger.info("✓ Neo4j connected")
+    except Exception as e:
+        logger.error(f"✗ Neo4j connection failed: {e}")
+        # Don't fail startup - allow degraded operation
+        pass
+
+    # Check Ollama availability
+    ollama = get_ollama_client()
+    try:
+        is_healthy = await ollama.health_check()
+        if is_healthy:
+            logger.info(f"✓ Ollama ready (model: {ollama.model})")
+        else:
+            logger.warning(f"✗ Ollama model '{ollama.model}' not available")
+    except Exception as e:
+        logger.error(f"✗ Ollama health check failed: {e}")
+        pass
+
+    # Qdrant, Wiki.js, SearXNG are lazy-initialized
+    logger.info("Service clients startup complete")
+
+
+async def shutdown_clients():
+    """
+    Cleanup all service clients at application shutdown.
+
+    Should be called in FastAPI lifespan or shutdown event.
+
+    Performs:
+    - Close Neo4j connection pool
+    - Close HTTP clients
+    """
+    logger.info("Shutting down service clients...")
+
+    # Close Neo4j driver
+    neo4j = get_neo4j_client()
+    try:
+        await neo4j.close()
+        logger.info("✓ Neo4j closed")
+    except Exception as e:
+        logger.error(f"Error closing Neo4j: {e}")
+
+    # Close HTTP clients
+    clients_to_close = [
+        ("Wiki.js", get_wikijs_client()),
+        ("SearXNG", get_searxng_client()),
+        ("Ollama", get_ollama_client())
+    ]
+
+    for name, client in clients_to_close:
+        try:
+            await client.close()
+            logger.info(f"✓ {name} client closed")
+        except Exception as e:
+            logger.error(f"Error closing {name} client: {e}")
+
+    logger.info("Service clients shutdown complete")
+
+
+async def check_service_health() -> dict:
+    """
+    Check health of all service clients.
+
+    Returns:
+        Dictionary with health status of each service:
+        {
+            "neo4j": bool,
+            "qdrant": bool,
+            "wikijs": bool,
+            "searxng": bool,
+            "ollama": bool
+        }
+
+    Usage:
+        >>> health = await check_service_health()
+        >>> health["neo4j"]
+        True
+    """
+    health = {}
+
+    # Neo4j
+    try:
+        neo4j = get_neo4j_client()
+        # Simple query to check connectivity
+        await neo4j.execute_query("RETURN 1 as test", {})
+        health["neo4j"] = True
+    except Exception as e:
+        logger.error(f"Neo4j health check failed: {e}")
+        health["neo4j"] = False
+
+    # Qdrant
+    try:
+        qdrant = get_qdrant_client()
+        # Check if we can list collections
+        collections = qdrant.client.get_collections()
+        health["qdrant"] = True
+    except Exception as e:
+        logger.error(f"Qdrant health check failed: {e}")
+        health["qdrant"] = False
+
+    # Wiki.js
+    try:
+        wikijs = get_wikijs_client()
+        # Try a simple query (list pages with limit 1)
+        await wikijs.list_pages(limit=1)
+        health["wikijs"] = True
+    except Exception as e:
+        logger.error(f"Wiki.js health check failed: {e}")
+        health["wikijs"] = False
+
+    # SearXNG
+    try:
+        searxng = get_searxng_client()
+        # Just check if service is up (no actual search)
+        health["searxng"] = await searxng.health_check()
+    except Exception as e:
+        logger.error(f"SearXNG health check failed: {e}")
+        health["searxng"] = False
+
+    # Ollama
+    try:
+        ollama = get_ollama_client()
+        is_healthy = await ollama.health_check()
+        health["ollama"] = is_healthy
+    except Exception as e:
+        logger.error(f"Ollama health check failed: {e}")
+        health["ollama"] = False
+
+    return health
+
+
+# Service factory functions
+@lru_cache
+def get_vector_service() -> "VectorService":
+    """Get VectorService singleton."""
+    from src.services.vector_service import VectorService
+    return VectorService(
+        qdrant_client=get_qdrant_client(),
+        wikijs_client=get_wikijs_client(),
+        ollama_client=get_ollama_client()
+    )
+
+
+@lru_cache
+def get_graph_service() -> "GraphService":
+    """Get GraphService singleton."""
+    from src.services.graph_service import GraphService
+    return GraphService(
+        neo4j_client=get_neo4j_client(),
+        wikijs_client=get_wikijs_client()
+    )
+
+
+@lru_cache
+def get_wiki_service() -> "WikiService":
+    """Get WikiService singleton."""
+    from src.services.wiki_service import WikiService
+    return WikiService(wiki_client=get_wikijs_client())
+
+
+@lru_cache
+def get_consolidation_service() -> "ConsolidationService":
+    """Get ConsolidationService singleton."""
+    from src.services.consolidation_service import ConsolidationService
+    return ConsolidationService(
+        neo4j=get_neo4j_client(),
+        ollama=get_ollama_client(),
+        wiki=get_wikijs_client(),
+        settings=get_settings(),
+        ingestion_service=get_ingestion_service()
+    )
+
+
+@lru_cache
+def get_ingestion_service() -> "IngestionService":
+    """Get IngestionService singleton."""
+    from src.services.ingestion_service import IngestionService
+    return IngestionService(
+        vector_service=get_vector_service(),
+        graph_service=get_graph_service(),
+        wiki_client=get_wikijs_client()
+    )
+
+
+@lru_cache
+def get_hybrid_rag_service() -> "HybridRAGService":
+    """Get HybridRAGService singleton."""
+    from src.services.hybrid_rag_service import HybridRAGService
+    return HybridRAGService(
+        vector_service=get_vector_service(),
+        graph_service=get_graph_service(),
+        searxng_client=get_searxng_client(),
+        ollama_client=get_ollama_client(),
+        settings=get_settings()
+    )
+
+
+# Utility: Get default user from settings or multi_tenancy
+def get_default_user() -> str:
+    """
+    Get default user for operations.
+
+    Returns:
+        Default user identifier
+    """
+    from src.core.multi_tenancy import DEFAULT_USER
+    return DEFAULT_USER
+
+
+# Authentication
+from fastapi import Security, HTTPException
+from fastapi.security import HTTPBearer
+
+security = HTTPBearer()
+
+
+async def verify_api_key(
+    credentials: Annotated[HTTPBearer, Security(security)],
+    settings: SettingsDep
+) -> str:
+    """
+    Verify API key from Bearer token.
+
+    Args:
+        credentials: HTTP Bearer credentials
+        settings: Application settings
+
+    Returns:
+        API key if valid
+
+    Raises:
+        HTTPException: If API key is invalid
+    """
+    if credentials.credentials != settings.library_api_key:
+        raise HTTPException(
+            status_code=403,
+            detail="Invalid API key"
+        )
+    return credentials.credentials

src/core/multi_tenancy.py

@@ -0,0 +1,209 @@
+"""
+Multi-tenancy helpers for Library Desk.
+
+Provides utilities for user namespace management across:
+- Wiki.js (path-based namespaces)
+- Neo4j (user-specific labels)
+- Qdrant (collection per user)
+"""
+
+import re
+
+# Default user for all operations
+DEFAULT_USER = "jpmschweitzer"
+
+
+def sanitize_user_id(user_id: str) -> str:
+    """
+    Sanitize user ID for use in collection names, labels, and paths.
+
+    Converts special characters to underscores and ensures alphanumeric safety.
+
+    Args:
+        user_id: Raw user identifier (email, username, etc.)
+
+    Returns:
+        Sanitized user ID safe for use in identifiers
+
+    Examples:
+        >>> sanitize_user_id("john@example.com")
+        'john_at_example_com'
+        >>> sanitize_user_id("user.name")
+        'user_name'
+        >>> sanitize_user_id("User Name")
+        'user_name'
+    """
+    sanitized = user_id.lower()
+
+    # Convert @ to _at_
+    sanitized = sanitized.replace("@", "_at_")
+
+    # Convert dots to underscores
+    sanitized = sanitized.replace(".", "_")
+
+    # Replace any non-alphanumeric characters with underscores
+    sanitized = re.sub(r'[^a-z0-9_]', '_', sanitized)
+
+    # Remove consecutive underscores
+    sanitized = re.sub(r'_+', '_', sanitized)
+
+    # Remove leading/trailing underscores
+    sanitized = sanitized.strip('_')
+
+    return sanitized
+
+
+def get_qdrant_collection_name(user_id: str) -> str:
+    """
+    Get Qdrant collection name for user.
+
+    Pattern: library_desk_{sanitized_user_id}
+
+    Args:
+        user_id: User identifier
+
+    Returns:
+        Qdrant collection name
+
+    Examples:
+        >>> get_qdrant_collection_name("jpmschweitzer")
+        'library_desk_jpmschweitzer'
+        >>> get_qdrant_collection_name("john@example.com")
+        'library_desk_john_at_example_com'
+    """
+    sanitized = sanitize_user_id(user_id)
+    return f"library_desk_{sanitized}"
+
+
+def get_wikijs_namespace(user_id: str) -> str:
+    """
+    Get Wiki.js namespace (path prefix) for user.
+
+    Pattern: /users/{sanitized_user_id}
+
+    All wiki pages for a user will be under this namespace.
+
+    Args:
+        user_id: User identifier
+
+    Returns:
+        Wiki.js path prefix
+
+    Examples:
+        >>> get_wikijs_namespace("jpmschweitzer")
+        '/users/jpmschweitzer'
+        >>> get_wikijs_namespace("john@example.com")
+        '/users/john_at_example_com'
+    """
+    sanitized = sanitize_user_id(user_id)
+    return f"/users/{sanitized}"
+
+
+def get_neo4j_user_base_label(user_id: str) -> str:
+    """
+    Get Neo4j base label for user's nodes (entities, etc).
+
+    Pattern: User_{Sanitized}
+
+    Uses title case for Neo4j label convention.
+
+    Args:
+        user_id: User identifier
+
+    Returns:
+        Neo4j base label for user's nodes
+
+    Examples:
+        >>> get_neo4j_user_base_label("jpmschweitzer")
+        'User_Jpmschweitzer'
+        >>> get_neo4j_user_base_label("john@example.com")
+        'User_John_At_Example_Com'
+    """
+    sanitized = sanitize_user_id(user_id)
+    # Title case each segment for Neo4j label convention
+    parts = sanitized.split('_')
+    titled = '_'.join(part.capitalize() for part in parts if part)
+    return f"User_{titled}"
+
+
+def get_neo4j_user_label(user_id: str) -> str:
+    """
+    Get Neo4j label for user's documents.
+
+    Pattern: User_{Sanitized}_Document
+
+    Uses title case for Neo4j label convention.
+
+    Args:
+        user_id: User identifier
+
+    Returns:
+        Neo4j label for user's document nodes
+
+    Examples:
+        >>> get_neo4j_user_label("jpmschweitzer")
+        'User_Jpmschweitzer_Document'
+        >>> get_neo4j_user_label("john@example.com")
+        'User_John_At_Example_Com_Document'
+    """
+    sanitized = sanitize_user_id(user_id)
+    # Title case each segment for Neo4j label convention
+    parts = sanitized.split('_')
+    titled = '_'.join(part.capitalize() for part in parts if part)
+    return f"User_{titled}_Document"
+
+
+def validate_user_id(user_id: str) -> bool:
+    """
+    Validate that a user ID is acceptable.
+
+    Checks:
+    - Not empty
+    - Not too long (max 100 chars)
+    - Contains some alphanumeric characters
+
+    Args:
+        user_id: User identifier to validate
+
+    Returns:
+        True if valid, False otherwise
+
+    Examples:
+        >>> validate_user_id("jpmschweitzer")
+        True
+        >>> validate_user_id("")
+        False
+        >>> validate_user_id("a" * 101)
+        False
+    """
+    if not user_id or len(user_id) > 100:
+        return False
+
+    # Must contain at least one alphanumeric character
+    if not re.search(r'[a-zA-Z0-9]', user_id):
+        return False
+
+    return True
+
+
+def is_path_in_user_namespace(path: str, user_id: str) -> bool:
+    """
+    Check if a Wiki.js path belongs to user's namespace.
+
+    Args:
+        path: Wiki.js page path
+        user_id: User identifier
+
+    Returns:
+        True if path is in user's namespace
+
+    Examples:
+        >>> is_path_in_user_namespace("/users/jpmschweitzer/projects", "jpmschweitzer")
+        True
+        >>> is_path_in_user_namespace("/users/other/projects", "jpmschweitzer")
+        False
+        >>> is_path_in_user_namespace("/public/docs", "jpmschweitzer")
+        False
+    """
+    namespace = get_wikijs_namespace(user_id)
+    return path.startswith(namespace)