feat: refactor volatile cache to vector storage with HybridRAG integration

- Migrate volatile backend from Redis to Qdrant for semantic search - Add natural language conversion for structured data embedding - Simplify API: /volatile/search, /volatile/store, /{namespace}/{key} - Integrate volatile into HybridRAG with priority boost in RRF fusion - Add POST /maintenance/cleanup/volatile for expiry purging - Update tests for new Qdrant-based architecture (37/37 pass) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-24 20:03:37 +01:00
parent 1f848c4878
commit 37f8e1819e
10 changed files with 1098 additions and 391 deletions
@@ -1,7 +1,7 @@
 """
 HybridRAG router for multi-source search API.

-Provides endpoint for combining vector, graph, and web search
+Provides endpoint for combining vector, graph, volatile cache, and web search
 with RRF fusion and LLM re-ranking.
 """

@@ -34,10 +34,12 @@ def get_hybrid_rag_service(
    """Get HybridRAG service instance with all dependencies."""
    from src.services.vector_service import VectorService
    from src.services.graph_service import GraphService
+    from src.services.volatile_service import VolatileCacheService

    # Create component services
    vector_service = VectorService(qdrant_client, wiki_client, ollama_client)
    graph_service = GraphService(neo4j_client, wiki_client)
+    volatile_service = VolatileCacheService(qdrant_client, ollama_client, settings)

    # Create HybridRAG service
    return HybridRAGService(
@@ -46,7 +48,8 @@ def get_hybrid_rag_service(
        searxng_client=searxng_client,
        ollama_client=ollama_client,
        content_extractor=content_extractor,
-        settings=settings
+        settings=settings,
+        volatile_service=volatile_service
    )


@@ -58,26 +61,28 @@ async def hybrid_search(
    api_key: str = Depends(verify_api_key)
 ):
    """
-    Execute HybridRAG query combining vector, graph, and web search.
+    Execute HybridRAG query combining vector, graph, volatile cache, and web search.

    **6-Phase Pipeline:**
    1. **Query Enhancement**: Extract keywords/synonyms with LLM
-    2. **Parallel Retrieval**: Search vector (Qdrant), graph (Neo4j), web (SearXNG)
-    3. **RRF Fusion**: Merge results with Reciprocal Rank Fusion
+    2. **Parallel Retrieval**: Search vector (Qdrant), graph (Neo4j), volatile cache, web (SearXNG)
+    3. **RRF Fusion**: Merge results with Reciprocal Rank Fusion (volatile gets priority boost)
    4. **Enrichment**: Add related documents via shared entities
-    5. **LLM Re-ranking**: Re-rank with mistral-nemo for relevance
+    5. **LLM Re-ranking**: Re-rank with configured model for relevance
    6. **Context Formatting**: Format for LLM consumption
    7. **Persistence**: Store for Librarian knowledge consolidation

    **Example Request:**
    ```json
    {
-        "query": "How does Docker orchestration work with Kubernetes?",
+        "query": "What's the weather in Rotterdam?",
        "user": "jpmschweitzer",
        "config": {
            "vector_limit": 10,
            "graph_limit": 10,
            "web_limit": 5,
+            "volatile_limit": 5,
+            "enable_volatile": true,
            "enable_reranking": true,
            "final_result_count": 10
        }
@@ -85,7 +90,7 @@ async def hybrid_search(
    ```

    **Returns:**
-    - Ranked results from all sources
+    - Ranked results from all sources (wiki, volatile, web)
    - Extracted keywords/synonyms
    - Related dossiers (via graph)
    - Formatted context for LLM
@@ -16,10 +16,12 @@ import time

 from src.services.vector_service import VectorService
 from src.services.graph_service import GraphService
+from src.services.volatile_service import VolatileCacheService
 from src.core.dependencies import (
    VectorServiceDep, GraphServiceDep, WikiJSDep, RedisDep,
-    verify_api_key
+    QdrantDep, OllamaDep, verify_api_key
 )
+from src.config import get_settings
 from datetime import datetime, timezone

 logger = logging.getLogger(__name__)
@@ -209,6 +211,15 @@ class ReconcileIndexResponse(BaseModel):
    total_duration_ms: float


+class VolatileCleanupResponse(BaseModel):
+    """Response from volatile cache cleanup operation."""
+    success: bool
+    collections_processed: int
+    total_expired_purged: int
+    by_collection: Dict[str, int] = Field(default_factory=dict)
+    duration_ms: float
+
+
 # ========== Endpoints ==========

@router.post("/cleanup/vectors", response_model=VectorCleanupResponse)
@@ -490,6 +501,61 @@ async def cleanup_all(
        raise HTTPException(status_code=500, detail=str(e))


+@router.post("/cleanup/volatile", response_model=VolatileCleanupResponse)
+async def cleanup_volatile(
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
+    api_key: str = Depends(verify_api_key)
+):
+    """
+    Purge expired volatile cache records across all users.
+
+    Loops through all volatile_* collections and removes records where
+    ttl_expiry < current_timestamp.
+
+    **Scheduler Task** - Recommended to run every 10 minutes.
+
+    **Scheduler Integration:**
+    ```json
+    {
+      "task_name": "volatile_cleanup",
+      "schedule": "*/10 * * * *",
+      "endpoint": "POST /maintenance/cleanup/volatile",
+      "description": "Purge expired volatile cache records"
+    }
+    ```
+    """
+    start_time = time.time()
+
+    try:
+        settings = get_settings()
+        service = VolatileCacheService(
+            qdrant_client=qdrant,
+            ollama_client=ollama,
+            settings=settings
+        )
+
+        # Purge expired from all volatile collections
+        results = await service.purge_all_expired()
+
+        total_purged = sum(results.values())
+        duration_ms = (time.time() - start_time) * 1000
+
+        logger.info(f"Volatile cleanup complete: {total_purged} expired records purged from {len(results)} collections")
+
+        return VolatileCleanupResponse(
+            success=True,
+            collections_processed=len(results),
+            total_expired_purged=total_purged,
+            by_collection=results,
+            duration_ms=duration_ms
+        )
+
+    except Exception as e:
+        logger.error(f"Volatile cleanup failed: {e}", exc_info=True)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
@router.get("/health", response_model=HealthCheckResponse)
 async def maintenance_health(
    user: str = Query(..., description="User identifier"),
@@ -2,6 +2,7 @@
 Volatile cache router for Library Desk API.

 Endpoints for ephemeral cached data with TTL - weather, news, financial, etc.
+Data is stored as vectors in Qdrant for semantic search retrieval.
 """

 from fastapi import APIRouter, HTTPException, Depends, Query
@@ -14,11 +15,11 @@ from src.models.volatile import (
    VolatileScheduledResponse,
    VolatileStatsResponse,
    VolatileDeleteResponse,
-    VolatileBulkDeleteResponse,
    VolatileNamespace,
+    NAMESPACE_DEFAULT_TTL,
 )
 from src.services.volatile_service import VolatileCacheService
-from src.core.dependencies import verify_api_key, RedisDep
+from src.core.dependencies import verify_api_key, QdrantDep, OllamaDep
 from src.core.multi_tenancy import DEFAULT_USER
 from src.config import get_settings

@@ -27,16 +28,21 @@ logger = logging.getLogger(__name__)
 router = APIRouter(prefix="/volatile", tags=["Volatile Cache"])


-def get_volatile_service(redis: RedisDep) -> VolatileCacheService:
+def get_volatile_service(qdrant: QdrantDep, ollama: OllamaDep) -> VolatileCacheService:
    """Get volatile cache service instance."""
    settings = get_settings()
-    return VolatileCacheService(redis_client=redis, settings=settings)
+    return VolatileCacheService(
+        qdrant_client=qdrant,
+        ollama_client=ollama,
+        settings=settings
+    )


@router.get("/stats", response_model=VolatileStatsResponse)
 async def get_stats(
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
@@ -44,14 +50,14 @@ async def get_stats(

    Returns counts of records by namespace and scheduled refresh info.
    """
-    service = get_volatile_service(redis)
+    service = get_volatile_service(qdrant, ollama)
    stats = await service.get_stats(user)

    return VolatileStatsResponse(
        total_records=stats["total_records"],
        by_namespace=stats["by_namespace"],
        scheduled_count=stats["scheduled_count"],
-        total_memory_bytes=stats.get("total_memory_bytes"),
+        total_memory_bytes=None,
        user=user,
    )

@@ -59,7 +65,8 @@ async def get_stats(
@router.get("/scheduled", response_model=VolatileScheduledResponse)
 async def get_scheduled(
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
@@ -68,7 +75,7 @@ async def get_scheduled(
    Used by scheduler to determine what volatile data needs refreshing.
    Returns all records that have a refresh_schedule cron expression set.
    """
-    service = get_volatile_service(redis)
+    service = get_volatile_service(qdrant, ollama)
    records = await service.get_scheduled(user)

    return VolatileScheduledResponse(
@@ -86,10 +93,7 @@ async def list_namespaces(
    List available namespaces and their default TTLs.

    Returns predefined namespaces with their default TTL values.
-    Custom namespaces can also be used with the default TTL.
    """
-    from src.models.volatile import NAMESPACE_DEFAULT_TTL
-
    return {
        "namespaces": [
            {
@@ -120,103 +124,62 @@ def _get_namespace_description(ns: VolatileNamespace) -> str:
    return descriptions.get(ns, "Custom namespace")


-@router.get("/{namespace}", response_model=VolatileListResponse)
-async def list_keys(
-    namespace: str,
+@router.get("/search")
+async def search_volatile(
+    q: str = Query(..., min_length=1, description="Search query"),
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    limit: int = Query(default=5, ge=1, le=20, description="Maximum results"),
+    threshold: float = Query(default=0.75, ge=0.5, le=1.0, description="Minimum similarity score"),
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
-    List all keys in a namespace.
+    Semantic search across volatile data.

-    Returns the list of keys stored in the specified namespace.
-    """
-    service = get_volatile_service(redis)
-    keys = await service.list_namespace(user, namespace)
-
-    return VolatileListResponse(
-        namespace=namespace,
-        keys=keys,
-        count=len(keys),
-        user=user,
-    )
-
-
-@router.delete("/{namespace}", response_model=VolatileBulkDeleteResponse)
-async def delete_namespace(
-    namespace: str,
-    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
-    api_key: str = Depends(verify_api_key)
-):
-    """
-    Delete all records in a namespace.
-
-    Removes all volatile data for the specified namespace.
-    """
-    service = get_volatile_service(redis)
-    deleted = await service.delete_namespace(user, namespace)
-
-    return VolatileBulkDeleteResponse(
-        namespace=namespace,
-        deleted_count=deleted,
-        user=user,
-    )
-
-
-@router.get("/{namespace}/{key}", response_model=VolatileRecordResponse)
-async def get_record(
-    namespace: str,
-    key: str,
-    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
-    api_key: str = Depends(verify_api_key)
-):
-    """
-    Get a volatile record.
-
-    Returns the record if it exists and has not expired.
+    Searches all volatile data for semantically similar content.
+    Higher threshold = stricter matching.

    **Example:**
    ```
-    GET /volatile/weather/rotterdam?user=jpmschweitzer
+    GET /volatile/search?q=weather%20rotterdam&user=jpmschweitzer
    ```
    """
-    service = get_volatile_service(redis)
-    record = await service.get(user, namespace, key)
+    service = get_volatile_service(qdrant, ollama)
+    results = await service.search(user, q, limit=limit, score_threshold=threshold)

-    if not record:
-        raise HTTPException(
-            status_code=404,
-            detail=f"Record '{key}' not found in namespace '{namespace}'"
-        )
-
-    return record
+    return {
+        "query": q,
+        "results": results,
+        "count": len(results),
+        "user": user,
+    }


-@router.post("/{namespace}/{key}", response_model=VolatileRecordResponse)
-async def set_record(
-    namespace: str,
-    key: str,
-    request: VolatileRecordCreate,
+@router.post("/store", response_model=VolatileRecordResponse)
+async def store_volatile(
+    namespace: str = Query(..., description="Data namespace (weather, news, etc.)"),
+    key: str = Query(..., description="Record key (e.g., 'rotterdam', 'nos-headlines')"),
+    request: VolatileRecordCreate = None,
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
-    Store or update a volatile record.
+    Store volatile data.

-    Creates or updates a record with the specified TTL.
-    If TTL is not provided, the namespace default is used.
+    Data is converted to natural language and embedded for semantic search.
+    If the same namespace+key already exists, it will be updated.

    **Example Request:**
    ```json
+    POST /volatile/store?namespace=weather&key=rotterdam
    {
        "data": {
-            "temperature": 18,
-            "conditions": "Partly cloudy",
-            "humidity": 65
+            "temperature": 8,
+            "conditions": "Cloudy",
+            "humidity": 85
        },
        "source": "openweathermap",
        "ttl": 1800,
@@ -228,10 +191,21 @@ async def set_record(
    Optional cron expression for automatic refresh. The scheduler
    will query `/volatile/scheduled` and trigger refreshes.
    """
-    service = get_volatile_service(redis)
+    # Validate namespace if not custom
+    if namespace != VolatileNamespace.CUSTOM:
+        try:
+            VolatileNamespace(namespace)
+        except ValueError:
+            valid = [ns.value for ns in VolatileNamespace]
+            raise HTTPException(
+                status_code=400,
+                detail=f"Invalid namespace '{namespace}'. Valid: {valid}"
+            )
+
+    service = get_volatile_service(qdrant, ollama)

    try:
-        record = await service.set(
+        record = await service.store(
            user=user,
            namespace=namespace,
            key=key,
@@ -244,7 +218,36 @@ async def set_record(

    except Exception as e:
        logger.error(f"Failed to store volatile record: {e}")
-        raise HTTPException(status_code=500, detail="Failed to store record")
+        raise HTTPException(status_code=500, detail=f"Failed to store record: {str(e)}")
+
+
+@router.get("/{namespace}/{key}", response_model=VolatileRecordResponse)
+async def get_record(
+    namespace: str,
+    key: str,
+    user: str = Query(default=DEFAULT_USER, description="User identifier"),
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
+    api_key: str = Depends(verify_api_key)
+):
+    """
+    Get a specific volatile record by namespace and key.
+
+    **Example:**
+    ```
+    GET /volatile/weather/rotterdam?user=jpmschweitzer
+    ```
+    """
+    service = get_volatile_service(qdrant, ollama)
+    record = await service.get(user, namespace, key)
+
+    if not record:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Record '{key}' not found in namespace '{namespace}'"
+        )
+
+    return record


@router.delete("/{namespace}/{key}", response_model=VolatileDeleteResponse)
@@ -252,15 +255,14 @@ async def delete_record(
    namespace: str,
    key: str,
    user: str = Query(default=DEFAULT_USER, description="User identifier"),
-    redis: RedisDep = None,
+    qdrant: QdrantDep = None,
+    ollama: OllamaDep = None,
    api_key: str = Depends(verify_api_key)
 ):
    """
-    Delete a volatile record.
-
-    Removes the record from the cache.
+    Delete a specific volatile record.
    """
-    service = get_volatile_service(redis)
+    service = get_volatile_service(qdrant, ollama)
    deleted = await service.delete(user, namespace, key)

    return VolatileDeleteResponse(