Files
library-desk/src/routers/volatile.py
T
jpmschweitzerandClaude Opus 4.5 37f8e1819e
Build and Push / build (release) Successful in 28s
feat: refactor volatile cache to vector storage with HybridRAG integration
- Migrate volatile backend from Redis to Qdrant for semantic search
- Add natural language conversion for structured data embedding
- Simplify API: /volatile/search, /volatile/store, /{namespace}/{key}
- Integrate volatile into HybridRAG with priority boost in RRF fusion
- Add POST /maintenance/cleanup/volatile for expiry purging
- Update tests for new Qdrant-based architecture (37/37 pass)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-24 20:03:37 +01:00

274 lines
8.3 KiB
Python

"""
Volatile cache router for Library Desk API.
Endpoints for ephemeral cached data with TTL - weather, news, financial, etc.
Data is stored as vectors in Qdrant for semantic search retrieval.
"""
from fastapi import APIRouter, HTTPException, Depends, Query
import logging
from src.models.volatile import (
VolatileRecordCreate,
VolatileRecordResponse,
VolatileListResponse,
VolatileScheduledResponse,
VolatileStatsResponse,
VolatileDeleteResponse,
VolatileNamespace,
NAMESPACE_DEFAULT_TTL,
)
from src.services.volatile_service import VolatileCacheService
from src.core.dependencies import verify_api_key, QdrantDep, OllamaDep
from src.core.multi_tenancy import DEFAULT_USER
from src.config import get_settings
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/volatile", tags=["Volatile Cache"])
def get_volatile_service(qdrant: QdrantDep, ollama: OllamaDep) -> VolatileCacheService:
"""Get volatile cache service instance."""
settings = get_settings()
return VolatileCacheService(
qdrant_client=qdrant,
ollama_client=ollama,
settings=settings
)
@router.get("/stats", response_model=VolatileStatsResponse)
async def get_stats(
user: str = Query(default=DEFAULT_USER, description="User identifier"),
qdrant: QdrantDep = None,
ollama: OllamaDep = None,
api_key: str = Depends(verify_api_key)
):
"""
Get volatile cache statistics.
Returns counts of records by namespace and scheduled refresh info.
"""
service = get_volatile_service(qdrant, ollama)
stats = await service.get_stats(user)
return VolatileStatsResponse(
total_records=stats["total_records"],
by_namespace=stats["by_namespace"],
scheduled_count=stats["scheduled_count"],
total_memory_bytes=None,
user=user,
)
@router.get("/scheduled", response_model=VolatileScheduledResponse)
async def get_scheduled(
user: str = Query(default=DEFAULT_USER, description="User identifier"),
qdrant: QdrantDep = None,
ollama: OllamaDep = None,
api_key: str = Depends(verify_api_key)
):
"""
Get records with refresh schedules.
Used by scheduler to determine what volatile data needs refreshing.
Returns all records that have a refresh_schedule cron expression set.
"""
service = get_volatile_service(qdrant, ollama)
records = await service.get_scheduled(user)
return VolatileScheduledResponse(
records=records,
count=len(records),
user=user,
)
@router.get("/namespaces")
async def list_namespaces(
api_key: str = Depends(verify_api_key)
):
"""
List available namespaces and their default TTLs.
Returns predefined namespaces with their default TTL values.
"""
return {
"namespaces": [
{
"name": ns.value,
"default_ttl": NAMESPACE_DEFAULT_TTL.get(ns, 3600),
"description": _get_namespace_description(ns),
}
for ns in VolatileNamespace
]
}
def _get_namespace_description(ns: VolatileNamespace) -> str:
"""Get human-readable description for namespace."""
descriptions = {
VolatileNamespace.WEATHER: "Weather conditions and forecasts",
VolatileNamespace.NEWS: "Headlines and breaking news",
VolatileNamespace.FINANCIAL: "Stock prices, exchange rates, crypto",
VolatileNamespace.TRANSIT: "Train/bus schedules, delays",
VolatileNamespace.TRAFFIC: "Commute times, road conditions",
VolatileNamespace.AIR_QUALITY: "Pollution levels, pollen counts",
VolatileNamespace.SPORTS: "Live scores, upcoming matches",
VolatileNamespace.SOCIAL: "Social media mentions, notifications",
VolatileNamespace.SYSTEM: "Service health, infrastructure status",
VolatileNamespace.CONTEXT: "Conversation context, session state",
VolatileNamespace.CUSTOM: "User-defined volatile data",
}
return descriptions.get(ns, "Custom namespace")
@router.get("/search")
async def search_volatile(
q: str = Query(..., min_length=1, description="Search query"),
user: str = Query(default=DEFAULT_USER, description="User identifier"),
limit: int = Query(default=5, ge=1, le=20, description="Maximum results"),
threshold: float = Query(default=0.75, ge=0.5, le=1.0, description="Minimum similarity score"),
qdrant: QdrantDep = None,
ollama: OllamaDep = None,
api_key: str = Depends(verify_api_key)
):
"""
Semantic search across volatile data.
Searches all volatile data for semantically similar content.
Higher threshold = stricter matching.
**Example:**
```
GET /volatile/search?q=weather%20rotterdam&user=jpmschweitzer
```
"""
service = get_volatile_service(qdrant, ollama)
results = await service.search(user, q, limit=limit, score_threshold=threshold)
return {
"query": q,
"results": results,
"count": len(results),
"user": user,
}
@router.post("/store", response_model=VolatileRecordResponse)
async def store_volatile(
namespace: str = Query(..., description="Data namespace (weather, news, etc.)"),
key: str = Query(..., description="Record key (e.g., 'rotterdam', 'nos-headlines')"),
request: VolatileRecordCreate = None,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
qdrant: QdrantDep = None,
ollama: OllamaDep = None,
api_key: str = Depends(verify_api_key)
):
"""
Store volatile data.
Data is converted to natural language and embedded for semantic search.
If the same namespace+key already exists, it will be updated.
**Example Request:**
```json
POST /volatile/store?namespace=weather&key=rotterdam
{
"data": {
"temperature": 8,
"conditions": "Cloudy",
"humidity": 85
},
"source": "openweathermap",
"ttl": 1800,
"refresh_schedule": "0 * * * *"
}
```
**Refresh Schedule:**
Optional cron expression for automatic refresh. The scheduler
will query `/volatile/scheduled` and trigger refreshes.
"""
# Validate namespace if not custom
if namespace != VolatileNamespace.CUSTOM:
try:
VolatileNamespace(namespace)
except ValueError:
valid = [ns.value for ns in VolatileNamespace]
raise HTTPException(
status_code=400,
detail=f"Invalid namespace '{namespace}'. Valid: {valid}"
)
service = get_volatile_service(qdrant, ollama)
try:
record = await service.store(
user=user,
namespace=namespace,
key=key,
data=request.data,
source=request.source,
ttl=request.ttl,
refresh_schedule=request.refresh_schedule,
)
return record
except Exception as e:
logger.error(f"Failed to store volatile record: {e}")
raise HTTPException(status_code=500, detail=f"Failed to store record: {str(e)}")
@router.get("/{namespace}/{key}", response_model=VolatileRecordResponse)
async def get_record(
namespace: str,
key: str,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
qdrant: QdrantDep = None,
ollama: OllamaDep = None,
api_key: str = Depends(verify_api_key)
):
"""
Get a specific volatile record by namespace and key.
**Example:**
```
GET /volatile/weather/rotterdam?user=jpmschweitzer
```
"""
service = get_volatile_service(qdrant, ollama)
record = await service.get(user, namespace, key)
if not record:
raise HTTPException(
status_code=404,
detail=f"Record '{key}' not found in namespace '{namespace}'"
)
return record
@router.delete("/{namespace}/{key}", response_model=VolatileDeleteResponse)
async def delete_record(
namespace: str,
key: str,
user: str = Query(default=DEFAULT_USER, description="User identifier"),
qdrant: QdrantDep = None,
ollama: OllamaDep = None,
api_key: str = Depends(verify_api_key)
):
"""
Delete a specific volatile record.
"""
service = get_volatile_service(qdrant, ollama)
deleted = await service.delete(user, namespace, key)
return VolatileDeleteResponse(
key=key,
namespace=namespace,
deleted=deleted,
user=user,
)