jpmschweitzerandClaude Fable 5
8a1c9ba5f3
A live /query/hybrid probe as user=llm_tester returned jpmschweitzer
pages. Audit of all legs (vector, graph, web-persistence, volatile,
documents) plus enrichment/persistence found and fixed these unscoped
paths:
- vector_service.update_from_page and graph_service.update_from_page now
refuse pages outside users/{user}/ - previously any tenant could
ingest any wiki page (incl. another tenant's) into its own collection
and graph labels, which is how foreign content entered the vector leg.
- ingestion_service.ingest_all_pages clamps path_prefix to the caller's
namespace (segment-exact, sanitized comparison) and defaults to
users/{user}; /ingest/all returns 400 on cross-tenant prefixes.
- hybrid_rag_service._persist_search_for_librarian linked SearchQuery
nodes to unscoped (d:Document {page_id}); now matches only
User_{Tenant}_Document nodes.
- graph_service: _get_entity_mention_count, entity-stub mention/related
queries, generate_entity_stubs, find/purge_orphan_entities matched
unscoped Document nodes; cleanup_broken_relationships matched all
tenants' SearchQuery nodes; _entity_has_wiki_page listed all wiki
pages. All are now tenant-label / namespace scoped.
- volatile_service collection names now use the sanitized user id.
- is_path_in_user_namespace enforces a path-segment boundary
(users/llm_tester2 is not llm_tester's namespace) and treats
hyphen/underscore tenant spellings as the same sanitized tenant.
- New offline unit tests per leg (mocked clients) assert the
tenant-scoped collection/label/path is used and cross-tenant access
is refused.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
572 lines
18 KiB
Python
572 lines
18 KiB
Python
"""
|
|
Volatile Cache service for Library Desk.
|
|
|
|
Provides ephemeral data storage with TTL using Qdrant vectors:
|
|
- Weather, news, financial data
|
|
- Transit schedules, traffic conditions
|
|
- System status, social notifications
|
|
|
|
Data is stored as embedded vectors for semantic search retrieval.
|
|
"""
|
|
|
|
import hashlib
|
|
import logging
|
|
import time
|
|
from datetime import datetime
|
|
from typing import List, Optional, Dict, Any
|
|
|
|
from src.clients.qdrant_client import QdrantClientWrapper
|
|
from src.clients.ollama_client import OllamaClient
|
|
from src.config import Settings
|
|
from src.models.volatile import (
|
|
VolatileRecordResponse,
|
|
VolatileNamespace,
|
|
NAMESPACE_DEFAULT_TTL,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class VolatileCacheService:
|
|
"""
|
|
Service for volatile data with TTL stored in Qdrant.
|
|
|
|
Stores ephemeral data as vectors for semantic search retrieval.
|
|
Each user has an isolated volatile collection.
|
|
"""
|
|
|
|
COLLECTION_PREFIX = "volatile_"
|
|
|
|
def __init__(
|
|
self,
|
|
qdrant_client: QdrantClientWrapper,
|
|
ollama_client: OllamaClient,
|
|
settings: Settings
|
|
):
|
|
"""
|
|
Initialize volatile cache service.
|
|
|
|
Args:
|
|
qdrant_client: Qdrant client for vector storage
|
|
ollama_client: Ollama client for embeddings
|
|
settings: Application settings
|
|
"""
|
|
self.qdrant = qdrant_client
|
|
self.ollama = ollama_client
|
|
self.settings = settings
|
|
|
|
logger.info("Initialized VolatileCacheService (Qdrant backend)")
|
|
|
|
def _collection_name(self, user: str) -> str:
|
|
"""
|
|
Get volatile collection name for user.
|
|
|
|
The user id is sanitized (same rules as the document collections)
|
|
so raw identifiers cannot alias or escape the per-tenant
|
|
collection naming scheme.
|
|
"""
|
|
from src.core.multi_tenancy import sanitize_user_id
|
|
return f"{self.COLLECTION_PREFIX}{sanitize_user_id(user)}"
|
|
|
|
def _make_vector_id(self, namespace: str, key: str) -> str:
|
|
"""
|
|
Generate deterministic vector ID for namespace/key.
|
|
|
|
Same namespace+key always produces same ID for upsert behavior.
|
|
"""
|
|
combined = f"{namespace}:{key}"
|
|
return hashlib.md5(combined.encode()).hexdigest()
|
|
|
|
def _get_default_ttl(self, namespace: str) -> int:
|
|
"""Get default TTL for a namespace."""
|
|
try:
|
|
ns = VolatileNamespace(namespace)
|
|
return NAMESPACE_DEFAULT_TTL.get(ns, self.settings.volatile_default_ttl)
|
|
except ValueError:
|
|
return self.settings.volatile_default_ttl
|
|
|
|
def _current_timestamp_ms(self) -> int:
|
|
"""Get current timestamp in milliseconds."""
|
|
return int(time.time() * 1000)
|
|
|
|
def _to_natural_language(
|
|
self,
|
|
namespace: str,
|
|
key: str,
|
|
data: Dict[str, Any]
|
|
) -> str:
|
|
"""
|
|
Convert structured data to natural language for embedding.
|
|
|
|
This creates a text representation that embeds well semantically.
|
|
"""
|
|
# Template-based conversion for known namespaces
|
|
if namespace == VolatileNamespace.WEATHER:
|
|
temp = data.get("temperature", data.get("temp", "unknown"))
|
|
conditions = data.get("conditions", data.get("weather", ""))
|
|
humidity = data.get("humidity", "")
|
|
text = f"Current weather in {key}: {temp}°C"
|
|
if conditions:
|
|
text += f", {conditions}"
|
|
if humidity:
|
|
text += f", humidity {humidity}%"
|
|
return text
|
|
|
|
elif namespace == VolatileNamespace.NEWS:
|
|
title = data.get("title", data.get("headline", ""))
|
|
summary = data.get("summary", data.get("description", ""))
|
|
source = data.get("source", "")
|
|
text = f"News: {title}"
|
|
if summary:
|
|
text += f". {summary}"
|
|
if source:
|
|
text += f" (Source: {source})"
|
|
return text
|
|
|
|
elif namespace == VolatileNamespace.FINANCIAL:
|
|
symbol = data.get("symbol", key)
|
|
price = data.get("price", "")
|
|
change = data.get("change", data.get("change_percent", ""))
|
|
text = f"Financial data for {symbol}"
|
|
if price:
|
|
text += f": price {price}"
|
|
if change:
|
|
text += f", change {change}%"
|
|
return text
|
|
|
|
elif namespace == VolatileNamespace.TRANSIT:
|
|
route = data.get("route", data.get("line", key))
|
|
status = data.get("status", "")
|
|
delay = data.get("delay", data.get("delay_minutes", ""))
|
|
text = f"Transit {route}"
|
|
if status:
|
|
text += f": {status}"
|
|
if delay:
|
|
text += f", delay {delay} minutes"
|
|
return text
|
|
|
|
elif namespace == VolatileNamespace.TRAFFIC:
|
|
location = data.get("location", key)
|
|
duration = data.get("duration", data.get("travel_time", ""))
|
|
congestion = data.get("congestion", "")
|
|
text = f"Traffic for {location}"
|
|
if duration:
|
|
text += f": {duration} minutes"
|
|
if congestion:
|
|
text += f", congestion level {congestion}"
|
|
return text
|
|
|
|
elif namespace == VolatileNamespace.AIR_QUALITY:
|
|
location = data.get("location", key)
|
|
aqi = data.get("aqi", data.get("index", ""))
|
|
quality = data.get("quality", "")
|
|
text = f"Air quality in {location}"
|
|
if aqi:
|
|
text += f": AQI {aqi}"
|
|
if quality:
|
|
text += f" ({quality})"
|
|
return text
|
|
|
|
elif namespace == VolatileNamespace.SPORTS:
|
|
event = data.get("event", data.get("match", key))
|
|
score = data.get("score", "")
|
|
status = data.get("status", "")
|
|
text = f"Sports: {event}"
|
|
if score:
|
|
text += f" - Score: {score}"
|
|
if status:
|
|
text += f" ({status})"
|
|
return text
|
|
|
|
elif namespace == VolatileNamespace.SYSTEM:
|
|
service = data.get("service", key)
|
|
status = data.get("status", "unknown")
|
|
message = data.get("message", "")
|
|
text = f"System status for {service}: {status}"
|
|
if message:
|
|
text += f". {message}"
|
|
return text
|
|
|
|
# Fallback: serialize key fields
|
|
text_parts = [f"{namespace} data for {key}:"]
|
|
for k, v in data.items():
|
|
if isinstance(v, (str, int, float, bool)):
|
|
text_parts.append(f"{k}: {v}")
|
|
return " ".join(text_parts)
|
|
|
|
async def store(
|
|
self,
|
|
user: str,
|
|
namespace: str,
|
|
key: str,
|
|
data: Dict[str, Any],
|
|
source: Optional[str] = None,
|
|
ttl: Optional[int] = None,
|
|
refresh_schedule: Optional[str] = None
|
|
) -> VolatileRecordResponse:
|
|
"""
|
|
Store volatile data as an embedded vector.
|
|
|
|
Args:
|
|
user: User identifier
|
|
namespace: Data namespace (from controlled list)
|
|
key: Record key (normalized slug)
|
|
data: Structured data to store
|
|
source: Origin API/service
|
|
ttl: TTL in seconds (uses namespace default if not set)
|
|
refresh_schedule: Optional cron expression for refresh
|
|
|
|
Returns:
|
|
The stored record
|
|
"""
|
|
collection = self._collection_name(user)
|
|
|
|
# Ensure collection exists
|
|
await self.qdrant.ensure_collection(collection)
|
|
|
|
# Calculate TTL and expiry
|
|
effective_ttl = ttl if ttl is not None else self._get_default_ttl(namespace)
|
|
now_ms = self._current_timestamp_ms()
|
|
expiry_ms = now_ms + (effective_ttl * 1000)
|
|
|
|
# Convert to natural language for embedding
|
|
text = self._to_natural_language(namespace, key, data)
|
|
|
|
# Generate embedding
|
|
embedding = await self.ollama.embed(text)
|
|
if not embedding:
|
|
raise ValueError("Failed to generate embedding for volatile data")
|
|
|
|
# Build payload
|
|
now = datetime.utcnow()
|
|
payload = {
|
|
"doc_type": "volatile",
|
|
"namespace": namespace,
|
|
"key": key,
|
|
"text": text,
|
|
"raw_data": data,
|
|
"source": source,
|
|
"created_at": now.isoformat(),
|
|
"updated_at": now.isoformat(),
|
|
"ttl": effective_ttl,
|
|
"ttl_expiry": expiry_ms,
|
|
"refresh_schedule": refresh_schedule,
|
|
"user": user,
|
|
}
|
|
|
|
# Upsert vector (same namespace+key = same ID = update)
|
|
vector_id = self._make_vector_id(namespace, key)
|
|
success = await self.qdrant.upsert_vector(
|
|
collection_name=collection,
|
|
vector_id=vector_id,
|
|
vector=embedding,
|
|
payload=payload
|
|
)
|
|
|
|
if not success:
|
|
raise ValueError("Failed to store volatile vector")
|
|
|
|
logger.debug(f"Stored volatile {namespace}:{key} with TTL {effective_ttl}s")
|
|
|
|
return VolatileRecordResponse(
|
|
key=key,
|
|
namespace=namespace,
|
|
data=data,
|
|
source=source,
|
|
created_at=now,
|
|
updated_at=now,
|
|
ttl=effective_ttl,
|
|
ttl_remaining=effective_ttl,
|
|
refresh_schedule=refresh_schedule,
|
|
user=user,
|
|
)
|
|
|
|
async def search(
|
|
self,
|
|
user: str,
|
|
query: str,
|
|
limit: int = 5,
|
|
score_threshold: float = 0.75
|
|
) -> List[VolatileRecordResponse]:
|
|
"""
|
|
Semantic search across volatile data.
|
|
|
|
Args:
|
|
user: User identifier
|
|
query: Search query
|
|
limit: Maximum results
|
|
score_threshold: Minimum similarity score (higher = stricter)
|
|
|
|
Returns:
|
|
List of matching volatile records
|
|
"""
|
|
collection = self._collection_name(user)
|
|
|
|
# Check if collection exists
|
|
if not await self.qdrant.collection_exists(collection):
|
|
return []
|
|
|
|
# Generate query embedding
|
|
query_embedding = await self.ollama.embed(query)
|
|
if not query_embedding:
|
|
logger.error("Failed to embed query for volatile search")
|
|
return []
|
|
|
|
# Search with expiry filter
|
|
now_ms = self._current_timestamp_ms()
|
|
results = await self.qdrant.search_with_expiry_filter(
|
|
collection_name=collection,
|
|
query_vector=query_embedding,
|
|
current_timestamp=now_ms,
|
|
limit=limit,
|
|
score_threshold=score_threshold
|
|
)
|
|
|
|
# Convert to response models
|
|
responses = []
|
|
for result in results:
|
|
payload = result["payload"]
|
|
ttl_expiry = payload.get("ttl_expiry", 0)
|
|
ttl_remaining = max(0, (ttl_expiry - now_ms) // 1000)
|
|
|
|
responses.append(VolatileRecordResponse(
|
|
key=payload["key"],
|
|
namespace=payload["namespace"],
|
|
data=payload.get("raw_data", {}),
|
|
source=payload.get("source"),
|
|
created_at=datetime.fromisoformat(payload["created_at"]),
|
|
updated_at=datetime.fromisoformat(payload["updated_at"]),
|
|
ttl=payload.get("ttl", 0),
|
|
ttl_remaining=ttl_remaining,
|
|
refresh_schedule=payload.get("refresh_schedule"),
|
|
user=payload["user"],
|
|
))
|
|
|
|
return responses
|
|
|
|
async def get(
|
|
self,
|
|
user: str,
|
|
namespace: str,
|
|
key: str
|
|
) -> Optional[VolatileRecordResponse]:
|
|
"""
|
|
Get a specific volatile record by namespace and key.
|
|
|
|
Args:
|
|
user: User identifier
|
|
namespace: Data namespace
|
|
key: Record key
|
|
|
|
Returns:
|
|
Record if found and not expired, None otherwise
|
|
"""
|
|
# Use search with high threshold to find exact match
|
|
query = self._to_natural_language(namespace, key, {"key": key})
|
|
results = await self.search(user, query, limit=10, score_threshold=0.5)
|
|
|
|
# Find exact namespace+key match
|
|
for result in results:
|
|
if result.namespace == namespace and result.key == key:
|
|
return result
|
|
|
|
return None
|
|
|
|
async def delete(
|
|
self,
|
|
user: str,
|
|
namespace: str,
|
|
key: str
|
|
) -> bool:
|
|
"""
|
|
Delete a specific volatile record.
|
|
|
|
Args:
|
|
user: User identifier
|
|
namespace: Data namespace
|
|
key: Record key
|
|
|
|
Returns:
|
|
True if deleted, False if not found
|
|
"""
|
|
collection = self._collection_name(user)
|
|
|
|
if not await self.qdrant.collection_exists(collection):
|
|
return False
|
|
|
|
vector_id = self._make_vector_id(namespace, key)
|
|
|
|
try:
|
|
deleted = await self.qdrant.delete_by_ids(
|
|
collection_name=collection,
|
|
point_ids=[vector_id]
|
|
)
|
|
return deleted > 0
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete volatile {namespace}:{key}: {e}")
|
|
return False
|
|
|
|
async def get_scheduled(
|
|
self,
|
|
user: str
|
|
) -> List[VolatileRecordResponse]:
|
|
"""
|
|
Get all records with refresh schedules.
|
|
|
|
Used by scheduler to determine what needs refreshing.
|
|
|
|
Args:
|
|
user: User identifier
|
|
|
|
Returns:
|
|
List of records with refresh_schedule set
|
|
"""
|
|
collection = self._collection_name(user)
|
|
|
|
if not await self.qdrant.collection_exists(collection):
|
|
return []
|
|
|
|
now_ms = self._current_timestamp_ms()
|
|
scheduled = []
|
|
|
|
# Scroll through all non-expired records
|
|
try:
|
|
all_points = await self.qdrant.scroll_all_points(
|
|
collection_name=collection,
|
|
with_payload=True
|
|
)
|
|
|
|
for point in all_points:
|
|
payload = point.get("payload", {})
|
|
ttl_expiry = payload.get("ttl_expiry", 0)
|
|
|
|
# Skip expired
|
|
if ttl_expiry <= now_ms:
|
|
continue
|
|
|
|
# Only include if has refresh schedule
|
|
if payload.get("refresh_schedule"):
|
|
ttl_remaining = max(0, (ttl_expiry - now_ms) // 1000)
|
|
scheduled.append(VolatileRecordResponse(
|
|
key=payload["key"],
|
|
namespace=payload["namespace"],
|
|
data=payload.get("raw_data", {}),
|
|
source=payload.get("source"),
|
|
created_at=datetime.fromisoformat(payload["created_at"]),
|
|
updated_at=datetime.fromisoformat(payload["updated_at"]),
|
|
ttl=payload.get("ttl", 0),
|
|
ttl_remaining=ttl_remaining,
|
|
refresh_schedule=payload["refresh_schedule"],
|
|
user=payload["user"],
|
|
))
|
|
|
|
return scheduled
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to get scheduled volatile records: {e}")
|
|
return []
|
|
|
|
async def get_stats(
|
|
self,
|
|
user: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Get cache statistics for user.
|
|
|
|
Args:
|
|
user: User identifier
|
|
|
|
Returns:
|
|
Statistics dict
|
|
"""
|
|
collection = self._collection_name(user)
|
|
|
|
if not await self.qdrant.collection_exists(collection):
|
|
return {
|
|
"total_records": 0,
|
|
"by_namespace": {},
|
|
"scheduled_count": 0,
|
|
"expired_count": 0,
|
|
}
|
|
|
|
now_ms = self._current_timestamp_ms()
|
|
by_namespace: Dict[str, int] = {}
|
|
total = 0
|
|
scheduled = 0
|
|
expired = 0
|
|
|
|
try:
|
|
all_points = await self.qdrant.scroll_all_points(
|
|
collection_name=collection,
|
|
with_payload=True
|
|
)
|
|
|
|
for point in all_points:
|
|
payload = point.get("payload", {})
|
|
namespace = payload.get("namespace", "unknown")
|
|
ttl_expiry = payload.get("ttl_expiry", 0)
|
|
|
|
if ttl_expiry <= now_ms:
|
|
expired += 1
|
|
else:
|
|
total += 1
|
|
by_namespace[namespace] = by_namespace.get(namespace, 0) + 1
|
|
if payload.get("refresh_schedule"):
|
|
scheduled += 1
|
|
|
|
return {
|
|
"total_records": total,
|
|
"by_namespace": by_namespace,
|
|
"scheduled_count": scheduled,
|
|
"expired_count": expired,
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to get volatile stats: {e}")
|
|
return {
|
|
"total_records": 0,
|
|
"by_namespace": {},
|
|
"scheduled_count": 0,
|
|
"expired_count": 0,
|
|
}
|
|
|
|
async def purge_expired(
|
|
self,
|
|
user: str
|
|
) -> int:
|
|
"""
|
|
Purge all expired volatile records for user.
|
|
|
|
Args:
|
|
user: User identifier
|
|
|
|
Returns:
|
|
Number of records purged
|
|
"""
|
|
collection = self._collection_name(user)
|
|
|
|
if not await self.qdrant.collection_exists(collection):
|
|
return 0
|
|
|
|
now_ms = self._current_timestamp_ms()
|
|
return await self.qdrant.delete_expired_vectors(collection, now_ms)
|
|
|
|
async def purge_all_expired(self) -> Dict[str, int]:
|
|
"""
|
|
Purge expired records from all volatile collections.
|
|
|
|
Returns:
|
|
Dict of collection -> purged count
|
|
"""
|
|
collections = await self.qdrant.get_volatile_collections()
|
|
results = {}
|
|
now_ms = self._current_timestamp_ms()
|
|
|
|
for collection in collections:
|
|
purged = await self.qdrant.delete_expired_vectors(collection, now_ms)
|
|
if purged > 0:
|
|
results[collection] = purged
|
|
logger.info(f"Purged {purged} expired from {collection}")
|
|
|
|
return results
|