jpmschweitzerandClaude Fable 5
51f9ce08ec
ROOT CAUSE (investigated read-only against prod): the production
container sets OLLAMA_MODEL=nomic-embed-text (the embedding model), which
the pre-rename generation setting also read, so every consolidation
/api/generate call failed with HTTP 400 ('"nomic-embed-text" does not
support generate' - confirmed in prod logs and by a direct Ollama probe).
_classify_web_results_unified swallowed that as an empty classification,
and consolidate_knowledge marked EVERY SearchQuery processed anyway -
permanently draining the queue with zero pages ever created. Live Neo4j
shows 197/200 SearchQuery nodes processed=true with no output; every
subsequent 30-minute run then logged 'No unprocessed searches found'.
The label/tenant scoping was NOT at fault: persistence writes both the
tenant label and the plain :SearchQuery label the loop matches on.
The model resolution itself was already fixed in Phase A (94482bc,
ollama_llm_model / OLLAMA_LLM_MODEL). This commit repairs the pipeline
defect that masked it:
- LLM infrastructure failure (no output from generate) now raises
ConsolidationLLMUnavailableError instead of returning an empty routing
- consolidate_knowledge leaves those searches UNPROCESSED for the next
run, aborts the rest of the batch (the LLM is down for all of them),
and reports searches_deferred
- unparseable-but-present model output is still consumed (avoids
retrying a bad prompt forever); low-web skips unchanged
- every run logs 'Consolidation run complete: searches_processed=N
searches_deferred=M duration_ms=X'; both fields added to the response
- lookback boundary is now timezone-aware UTC (Neo4j datetime() reads
naive strings as UTC, shifting the window on CET hosts)
10 new offline regression tests.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QbFZyDvYksazX6nYQYZ67L
105 lines
4.2 KiB
Python
105 lines
4.2 KiB
Python
"""
|
|
Knowledge Consolidation models for Librarian processing.
|
|
|
|
Used by the consolidation endpoint to process SearchQuery nodes
|
|
and consolidate knowledge into wiki pages.
|
|
"""
|
|
from pydantic import BaseModel, Field
|
|
from typing import List, Optional, Dict, Any
|
|
|
|
|
|
class ConsolidationRequest(BaseModel):
|
|
"""Request for knowledge consolidation from search results."""
|
|
process_limit: int = Field(default=10, ge=1, le=100, description="Max searches to process")
|
|
lookback_days: int = Field(default=7, ge=1, le=90, description="Process searches from last N days")
|
|
min_web_results: int = Field(default=2, ge=1, le=20, description="Minimum web results needed")
|
|
dry_run: bool = Field(default=False, description="If true, analyze but don't create pages")
|
|
|
|
|
|
class SearchQueryInfo(BaseModel):
|
|
"""Information about a search query to process."""
|
|
id: str
|
|
query: str
|
|
user: str
|
|
timestamp: str
|
|
total_results: int
|
|
web_count: int
|
|
keywords: List[str] = []
|
|
|
|
|
|
class ConsolidationResult(BaseModel):
|
|
"""Result of processing a single search query."""
|
|
search_id: str
|
|
query: str
|
|
pages_created: int = 0
|
|
pages_updated: int = 0
|
|
entities_added: int = 0
|
|
volatile_cached: int = 0
|
|
files_queued: int = 0
|
|
prefetch_registered: int = 0
|
|
error: Optional[str] = None
|
|
|
|
|
|
class ConsolidationResponse(BaseModel):
|
|
"""Response from knowledge consolidation."""
|
|
total_found: int = Field(description="Total unprocessed searches found")
|
|
processed_count: int = Field(description="Successfully processed searches")
|
|
pages_created: int = Field(description="New wiki pages created")
|
|
pages_updated: int = Field(description="Existing pages updated")
|
|
entities_added: int = Field(description="New entities added to graph")
|
|
volatile_cached: int = Field(default=0, description="Items cached to volatile storage")
|
|
files_queued: int = Field(default=0, description="Files queued for Paperless")
|
|
prefetch_registered: int = Field(default=0, description="Prefetch patterns registered")
|
|
searches_deferred: int = Field(
|
|
default=0,
|
|
description="Searches left unprocessed for the next run because the generation LLM was unavailable"
|
|
)
|
|
errors: List[str] = Field(default=[], description="Error messages")
|
|
results: List[ConsolidationResult] = Field(description="Per-search results")
|
|
dry_run: bool = Field(description="Whether this was a dry run")
|
|
duration_ms: float = Field(default=0.0, description="Run duration in milliseconds")
|
|
|
|
|
|
class MemoryRouteClassification(BaseModel):
|
|
"""
|
|
Unified classification of a web result for memory routing.
|
|
|
|
Route types:
|
|
- wiki: Stable reference content → wiki page creation/update
|
|
- volatile: Ephemeral data (weather, news, prices) → volatile cache
|
|
- file: Downloadable file (PDF, doc, xls, images) → Paperless ingestion
|
|
- prefetch: Regularly updated source → scheduler registration
|
|
- skip: Low value, ads, errors → discard
|
|
"""
|
|
url: str
|
|
title: str
|
|
route_type: str = Field(description="One of: wiki, volatile, file, prefetch, skip")
|
|
|
|
# Wiki routing fields
|
|
wiki_action: Optional[str] = Field(default=None, description="create or update")
|
|
wiki_path: Optional[str] = Field(default=None, description="Wiki path for page")
|
|
wiki_summary: Optional[str] = Field(default=None, description="Summary for wiki page")
|
|
|
|
# Volatile routing fields
|
|
volatile_namespace: Optional[str] = Field(default=None, description="weather, news, financial, etc.")
|
|
volatile_key: Optional[str] = Field(default=None, description="Cache key")
|
|
volatile_ttl_hours: Optional[int] = Field(default=None, description="TTL in hours")
|
|
|
|
# Prefetch routing fields
|
|
prefetch_cron: Optional[str] = Field(default=None, description="Cron expression for refresh")
|
|
prefetch_endpoint: Optional[str] = Field(default=None, description="API endpoint to call")
|
|
|
|
# Classification metadata
|
|
confidence: float = Field(default=0.0, ge=0.0, le=1.0)
|
|
reason: str = Field(default="")
|
|
|
|
|
|
class MemoryRoutingResult(BaseModel):
|
|
"""Aggregated result of memory routing for a search."""
|
|
wiki_routed: int = 0
|
|
volatile_cached: int = 0
|
|
files_queued: int = 0
|
|
prefetch_registered: int = 0
|
|
skipped: int = 0
|
|
classifications: List[MemoryRouteClassification] = []
|