Compare commits
+8
-3
@@ -8,7 +8,14 @@ API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
API_PREFIX=/v1
|
||||
|
||||
# Ollama Configuration
|
||||
# Anthropic Configuration (Claude - preferred backend)
|
||||
# Set ANTHROPIC_API_KEY to enable Claude as the default backend
|
||||
# Without an API key, Tatlock uses Ollama exclusively
|
||||
# ANTHROPIC_API_KEY=sk-ant-api03-your-key-here
|
||||
ANTHROPIC_MODEL=claude-sonnet-4-20250514
|
||||
PREFER_CLOUD_BACKEND=true
|
||||
|
||||
# Ollama Configuration (local fallback when Claude unavailable)
|
||||
OLLAMA_HOST=http://localhost:11434
|
||||
OLLAMA_DEFAULT_MODEL=mistral-nemo:latest
|
||||
OLLAMA_TIMEOUT=120
|
||||
@@ -21,7 +28,6 @@ SEARXNG_TIMEOUT=30
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
REDIS_MEMORY_DB=1
|
||||
REDIS_BENCHMARK_DB=6
|
||||
REDIS_TIMEOUT=5
|
||||
|
||||
# Qdrant Configuration
|
||||
@@ -33,7 +39,6 @@ QDRANT_PORT=6333
|
||||
# - development: DEBUG (maximum verbosity)
|
||||
# - production: WARNING (minimal noise)
|
||||
# Uncomment to override: LOG_LEVEL=INFO
|
||||
ENABLE_BENCHMARKS=true
|
||||
# Note: Log format is auto-selected based on ENVIRONMENT (console for dev, json for production)
|
||||
|
||||
# User Configuration
|
||||
|
||||
@@ -5,6 +5,17 @@ on:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create Gitea Release
|
||||
run: |
|
||||
curl -sf -X POST \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"tag_name": "${{ github.ref_name }}", "name": "Release ${{ github.ref_name }}", "body": "Automated release for ${{ github.ref_name }}"}' \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases"
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
+4
-1
@@ -68,7 +68,10 @@ dmypy.json
|
||||
.ruff_cache/
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
logs/*
|
||||
!logs/traces/
|
||||
logs/traces/*
|
||||
!logs/traces/viewer.html
|
||||
*.log
|
||||
|
||||
# Database
|
||||
|
||||
+144
-1
@@ -7,6 +7,137 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.0.1] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Expert agent registration failure** - `AnthropicModel` does not accept `api_key` directly; now passes it via `AnthropicProvider`
|
||||
|
||||
## [2.0.0] - 2026-02-05
|
||||
|
||||
### Added
|
||||
|
||||
- **Claude backend support (Claudification Phase 1)** - All agents now prefer Claude over Ollama
|
||||
- New `src/anthropic/` module with model selector and health check
|
||||
- `get_model()` factory returns Claude if available, Ollama as fallback
|
||||
- Startup health check caches Claude API availability
|
||||
- Configuration: `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `PREFER_CLOUD_BACKEND`
|
||||
- 200k token context when using Claude backend
|
||||
|
||||
- **Steward dual-backend support** - Direct API calls to Claude or Ollama
|
||||
- `_call_claude()`: Anthropic Messages API path
|
||||
- `_call_ollama()`: Existing Ollama generate API path (preserved)
|
||||
- Automatic fallback: if Claude call fails mid-request, retries with Ollama
|
||||
|
||||
- **Claudification project tracking** - `PROJECT_CLAUDIFICATION.md` with Phase 1/2 roadmap
|
||||
|
||||
### Changed
|
||||
|
||||
- **All PydanticAI agents refactored to use `get_model()`**:
|
||||
- Tatlock (6 instantiation locations)
|
||||
- Librarian
|
||||
- Biographer
|
||||
- Housekeeper
|
||||
- **`initialize_application()` is now async** - Supports async Claude health check at startup
|
||||
- **Dependencies**: `pydantic-ai-slim[openai,anthropic]` replaces `pydantic-ai-slim[openai]`
|
||||
- **Startup logging** now includes backend selection info (claude/ollama)
|
||||
- **Agent creation logging** now includes backend and model info
|
||||
|
||||
### Removed
|
||||
|
||||
- Stale `tests/core/test_benchmarks.py` (benchmark system was removed in v1.10.0)
|
||||
|
||||
## [1.11.0] - 2025-12-30
|
||||
|
||||
### Added
|
||||
|
||||
- **Paperless document integration** - HybridRAG now includes indexed PDFs and scanned documents from Paperless-ngx
|
||||
- New `include_documents` parameter in `hybrid_search` tool
|
||||
- 📑 icon for document sources in search results
|
||||
- Librarian prompt updated with document awareness
|
||||
|
||||
- **Volatile cache integration** - HybridRAG now includes pre-fetched real-time data
|
||||
- New `include_volatile` parameter in `hybrid_search` tool
|
||||
- ⚡ icon for volatile sources in search results
|
||||
- Supports weather, forecast, news, stock, crypto, sun, air_quality namespaces
|
||||
- Librarian prompt updated with volatile cache awareness (user-configured items only)
|
||||
|
||||
- **Biographer routing in Steward** - Personal memory queries now correctly route to The Biographer
|
||||
- Added explicit routing rules for "where do I live", "what car do I drive", etc.
|
||||
- Added biographer delegation examples to Steward prompt
|
||||
- Location keywords ("live", "where", "home") now trigger profile pre-fetch
|
||||
|
||||
### Changed
|
||||
|
||||
- **LibraryDeskClient.hybrid_search** - Now passes full config including `document_limit`, `volatile_limit`, and enable flags
|
||||
- **Steward guidelines** - Clarified that research queries about TOPICS go to Librarian, queries about USER go to Biographer
|
||||
|
||||
## [1.10.1] - 2025-12-23
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Tatlock's excessive apologizing** - Strengthened personality prompt to prevent unnecessary apologies after successful Librarian delegations. Added explicit "do NOT apologize" instructions to both system prompt and synthesis prompt.
|
||||
|
||||
## [1.10.0] - 2025-12-22
|
||||
|
||||
### Added
|
||||
|
||||
#### Lightweight Request Tracing
|
||||
- **JSON-based tracing system** for local development debugging
|
||||
- Captures full request flow through multi-agent architecture
|
||||
- `Trace` and `Span` dataclasses with automatic timing and nesting
|
||||
- ContextVar-based propagation for async-safe tracing
|
||||
- `trace_span` async context manager for clean instrumentation
|
||||
- Traces written to `logs/traces/{trace_id}.json`
|
||||
- Enabled via `DEBUG=true` environment variable
|
||||
- **Trace Viewer UI** (`logs/traces/viewer.html`)
|
||||
- Standalone HTML viewer with timeline visualization
|
||||
- Filter by status, search by request text
|
||||
- Expandable span details with prompts and responses
|
||||
- **Tracing REST API** (`/traces`)
|
||||
- `GET /traces` - Serve trace viewer UI
|
||||
- `GET /traces/list` - List available traces with filtering
|
||||
- `GET /traces/{trace_id}` - Retrieve specific trace JSON
|
||||
- Only available when `DEBUG=true`
|
||||
- **Full pipeline instrumentation**
|
||||
- Router-level trace start/end with context management
|
||||
- Steward analysis spans in preprocessing
|
||||
- Tatlock orchestrate/synthesize spans
|
||||
- Expert delegation spans (librarian/biographer/housekeeper)
|
||||
- Tool-level spans extracted from PydanticAI messages
|
||||
|
||||
### Changed
|
||||
|
||||
- **Replaced Redis benchmarks with file-based tracing** - Simpler, more useful for debugging
|
||||
- **Context management moved to service layer** - Router simplified, context set in response service
|
||||
- **Server binds to all interfaces** - `wakeup.sh` now uses `0.0.0.0` for network access
|
||||
|
||||
### Removed
|
||||
|
||||
- **Redis benchmark system** (`src/core/benchmarks.py`)
|
||||
- `ENABLE_BENCHMARKS` config setting
|
||||
- `REDIS_BENCHMARK_DB` config setting
|
||||
- `redis_url` property (kept `redis_memory_url`)
|
||||
- Benchmark recording in Steward service and tool tracking
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Librarian fabrication prevention** - Added explicit instructions to never invent data when tools fail or sources are unavailable
|
||||
|
||||
## [1.9.0] - 2025-12-18
|
||||
|
||||
### Changed
|
||||
|
||||
- **Housekeeper prompt optimization** - Rewrote system prompt for Mistral-Nemo function calling with negative constraints, step-by-step process, and explicit entity ID format guidance
|
||||
- **Housekeeper temperature setting** - Set temperature to 0.1 for deterministic tool calling behavior
|
||||
- **Device list room group priority** - Room groups now appear first in `list_devices` output with `[ROOM GROUP]` marker to address positional bias
|
||||
- **Tool docstring improvements** - Updated turn_on/turn_off/toggle with explicit `entity_id=` parameter examples
|
||||
|
||||
### Added
|
||||
|
||||
- **Housekeeper optimization findings** - Added `docs/housekeeper-optimization-findings.md` documenting the experiment journey from 0% to 100% success rate
|
||||
- **Housekeeper test script** - Added `scripts/test_housekeeper.sh` for room group detection regression testing
|
||||
|
||||
## [1.8.6] - 2025-12-17
|
||||
|
||||
### Fixed
|
||||
@@ -765,7 +896,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- CORS middleware
|
||||
- Exception handlers (OpenAI-compatible error format)
|
||||
|
||||
[Unreleased]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.6.0...main
|
||||
[Unreleased]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.0.0...main
|
||||
[2.0.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.11.0...v2.0.0
|
||||
[1.11.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.10.0...v1.11.0
|
||||
[1.10.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.9.0...v1.10.0
|
||||
[1.9.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.6...v1.9.0
|
||||
[1.8.6]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.5...v1.8.6
|
||||
[1.8.5]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.4...v1.8.5
|
||||
[1.8.4]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.3...v1.8.4
|
||||
[1.8.3]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.2...v1.8.3
|
||||
[1.8.2]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.1...v1.8.2
|
||||
[1.8.1]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.0...v1.8.1
|
||||
[1.8.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.7.0...v1.8.0
|
||||
[1.7.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.6.0...v1.7.0
|
||||
[1.6.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.5.0...v1.6.0
|
||||
[1.5.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.4.0...v1.5.0
|
||||
[1.4.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.3.3...v1.4.0
|
||||
|
||||
@@ -0,0 +1,401 @@
|
||||
# Tatlock Enhancement Plan: Bidirectional Claude Integration
|
||||
|
||||
## Executive Summary
|
||||
|
||||
Implement a **bidirectional architecture** that:
|
||||
1. **Superpowers Tatlock** by swapping Ollama→Claude backend (200k context, better reasoning, same butler personality)
|
||||
2. **Exposes Tatlock as MCP server** for Claude instances on any device (phone, browser, desktop)
|
||||
|
||||
This gives you the flexibility to use whichever AI is best/most accessible at any moment.
|
||||
|
||||
## Key Insight: Blanket Backend Swap (Simpler Than Sidecar)
|
||||
|
||||
Instead of adding a Claude "Analyst" sidecar agent, **swap the underlying model for ALL agents**:
|
||||
|
||||
```
|
||||
CURRENT: TatlockAgent → OpenAIChatModel → OllamaProvider → Ollama (mistral-nemo)
|
||||
PROPOSED: TatlockAgent → AnthropicModel → AnthropicProvider → Claude API
|
||||
↘ (fallback when offline) → OllamaProvider → Ollama
|
||||
```
|
||||
|
||||
**Why this works:**
|
||||
- PydanticAI natively supports Anthropic via `AnthropicModel` + `AnthropicProvider`
|
||||
- The same `TATLOCK_SYSTEM_PROMPT` is passed to Claude - butler personality preserved
|
||||
- Claude is **better** at following system prompts than mistral-nemo
|
||||
- 200k context for ALL queries, not just "complex" ones
|
||||
- Simpler architecture: no routing logic, no sidecar delegation
|
||||
|
||||
---
|
||||
|
||||
## Research Findings
|
||||
|
||||
### Industry Best Practices (2025-2026)
|
||||
|
||||
**MCP Protocol Updates** ([MCP Spec Updates June 2025](https://auth0.com/blog/mcp-specs-update-all-about-auth/)):
|
||||
- Streamable HTTP replaced SSE (March 2025) - better for cloud deployment
|
||||
- OAuth 2.0 required for remote servers - MCP servers are OAuth Resource Servers
|
||||
- Tool Output Schemas now available - better structured data handling
|
||||
- MCP Registry launched (Sept 2025) - community server discovery
|
||||
|
||||
**Community Patterns** ([Claude Code Router](https://github.com/musistudio/claude-code-router)):
|
||||
- Task-based routing is becoming standard: route simple→local, complex→cloud
|
||||
- Translation proxies bridge Anthropic Messages API ↔ OpenAI format
|
||||
- Cost savings of up to 98% reported with smart routing
|
||||
|
||||
**Home Automation MCP** ([ha-mcp](https://github.com/homeassistant-ai/ha-mcp)):
|
||||
- Production-ready MCP servers exist for Home Assistant
|
||||
- Support Claude Code, Gemini CLI, Open WebUI, VSCode, Cursor
|
||||
- Pattern: expose local tools securely to remote AI clients
|
||||
|
||||
**Remote MCP Access** ([mcp-remote](https://www.npmjs.com/package/mcp-remote)):
|
||||
- Bridge local MCP servers to Claude Desktop/Browser via proxy
|
||||
- Supports authentication headers for security
|
||||
- Works with ngrok/Cloudflare Tunnel for HTTPS
|
||||
|
||||
---
|
||||
|
||||
## Recommended Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ BIDIRECTIONAL TATLOCK-CLAUDE ARCHITECTURE │
|
||||
├─────────────────────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ╔═══════════════════════════════════════════════════════════════════════════════╗ │
|
||||
│ ║ SCENARIO A: Using Tatlock (Open WebUI, local apps) ║ │
|
||||
│ ║ ───────────────────────────────────────────────── ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Request → Steward → Tatlock → Tools + Expert Delegation ║ │
|
||||
│ ║ │ ║ │
|
||||
│ ║ ├─→ Librarian (Claude) → research, wiki, RAG ║ │
|
||||
│ ║ ├─→ Biographer (Claude) → memory, preferences ║ │
|
||||
│ ║ ├─→ Housekeeper (Claude) → home automation ║ │
|
||||
│ ║ └─→ All powered by Claude with Ollama fallback ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Butler personality preserved, 200k context for all queries ║ │
|
||||
│ ╚═══════════════════════════════════════════════════════════════════════════════╝ │
|
||||
│ │
|
||||
│ ╔═══════════════════════════════════════════════════════════════════════════════╗ │
|
||||
│ ║ SCENARIO B: Using Claude.ai / Claude Desktop / Phone ║ │
|
||||
│ ║ ──────────────────────────────────────────────────── ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Claude ──[MCP over HTTPS]──► Tatlock MCP Server → Household Tools ║ │
|
||||
│ ║ │ ║ │
|
||||
│ ║ ├─→ calculator, datetime ║ │
|
||||
│ ║ ├─→ web_search, wiki_search ║ │
|
||||
│ ║ ├─→ hybrid_search (RAG) ║ │
|
||||
│ ║ ├─→ memory_recall, store_insight ║ │
|
||||
│ ║ └─→ home_control (lights, climate) ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Full 200k context, your local tools accessible from anywhere ║ │
|
||||
│ ╚═══════════════════════════════════════════════════════════════════════════════╝ │
|
||||
│ │
|
||||
│ ╔═══════════════════════════════════════════════════════════════════════════════╗ │
|
||||
│ ║ SCENARIO C: Offline (internet down) ║ │
|
||||
│ ║ ────────────────────────────────── ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Tatlock operates fully locally with Ollama ║ │
|
||||
│ ║ • All tools work (except web search) ║ │
|
||||
│ ║ • Graceful degradation with same butler personality ║ │
|
||||
│ ╚═══════════════════════════════════════════════════════════════════════════════╝ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: Blanket Backend Swap (Claude for All Agents)
|
||||
|
||||
Replace Ollama with Claude as the default backend for all PydanticAI agents, with automatic offline fallback.
|
||||
|
||||
**New Files:**
|
||||
```
|
||||
src/anthropic/
|
||||
├── __init__.py
|
||||
├── provider.py # Claude provider with health check
|
||||
└── model_selector.py # Chooses Claude or Ollama based on availability
|
||||
```
|
||||
|
||||
**Key Implementation (`src/anthropic/provider.py`):**
|
||||
```python
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
from pydantic_ai.providers.anthropic import AnthropicProvider
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from src.core.config import config
|
||||
|
||||
_anthropic_available: bool | None = None
|
||||
|
||||
async def check_anthropic_health() -> bool:
|
||||
"""Check if Anthropic API is reachable."""
|
||||
global _anthropic_available
|
||||
try:
|
||||
from anthropic import AsyncAnthropic
|
||||
client = AsyncAnthropic(api_key=config.ANTHROPIC_API_KEY)
|
||||
await client.messages.create(
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
max_tokens=1,
|
||||
messages=[{"role": "user", "content": "hi"}]
|
||||
)
|
||||
_anthropic_available = True
|
||||
except Exception:
|
||||
_anthropic_available = False
|
||||
return _anthropic_available
|
||||
|
||||
def get_model(prefer_cloud: bool = True):
|
||||
"""Get the best available model. Returns Claude if available, otherwise Ollama."""
|
||||
if prefer_cloud and config.ANTHROPIC_API_KEY and _anthropic_available:
|
||||
provider = AnthropicProvider(api_key=config.ANTHROPIC_API_KEY)
|
||||
return AnthropicModel(
|
||||
model_name=config.ANTHROPIC_MODEL,
|
||||
provider=provider,
|
||||
)
|
||||
else:
|
||||
return OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
```
|
||||
|
||||
**Modify TatlockAgent (`src/agents/tatlock.py`):**
|
||||
```python
|
||||
def _ensure_agent(self):
|
||||
if self._agent is not None:
|
||||
return
|
||||
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
model = get_model(prefer_cloud=True)
|
||||
|
||||
self._agent = Agent(
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT, # Same butler personality!
|
||||
)
|
||||
self._register_tools()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: MCP Server (Expose Tools to Claude)
|
||||
|
||||
Create an MCP server that exposes Tatlock's household tools to external Claude instances.
|
||||
|
||||
**New Files:**
|
||||
```
|
||||
src/mcp/
|
||||
├── __init__.py
|
||||
├── server.py # MCP server using mcp Python SDK
|
||||
├── tool_adapters.py # Convert PydanticAI tools → MCP schemas
|
||||
├── auth.py # API key authentication
|
||||
└── transport.py # Streamable HTTP transport
|
||||
```
|
||||
|
||||
**Docker Stack Addition (`stacks/agents.yml`):**
|
||||
```yaml
|
||||
tatlock-mcp:
|
||||
image: git.schweitz.internal/jpmschweitzer/tatlock:latest
|
||||
command: ["python", "-m", "src.mcp.server"]
|
||||
ports:
|
||||
- "8002:8002"
|
||||
environment:
|
||||
- MCP_AUTH_TOKEN=${MCP_AUTH_TOKEN}
|
||||
networks:
|
||||
- docker-dataplane
|
||||
```
|
||||
|
||||
**Claude Desktop Configuration:**
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"tatlock": {
|
||||
"command": "npx",
|
||||
"args": ["mcp-remote", "https://mcp.schweitz.net/sse", "--header", "Authorization: Bearer ${MCP_AUTH_TOKEN}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Files to Modify
|
||||
|
||||
### Phase 1 - Backend Swap
|
||||
|
||||
**New Files:**
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `src/anthropic/__init__.py` | Package init |
|
||||
| `src/anthropic/provider.py` | Claude provider with health check |
|
||||
| `src/anthropic/model_selector.py` | Choose Claude or Ollama based on availability |
|
||||
|
||||
**Modified Files:**
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `src/core/config.py` | Add `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `PREFER_CLOUD_BACKEND` |
|
||||
| `src/agents/tatlock.py` | Use `get_model()` instead of hardcoded Ollama |
|
||||
| `src/agents/librarian/agent.py` | Use `get_model()` instead of hardcoded Ollama |
|
||||
| `src/agents/biographer/agent.py` | Use `get_model()` instead of hardcoded Ollama |
|
||||
| `src/agents/steward/agent.py` | Convert to PydanticAI or add Anthropic API support |
|
||||
| `src/core/startup.py` | Add Anthropic health check on startup |
|
||||
| `requirements.txt` | Add `anthropic>=0.40.0` |
|
||||
| `.env.example` | Document new environment variables |
|
||||
|
||||
### Phase 2 - MCP Server
|
||||
|
||||
**New Files:**
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `src/mcp/__init__.py` | Package init |
|
||||
| `src/mcp/server.py` | MCP server implementation |
|
||||
| `src/mcp/tool_adapters.py` | PydanticAI → MCP schema conversion |
|
||||
| `src/mcp/auth.py` | Token-based authentication |
|
||||
|
||||
---
|
||||
|
||||
## Cost Analysis
|
||||
|
||||
- **Claude API**: $5-30/month (10-50 calls/day, ~2k input + 1k output tokens/call)
|
||||
- **MCP via Claude Pro**: Included in subscription
|
||||
- **Total**: ~$10-80/month for full bidirectional integration
|
||||
|
||||
---
|
||||
|
||||
## Verification Plan
|
||||
|
||||
### Phase 1 Testing
|
||||
```bash
|
||||
# 1. Run with Claude backend
|
||||
ANTHROPIC_API_KEY=your-key docker-compose up -d tatlock
|
||||
|
||||
# 2. Verify Claude is being used
|
||||
docker logs tatlock 2>&1 | grep -i "anthropic\|claude"
|
||||
|
||||
# 3. Test butler personality
|
||||
curl -X POST http://tatlock.schweitz.internal:8000/v1/responses \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "Tatlock", "input": "Hello, who are you?"}'
|
||||
|
||||
# 4. Test offline fallback
|
||||
ANTHROPIC_API_KEY="" docker-compose up -d tatlock
|
||||
docker logs tatlock 2>&1 | grep -i "ollama\|fallback"
|
||||
```
|
||||
|
||||
### Phase 2 Testing
|
||||
```bash
|
||||
# 1. Start MCP server
|
||||
docker-compose up -d tatlock-mcp
|
||||
|
||||
# 2. Test MCP endpoint
|
||||
curl -X POST https://mcp.schweitz.net/tools/list \
|
||||
-H "Authorization: Bearer $MCP_AUTH_TOKEN"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Priority
|
||||
|
||||
1. **Phase 1: Backend Swap** (~1 week)
|
||||
- Immediate value: 200k context for ALL queries
|
||||
- Low risk: provider abstraction, graceful offline fallback
|
||||
|
||||
2. **Phase 2: MCP Server** (~2-3 weeks)
|
||||
- Enables cross-device access
|
||||
- Bidirectional: Tatlock superpowered by Claude AND accessible to Claude
|
||||
|
||||
---
|
||||
|
||||
## Future Phases (Optional)
|
||||
|
||||
- **Phase 3: LiteLLM Gateway** - Unified endpoint for all models, config-driven routing
|
||||
- **Phase 4: Multi-Provider** - Add OpenAI, Vertex AI, etc.
|
||||
- **Phase 5: Smart Routing** - Context-aware model selection, cost ceiling enforcement
|
||||
|
||||
---
|
||||
|
||||
## Offline Behavior
|
||||
|
||||
| Scenario | Behavior |
|
||||
|----------|----------|
|
||||
| No API key | Use Ollama exclusively |
|
||||
| API unreachable | Use Ollama, log warning |
|
||||
| API rate limited | Fallback to Ollama |
|
||||
|
||||
| Aspect | Claude | Ollama |
|
||||
|--------|--------|--------|
|
||||
| Context | 200k tokens | ~8k tokens |
|
||||
| Latency | 1-3s (network) | 0.5-1s (local) |
|
||||
| Personality | Preserved | Preserved |
|
||||
| Tools | All work | All work |
|
||||
| Cost | API charges | Free |
|
||||
|
||||
---
|
||||
|
||||
## Implementation Status
|
||||
|
||||
### Phase 1: Backend Swap - CODE COMPLETE (awaiting API access)
|
||||
|
||||
- [x] Add Anthropic config settings to `src/core/config.py`
|
||||
- [x] Add `pydantic-ai-slim[openai,anthropic]` to requirements.txt
|
||||
- [x] Create `src/anthropic/` module (model_selector.py)
|
||||
- [x] Add Claude health check to startup.py
|
||||
- [x] Refactor all PydanticAI agents to use `get_model()`
|
||||
- [x] Librarian
|
||||
- [x] Biographer
|
||||
- [x] Housekeeper
|
||||
- [x] Tatlock (6 locations)
|
||||
- [x] Add Claude API path to Steward agent (direct API calls)
|
||||
- [x] Update `.env.example` with new variables
|
||||
- [x] Test Ollama fallback (working)
|
||||
- [ ] Test with Claude API key (blocked: no API access currently)
|
||||
|
||||
**Note:** Implementation complete. Currently runs in Ollama-only mode. Will automatically use Claude when `ANTHROPIC_API_KEY` is configured.
|
||||
|
||||
### Phase 2: MCP Server - NOT STARTED
|
||||
|
||||
- [ ] Create `src/mcp/` module
|
||||
- [ ] Tool adapters (PydanticAI → MCP schema)
|
||||
- [ ] Authentication middleware
|
||||
- [ ] Streamable HTTP transport
|
||||
- [ ] Docker stack configuration
|
||||
|
||||
---
|
||||
|
||||
## Related Repository Handovers
|
||||
|
||||
Handover documents created in each repo: `PROJECT_CLAUDIFICATION_HANDOVER.md`
|
||||
|
||||
### library-desk - HANDOVER CREATED
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Review HybridRAG response size limits
|
||||
- [ ] Review smart_create endpoint for Claude optimization
|
||||
- [ ] Evaluate response formats for LLM consumption
|
||||
|
||||
### core-api - HANDOVER CREATED
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Review list_devices response format
|
||||
- [ ] Review error messages for LLM consumption
|
||||
- [ ] Evaluate rate limiting for faster Claude processing
|
||||
|
||||
### portainer-core - HANDOVER CREATED (blocking for production)
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Update stack with new environment variables
|
||||
- [ ] Configure secrets management for API key
|
||||
- [ ] Update CONTAINERS.md documentation
|
||||
|
||||
### webber - HANDOVER CREATED
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Review content truncation limits
|
||||
- [ ] Evaluate extraction quality for LLM consumption
|
||||
|
||||
### tatlock-ui - HANDOVER CREATED
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Test streaming responses with Claude backend
|
||||
- [ ] Test conversation history with larger context
|
||||
- [ ] Verify tool call display and reasoning rendering
|
||||
@@ -0,0 +1,246 @@
|
||||
# Housekeeper Agent Optimization Findings
|
||||
|
||||
## Background
|
||||
|
||||
Research with Gemini identified key issues with mistral-nemo and tool calling:
|
||||
- "Pre-computation Hallucination" - model answers before using tools
|
||||
- High default temperature (0.7-0.8) causes wandering
|
||||
- Model is "chatty and confident" - needs explicit constraints
|
||||
|
||||
## Key Recommendations from Gemini Research
|
||||
|
||||
1. **Temperature 0.0** for tool-calling agents (deterministic, follows schema)
|
||||
2. **Chain of Thought (CoT)** - force step-by-step reasoning
|
||||
3. **Negative constraints** - tell model what NOT to do (Nemo responds better)
|
||||
4. **Explicit tool descriptions** - verbose docstrings with "never estimate yourself"
|
||||
5. **"Strictly tool-based assistant"** pattern - NO internal knowledge claim
|
||||
|
||||
---
|
||||
|
||||
## Experiment Log
|
||||
|
||||
### Baseline (v1.8.6)
|
||||
- **Date**: 2025-12-17
|
||||
- **Configuration**: Default temperature, improved prompt requiring list_devices first
|
||||
- **Results**:
|
||||
- Called list_devices first ✓
|
||||
- Still hallucinated `light.study_desk` despite seeing list with only `light.study` and `light.study_main`
|
||||
- Partial success: turned off `light.study_main`, failed on hallucinated entity
|
||||
- **Success rate**: ~50% (1 of 2 study lights controlled correctly)
|
||||
|
||||
---
|
||||
|
||||
### Experiment 1: Temperature 0.0
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Set `model_settings=ModelSettings(temperature=0.0)` for Housekeeper
|
||||
- **Hypothesis**: Deterministic output will force model to use exact entity IDs from tool results
|
||||
- **Results**:
|
||||
|
||||
**Study lights test:**
|
||||
- Called `list_devices()` first ✓ (but no domain filter)
|
||||
- Used wrong parameter `device_id` instead of `entity_id` (recovered after validation error)
|
||||
- Only identified `light.studeerlamp` as "study" related (Dutch name)
|
||||
- **Missed `light.study` and `light.study_main`** - didn't match English "study"
|
||||
- Turned off 1 wrong light, missed 2 actual study lights
|
||||
|
||||
**Kitchen lights test:**
|
||||
- Called `list_devices()` first ✓ (no domain filter)
|
||||
- Saw full device list including `light.kitchen`
|
||||
- Used wrong parameter `device_id` instead of `entity_id` (recovered after validation)
|
||||
- After correction, dropped domain prefix: used `kitchen` instead of `light.kitchen`
|
||||
- 404 error - device not found
|
||||
|
||||
- **Success rate**: 0% (no target lights successfully controlled)
|
||||
- **Observations**:
|
||||
- Temperature 0.0 alone is insufficient
|
||||
- Model consistently confuses `device_id` vs `entity_id` parameter name
|
||||
- After validation error correction, model truncates entity_id (drops domain prefix)
|
||||
- Semantic matching of room names to devices is weak
|
||||
- Model doesn't understand entity_id format: `domain.name`
|
||||
|
||||
---
|
||||
|
||||
### Experiment 2: Negative Constraints + CoT
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Complete prompt rewrite with:
|
||||
- "You have NO Internal Knowledge" - negative framing
|
||||
- Explicit entity_id format with WRONG/RIGHT examples
|
||||
- Step-by-step process (ALWAYS FOLLOW)
|
||||
- Explicit parameter names section
|
||||
- "What NOT To Do" negative constraints
|
||||
- **Hypothesis**: Negative constraints work better with Mistral-Nemo
|
||||
- **Results**:
|
||||
|
||||
**Study lights test:**
|
||||
- Called `list_devices(domain="light")` ✓ with domain filter (improvement!)
|
||||
- Still used `device_id` first, recovered to `entity_id` after validation error
|
||||
- After recovery, used correct full format: `light.studeerlamp`
|
||||
- **Still only matched `studeerlamp` not `light.study` or `light.study_main`**
|
||||
|
||||
**Kitchen lights test:**
|
||||
- Called `list_devices(domain="light")` ✓
|
||||
- Called `turn_off(entity_id="light.kitchen")` ✓ correct format!
|
||||
- All 4 kitchen lights turned off (light.kitchen is a group)
|
||||
- **100% success for kitchen!**
|
||||
|
||||
- **Success rate**:
|
||||
- Study: 0% (wrong semantic match)
|
||||
- Kitchen: 100% (4/4 lights off)
|
||||
- Combined: ~50% (1 of 2 tests successful)
|
||||
- **Observations**:
|
||||
- Domain filter now consistently used ✓
|
||||
- Entity_id format correct after recovery ✓
|
||||
- Semantic matching still fails for "study" → prefers Dutch "studeerlamp" over English "study"
|
||||
- Parameter name confusion persists (`device_id` vs `entity_id`)
|
||||
- Simple room names (kitchen) work; mixed language fails (study/studeerlamp)
|
||||
|
||||
---
|
||||
|
||||
### Experiment 3: Temperature 0.1 + Explicit Tool Docstrings
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**:
|
||||
- Temperature 0.1
|
||||
- Updated turn_on/turn_off docstrings with explicit `entity_id=` in examples
|
||||
- **Results**:
|
||||
- Still uses `device_id` first, recovers to `entity_id` after validation
|
||||
- Still picks wrong entity (studeerlamp over study)
|
||||
- **Success rate**: 0%
|
||||
|
||||
---
|
||||
|
||||
### Experiment 4: Room Group Priority (with explicit examples)
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Updated prompt with:
|
||||
- Explicit instruction: "Look for EXACT match `light.<room_name>` first!"
|
||||
- Concrete examples: "For 'study lights' → look for `light.study`"
|
||||
- Working example showing `turn_off(entity_id="light.study")`
|
||||
- **Hypothesis**: Explicit examples will guide model to use room groups
|
||||
- **Results**:
|
||||
|
||||
**Test 1 & 2 (consecutive):**
|
||||
- Called `list_devices(domain="light")` ✓
|
||||
- Device list clearly shows `light.study` at the bottom
|
||||
- First call: `turn_off({"devices":["studeerlamp"]})` - wrong param AND wrong device
|
||||
- After validation error: `turn_off(entity_id="light.studeerlamp")` - correct param, still wrong device
|
||||
- **Completely ignored `light.study` despite prompt explicitly saying to use it**
|
||||
|
||||
- **Success rate**: 0% (wrong device controlled)
|
||||
- **Observations**:
|
||||
- Model ignores explicit step-by-step instructions in favor of substring matching
|
||||
- Dutch "studeerlamp" contains "studer" which the model prefers over exact "study" match
|
||||
- Even when prompt has a literal example `turn_off(entity_id="light.study")`, model uses `light.studeerlamp`
|
||||
- Positional bias possible - `light.study` appears at end of 21-item list
|
||||
- **Fundamental limitation**: Mistral-Nemo cannot follow explicit matching rules
|
||||
|
||||
---
|
||||
|
||||
### Experiment 5: Room Groups First (Tool Output Ordering)
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Modified `list_devices` to sort room groups to top of list using HA attributes (`is_hue_group`, `hue_type="room"`)
|
||||
- **Hypothesis**: Positional bias - model focuses on items earlier in list
|
||||
- **Results**:
|
||||
- Room groups (`light.study`, `light.kitchen`, etc.) now appear first in device list
|
||||
- Combined with improved prompt, model now consistently uses room groups
|
||||
- **70% success rate** (7/10 tests) with default q4 quantization
|
||||
|
||||
---
|
||||
|
||||
### Experiment 6: Model Quantization (q5_1)
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Upgraded from default Mistral-Nemo quantization (q4) to `mistral-nemo:12b-instruct-2407-q5_1`
|
||||
- **Hypothesis**: Higher precision weights improve tool calling accuracy
|
||||
- **Results**:
|
||||
|
||||
| Test | Action | Result |
|
||||
|------|--------|--------|
|
||||
| 1 | Turn off study | PASS |
|
||||
| 2 | Turn on study | PASS |
|
||||
| 3 | Toggle study | PASS |
|
||||
| 4 | Turn off kitchen | PASS |
|
||||
| 5 | Turn on kitchen | PASS |
|
||||
| 6 | Toggle kitchen | PASS |
|
||||
| 7 | Turn off bedroom | PASS |
|
||||
| 8 | Turn on bedroom | PASS |
|
||||
| 9 | Turn off living room | PASS |
|
||||
| 10 | Turn on living room | PASS |
|
||||
|
||||
- **Success rate**: **100%** (10/10 tests)
|
||||
- **Observations**:
|
||||
- q5_1 quantization dramatically improves tool calling accuracy
|
||||
- All room groups correctly identified and used
|
||||
- No parameter confusion (`entity_id` used correctly)
|
||||
- No entity_id truncation issues
|
||||
- Toggle operations now work reliably
|
||||
- Model fits within 10GB VRAM (q6 did not)
|
||||
|
||||
---
|
||||
|
||||
### Experiment 7: Device List in System Prompt (Context Injection)
|
||||
- **Date**: [PENDING]
|
||||
- **Change**: Store device list in database (per user/household) and inject into system prompt
|
||||
- **Approach**:
|
||||
1. Periodically sync device list from Home Assistant to PostgreSQL
|
||||
2. On each Housekeeper invocation, fetch device list and include in prompt
|
||||
3. Remove need for model to call list_devices() - just match from context
|
||||
- **Hypothesis**:
|
||||
- Eliminates tool call step where errors occur
|
||||
- Reduces context size by not returning full device list as tool output
|
||||
- Makes entity matching a language task (in prompt) rather than tool result parsing
|
||||
- **Trade-offs**:
|
||||
- Stale data if sync is infrequent
|
||||
- Prompt size increase (but less than tool call response)
|
||||
- Need sync mechanism and storage
|
||||
- **Results**: [TO BE RECORDED]
|
||||
- **Success rate**: [TO BE RECORDED]
|
||||
|
||||
---
|
||||
|
||||
## Key Problem Identified (Solved)
|
||||
|
||||
The model struggled with:
|
||||
1. **Parameter schema adherence** - uses `device_id` when schema requires `entity_id`
|
||||
2. **Value preservation** - truncates values after validation errors (drops `light.` prefix)
|
||||
3. **Semantic matching** - prefers substring matches ("studeerlamp" contains "studer") over exact matches (`light.study`)
|
||||
4. **Following explicit instructions** - ignores step-by-step processes even when examples are provided
|
||||
5. **Positional bias** - may not "see" items at the end of long lists
|
||||
|
||||
**Solution**: These issues were resolved by:
|
||||
1. Using q5_1 quantization instead of default q4 (higher precision weights)
|
||||
2. Sorting room groups to top of device list (address positional bias)
|
||||
3. Explicit prompt guidance with negative constraints and examples
|
||||
|
||||
---
|
||||
|
||||
## Potential Next Experiments
|
||||
|
||||
### Experiment 5: Room Groups First (List Ordering)
|
||||
- **Hypothesis**: Positional bias - model focuses on items earlier in list
|
||||
- **Change**: Sort device list to put room groups (entities matching `light.<single_word>`) at the TOP
|
||||
- **Effort**: Low - modify list_devices output formatting
|
||||
- **Risk**: May affect other use cases where individual devices are needed
|
||||
|
||||
### Experiment 6: Simplified Device List Format
|
||||
- **Hypothesis**: Markdown formatting adds noise that confuses the model
|
||||
- **Change**: Return simple list: `light.study (Study - GROUP), light.study_main (Ceiling light), ...`
|
||||
- **Effort**: Low - modify list_devices output
|
||||
- **Risk**: Less human-readable responses
|
||||
|
||||
---
|
||||
|
||||
## Learnings to Apply Elsewhere
|
||||
|
||||
1. **Quantization matters** - q5_1 dramatically outperforms q4 for tool calling (100% vs 70%)
|
||||
2. **Positional bias is real** - sort important items to top of lists
|
||||
3. **Smaller models need simpler workflows** - fewer tool calls, more context injection
|
||||
4. **Validation errors don't teach** - model often makes worse mistakes on retry
|
||||
5. **Entity IDs are hard** - domain.name format confuses the model
|
||||
6. **Consider pre-computation** - move matching logic to code, not LLM
|
||||
7. **Use explicit negative constraints** - "NEVER do X" works better than "always do Y"
|
||||
|
||||
---
|
||||
|
||||
## Notes
|
||||
|
||||
- Librarian may need higher temperature for creative synthesis
|
||||
- All "action" agents (Housekeeper, future agents) should use low temperature
|
||||
- Consider testing with Gemma 2 9B for better function calling (Google, open weights)
|
||||
@@ -1,317 +0,0 @@
|
||||
# Home Automation API Interface Specification
|
||||
|
||||
## Purpose
|
||||
|
||||
This document specifies the expected endpoints for a home automation abstraction layer in core-api. These endpoints will be consumed by the Tatlock Housekeeper agent and potentially other projects (scheduler, dashboards).
|
||||
|
||||
The goal is to provide a simplified, domain-specific interface for home automation that abstracts away the underlying platform (initially Home Assistant, but swappable).
|
||||
|
||||
---
|
||||
|
||||
## Endpoints
|
||||
|
||||
### Device Discovery
|
||||
|
||||
#### `GET /housekeeping/devices`
|
||||
|
||||
List available devices.
|
||||
|
||||
**Query Parameters:**
|
||||
- `domain` (optional): Filter by device type (e.g., `light`, `switch`, `climate`, `media_player`)
|
||||
- `area` (optional): Filter by area/room name
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"devices": [
|
||||
{
|
||||
"entity_id": "light.living_room",
|
||||
"name": "Living Room Light",
|
||||
"domain": "light",
|
||||
"area": "Living Room",
|
||||
"state": "on",
|
||||
"attributes": {
|
||||
"brightness": 255,
|
||||
"color_temp": 370
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `GET /housekeeping/devices/{entity_id}`
|
||||
|
||||
Get detailed state of a specific device.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"entity_id": "light.living_room",
|
||||
"name": "Living Room Light",
|
||||
"domain": "light",
|
||||
"area": "Living Room",
|
||||
"state": "on",
|
||||
"attributes": {
|
||||
"brightness": 255,
|
||||
"color_temp": 370,
|
||||
"supported_features": ["brightness", "color_temp"]
|
||||
},
|
||||
"last_changed": "2025-12-16T10:30:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `GET /housekeeping/areas`
|
||||
|
||||
List all areas/rooms.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"areas": [
|
||||
{"id": "living_room", "name": "Living Room"},
|
||||
{"id": "bedroom", "name": "Bedroom"},
|
||||
{"id": "kitchen", "name": "Kitchen"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Device Control
|
||||
|
||||
#### `POST /housekeeping/devices/{entity_id}/control`
|
||||
|
||||
Control a device (turn on, turn off, toggle, or set attributes).
|
||||
|
||||
**Request Body:**
|
||||
```json
|
||||
{
|
||||
"action": "turn_on",
|
||||
"brightness": 128,
|
||||
"color_temp": 400
|
||||
}
|
||||
```
|
||||
|
||||
- `action` (required): One of `turn_on`, `turn_off`, `toggle`
|
||||
- Additional attributes vary by device type (brightness, color_temp, rgb_color, etc.)
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"entity_id": "light.living_room",
|
||||
"new_state": "on",
|
||||
"message": "Light turned on"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Scenes
|
||||
|
||||
#### `GET /housekeeping/scenes`
|
||||
|
||||
List available scenes.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"scenes": [
|
||||
{"id": "scene.movie_night", "name": "Movie Night"},
|
||||
{"id": "scene.good_morning", "name": "Good Morning"},
|
||||
{"id": "scene.all_off", "name": "All Off"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `POST /housekeeping/scenes/{scene_id}/activate`
|
||||
|
||||
Activate a scene.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"scene_id": "scene.movie_night",
|
||||
"message": "Scene activated"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Scripts
|
||||
|
||||
#### `GET /housekeeping/scripts`
|
||||
|
||||
List available scripts/sequences.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"scripts": [
|
||||
{"id": "script.bedtime_routine", "name": "Bedtime Routine"},
|
||||
{"id": "script.welcome_home", "name": "Welcome Home"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `POST /housekeeping/scripts/{script_id}/run`
|
||||
|
||||
Execute a script with optional variables.
|
||||
|
||||
**Request Body (optional):**
|
||||
```json
|
||||
{
|
||||
"variables": {
|
||||
"brightness_level": 50,
|
||||
"target_room": "bedroom"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"script_id": "script.bedtime_routine",
|
||||
"message": "Script executed"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Automations
|
||||
|
||||
#### `GET /housekeeping/automations`
|
||||
|
||||
List automations and their enabled/disabled status.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"automations": [
|
||||
{
|
||||
"id": "automation.motion_lights",
|
||||
"name": "Motion Lights",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"id": "automation.night_mode",
|
||||
"name": "Night Mode",
|
||||
"enabled": false
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `POST /housekeeping/automations/{automation_id}/toggle`
|
||||
|
||||
Enable or disable an automation.
|
||||
|
||||
**Request Body:**
|
||||
```json
|
||||
{
|
||||
"enabled": true
|
||||
}
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"automation_id": "automation.motion_lights",
|
||||
"enabled": true,
|
||||
"message": "Automation enabled"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Utility
|
||||
|
||||
#### `GET /housekeeping/history`
|
||||
|
||||
Get state history for a device.
|
||||
|
||||
**Query Parameters:**
|
||||
- `entity_id` (required): Device to get history for
|
||||
- `hours` (optional, default 24): Hours of history to retrieve
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"entity_id": "light.living_room",
|
||||
"history": [
|
||||
{
|
||||
"state": "on",
|
||||
"timestamp": "2025-12-16T10:30:00Z",
|
||||
"attributes": {"brightness": 255}
|
||||
},
|
||||
{
|
||||
"state": "off",
|
||||
"timestamp": "2025-12-16T08:00:00Z",
|
||||
"attributes": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `GET /housekeeping/health`
|
||||
|
||||
Health check for home automation connection.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"status": "healthy",
|
||||
"connected": true,
|
||||
"platform": "home_assistant",
|
||||
"version": "2024.12.0"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Error Responses
|
||||
|
||||
All endpoints should return consistent error responses:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"code": "DEVICE_NOT_FOUND",
|
||||
"message": "Device light.nonexistent not found"
|
||||
}
|
||||
```
|
||||
|
||||
Common error codes:
|
||||
- `DEVICE_NOT_FOUND` - Entity ID doesn't exist
|
||||
- `INVALID_ACTION` - Unsupported action for device type
|
||||
- `CONNECTION_ERROR` - Cannot reach home automation platform
|
||||
- `UNAUTHORIZED` - Invalid or missing credentials
|
||||
|
||||
---
|
||||
|
||||
## Authentication
|
||||
|
||||
All endpoints require authentication via Bearer token in the `Authorization` header.
|
||||
|
||||
---
|
||||
|
||||
## Consuming Client
|
||||
|
||||
The Tatlock project has an existing client (`src/agents/housekeeper/client.py`) that expects these endpoints. No changes to Tatlock are needed once these endpoints are available.
|
||||
|
||||
Reference: `CoreAPIClient` class in Tatlock expects these exact endpoint patterns.
|
||||
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "tatlock"
|
||||
version = "1.8.6"
|
||||
version = "2.0.1"
|
||||
description = "OpenAI-compatible API with Ollama backend"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = []
|
||||
|
||||
+2
-3
@@ -21,10 +21,9 @@ pydantic-settings>=2.12,<2.13
|
||||
|
||||
# AI/LLM integration
|
||||
# PydanticAI: Agent framework for using Pydantic with LLMs
|
||||
# Using slim version with only openai extra (Ollama uses OpenAI-compatible API)
|
||||
# This avoids installing SDKs for anthropic, cohere, google, groq, huggingface, etc.
|
||||
# Using slim version with openai (Ollama) and anthropic (Claude) extras
|
||||
# See DEPENDENCY_SLIM.md for rollback instructions if this breaks
|
||||
pydantic-ai-slim[openai]>=1.27,<1.28
|
||||
pydantic-ai-slim[openai,anthropic]>=1.27,<1.28
|
||||
|
||||
# HTTP client for Ollama communication
|
||||
# Latest: 0.28.1 - No known CVEs
|
||||
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/bin/bash
|
||||
# Housekeeper Room Group Detection Test Suite
|
||||
# Verifies room groups are controlled by checking actual state changes
|
||||
|
||||
API_URL="http://localhost:8777/v1/chat/completions"
|
||||
CORE_API="http://192.168.86.149:8083"
|
||||
RESULTS_FILE="/tmp/housekeeper_test_results.txt"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
get_state() {
|
||||
curl -s "$CORE_API/housekeeping/devices/$1" 2>/dev/null | jq -r '.state' 2>/dev/null
|
||||
}
|
||||
|
||||
echo "=========================================="
|
||||
echo "Housekeeper Room Group Test Suite"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
> "$RESULTS_FILE"
|
||||
|
||||
run_toggle_test() {
|
||||
local test_num=$1
|
||||
local room=$2
|
||||
local entity="light.$room"
|
||||
local prompt_room="${room//_/ }"
|
||||
|
||||
printf "Test %2d: Toggle %-12s lights ... " "$test_num" "$prompt_room"
|
||||
|
||||
local before=$(get_state "$entity")
|
||||
if [ -z "$before" ] || [ "$before" = "null" ]; then
|
||||
echo -e "${YELLOW}SKIP${NC} (cannot get state)"
|
||||
echo "SKIP|$test_num|Toggle $room|error" >> "$RESULTS_FILE"
|
||||
return
|
||||
fi
|
||||
|
||||
curl -s -X POST "$API_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\": \"tatlock\", \"messages\": [{\"role\": \"user\", \"content\": \"Toggle the $prompt_room lights\"}]}" > /dev/null
|
||||
|
||||
sleep 4
|
||||
|
||||
local after=$(get_state "$entity")
|
||||
|
||||
if [ "$before" != "$after" ]; then
|
||||
echo -e "${GREEN}PASS${NC} ($before -> $after)"
|
||||
echo "PASS|$test_num|Toggle $room|$before->$after" >> "$RESULTS_FILE"
|
||||
else
|
||||
echo -e "${RED}FAIL${NC} (state unchanged: $before)"
|
||||
echo "FAIL|$test_num|Toggle $room|unchanged:$before" >> "$RESULTS_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
run_onoff_test() {
|
||||
local test_num=$1
|
||||
local room=$2
|
||||
local action=$3
|
||||
local expected_state=$4
|
||||
# Entity uses underscore, prompt uses space
|
||||
local entity="light.${room//_/ }"
|
||||
entity="light.$room"
|
||||
local prompt_room="${room//_/ }"
|
||||
|
||||
printf "Test %2d: %-8s %-12s lights ... " "$test_num" "$action" "$prompt_room"
|
||||
|
||||
curl -s -X POST "$API_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\": \"tatlock\", \"messages\": [{\"role\": \"user\", \"content\": \"$action the $prompt_room lights\"}]}" > /dev/null
|
||||
|
||||
sleep 4
|
||||
|
||||
local after=$(get_state "$entity")
|
||||
|
||||
if [ "$after" = "$expected_state" ]; then
|
||||
echo -e "${GREEN}PASS${NC} ($after)"
|
||||
echo "PASS|$test_num|$action $room|$after" >> "$RESULTS_FILE"
|
||||
else
|
||||
echo -e "${RED}FAIL${NC} (got $after, expected $expected_state)"
|
||||
echo "FAIL|$test_num|$action $room|got:$after,expected:$expected_state" >> "$RESULTS_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Running tests (~4s each)..."
|
||||
echo ""
|
||||
|
||||
# Study tests
|
||||
run_onoff_test 1 "study" "Turn off" "off"
|
||||
run_onoff_test 2 "study" "Turn on" "on"
|
||||
run_toggle_test 3 "study"
|
||||
|
||||
# Kitchen tests
|
||||
run_onoff_test 4 "kitchen" "Turn off" "off"
|
||||
run_onoff_test 5 "kitchen" "Turn on" "on"
|
||||
run_toggle_test 6 "kitchen"
|
||||
|
||||
# Bedroom tests
|
||||
run_onoff_test 7 "bedroom" "Turn off" "off"
|
||||
run_onoff_test 8 "bedroom" "Turn on" "on"
|
||||
|
||||
# Living room tests (entity is light.living_room)
|
||||
run_onoff_test 9 "living_room" "Turn off" "off"
|
||||
run_onoff_test 10 "living_room" "Turn on" "on"
|
||||
|
||||
# Ensure all lights end up ON
|
||||
echo ""
|
||||
echo "Restoring all lights to ON..."
|
||||
for room in "study" "kitchen" "bedroom" "living room"; do
|
||||
curl -s -X POST "$API_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\": \"tatlock\", \"messages\": [{\"role\": \"user\", \"content\": \"Turn on the $room lights\"}]}" > /dev/null
|
||||
sleep 3
|
||||
done
|
||||
echo "Done."
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Results"
|
||||
echo "=========================================="
|
||||
|
||||
PASS=$(grep -c "^PASS" "$RESULTS_FILE" 2>/dev/null || echo 0)
|
||||
FAIL=$(grep -c "^FAIL" "$RESULTS_FILE" 2>/dev/null || echo 0)
|
||||
SKIP=$(grep -c "^SKIP" "$RESULTS_FILE" 2>/dev/null || echo 0)
|
||||
TOTAL=$((PASS + FAIL))
|
||||
|
||||
echo "Passed: $PASS"
|
||||
echo "Failed: $FAIL"
|
||||
echo "Skipped: $SKIP"
|
||||
|
||||
if [ "$TOTAL" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "Success Rate: $((PASS * 100 / TOTAL))% ($PASS/$TOTAL)"
|
||||
fi
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "Failures:"
|
||||
grep "^FAIL" "$RESULTS_FILE"
|
||||
fi
|
||||
@@ -102,16 +102,10 @@ _biographer_agent: Optional[Agent[None, str]] = None
|
||||
|
||||
def _create_biographer_agent() -> Agent[None, str]:
|
||||
"""Create The Biographer PydanticAI agent."""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
# Create Ollama model with sanitized provider
|
||||
# (fixes 'content: null' issue with tool calls)
|
||||
model = OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
agent: Agent[None, str] = Agent(
|
||||
model=model,
|
||||
@@ -131,9 +125,12 @@ def _create_biographer_agent() -> Agent[None, str]:
|
||||
# Register management tools
|
||||
agent.tool_plain(forget_memory)
|
||||
|
||||
from src.anthropic.model_selector import get_model_info
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"biographer_agent_created",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
tool_count=6,
|
||||
)
|
||||
|
||||
|
||||
+145
-84
@@ -13,6 +13,7 @@ from enum import Enum
|
||||
from typing import AsyncGenerator, Callable, Optional, Any
|
||||
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import trace_span, SpanType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -239,38 +240,58 @@ async def delegate_to_librarian(
|
||||
has_context=bool(context),
|
||||
)
|
||||
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_librarian(task=task, context=context)
|
||||
async with trace_span(
|
||||
"delegate_to_librarian",
|
||||
SpanType.EXPERT,
|
||||
metadata={
|
||||
"expert": "librarian",
|
||||
"task_preview": task[:100],
|
||||
"has_context": bool(context),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_librarian(task=task, context=context)
|
||||
|
||||
logger.info(
|
||||
"delegation_to_librarian_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
logger.info(
|
||||
"delegation_to_librarian_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
if span:
|
||||
span.metadata["success"] = True
|
||||
span.metadata["output_length"] = len(output)
|
||||
span.details["task"] = task
|
||||
span.details["context"] = context[:500] if context else None
|
||||
span.details["result_preview"] = output[:1000]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_librarian_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_librarian_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if span:
|
||||
span.metadata["success"] = False
|
||||
span.details["error"] = str(e)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
async def delegate_to_biographer(
|
||||
@@ -317,38 +338,58 @@ async def delegate_to_biographer(
|
||||
has_context=bool(context),
|
||||
)
|
||||
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_biographer(task=task, context=context)
|
||||
async with trace_span(
|
||||
"delegate_to_biographer",
|
||||
SpanType.EXPERT,
|
||||
metadata={
|
||||
"expert": "biographer",
|
||||
"task_preview": task[:100],
|
||||
"has_context": bool(context),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_biographer(task=task, context=context)
|
||||
|
||||
logger.info(
|
||||
"delegation_to_biographer_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
logger.info(
|
||||
"delegation_to_biographer_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
if span:
|
||||
span.metadata["success"] = True
|
||||
span.metadata["output_length"] = len(output)
|
||||
span.details["task"] = task
|
||||
span.details["context"] = context[:500] if context else None
|
||||
span.details["result_preview"] = output[:1000]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_biographer_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_biographer_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if span:
|
||||
span.metadata["success"] = False
|
||||
span.details["error"] = str(e)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
async def delegate_to_housekeeper(
|
||||
@@ -394,38 +435,58 @@ async def delegate_to_housekeeper(
|
||||
has_context=bool(context),
|
||||
)
|
||||
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_housekeeper(task=task, context=context)
|
||||
async with trace_span(
|
||||
"delegate_to_housekeeper",
|
||||
SpanType.EXPERT,
|
||||
metadata={
|
||||
"expert": "housekeeper",
|
||||
"task_preview": task[:100],
|
||||
"has_context": bool(context),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_housekeeper(task=task, context=context)
|
||||
|
||||
logger.info(
|
||||
"delegation_to_housekeeper_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
logger.info(
|
||||
"delegation_to_housekeeper_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
if span:
|
||||
span.metadata["success"] = True
|
||||
span.metadata["output_length"] = len(output)
|
||||
span.details["task"] = task
|
||||
span.details["context"] = context[:500] if context else None
|
||||
span.details["result_preview"] = output[:1000]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_housekeeper_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_housekeeper_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if span:
|
||||
span.metadata["success"] = False
|
||||
span.details["error"] = str(e)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@@ -32,93 +32,69 @@ from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Housekeeper system prompt
|
||||
HOUSEKEEPER_SYSTEM_PROMPT = """You are The Housekeeper, an expert home automation assistant in the Tatlock household.
|
||||
# Housekeeper system prompt - Optimized for Mistral-Nemo function calling
|
||||
HOUSEKEEPER_SYSTEM_PROMPT = """You are a strictly tool-based home automation assistant.
|
||||
|
||||
Your role is to help users control and monitor their smart home through Home Assistant:
|
||||
- Lights, switches, and other devices
|
||||
- Scenes (pre-configured device states)
|
||||
- Scripts (automation sequences)
|
||||
- Automations (event-triggered rules)
|
||||
## CRITICAL: You Have NO Internal Knowledge
|
||||
|
||||
## Your Personality
|
||||
- Efficient and practical
|
||||
- Safety-conscious (confirm destructive actions)
|
||||
- Proactive in suggesting optimizations
|
||||
- Clear about what actions you're taking
|
||||
You do NOT know what devices exist. You do NOT know any entity IDs.
|
||||
Entity IDs are different in every installation. You MUST discover them using tools.
|
||||
|
||||
## Your Tools
|
||||
## Entity ID Format
|
||||
|
||||
### Discovery Tools
|
||||
- **list_areas**: See all rooms/areas configured in Home Assistant
|
||||
- **list_devices**: Find devices by type (domain) or location (area)
|
||||
- **get_device_state**: Check a device's current state and attributes
|
||||
Entity IDs follow the format: `domain.name`
|
||||
Examples: `light.kitchen`, `light.study_main`, `switch.coffee_maker`
|
||||
|
||||
### Control Tools
|
||||
- **turn_on**: Turn on lights, switches, etc. (supports brightness/color for lights)
|
||||
- **turn_off**: Turn off devices
|
||||
- **toggle**: Flip a device's state
|
||||
The `entity_id` parameter MUST be the COMPLETE value including the domain prefix.
|
||||
WRONG: `entity_id="kitchen"`
|
||||
RIGHT: `entity_id="light.kitchen"`
|
||||
|
||||
### Scene Tools
|
||||
- **list_scenes**: See available scene presets
|
||||
- **activate_scene**: Activate a scene (e.g., "movie night", "good morning")
|
||||
## Step-by-Step Process (ALWAYS FOLLOW)
|
||||
|
||||
### Script Tools
|
||||
- **list_scripts**: See available automation scripts
|
||||
- **run_script**: Execute a script
|
||||
When asked to control devices in a room:
|
||||
|
||||
### Automation Tools
|
||||
- **list_automations**: See all automations and their status
|
||||
- **toggle_automation**: Enable or disable an automation
|
||||
1. THINK: What domain? (light, switch, climate, etc.)
|
||||
2. CALL: list_devices(domain="light") to discover available devices
|
||||
3. CHECK: Look for EXACT match `light.<room_name>` first!
|
||||
- For "study lights" → look for `light.study` (not light.study_main, not light.studeerlamp)
|
||||
- For "kitchen lights" → look for `light.kitchen` (not light.kitchen_spot_1)
|
||||
- These room groups control ALL lights in that room at once
|
||||
- If found, use ONLY the group (stop looking for individual lights)
|
||||
4. FALLBACK: Only if no exact room group exists, find entity_ids containing the room name
|
||||
5. CALL: turn_on/turn_off using the EXACT entity_id from step 3 or 4
|
||||
|
||||
### History Tools
|
||||
- **get_history**: Check a device's state history
|
||||
Example for "Turn off study lights":
|
||||
1. Domain is "light"
|
||||
2. Call list_devices(domain="light")
|
||||
3. Look for room group: `light.study` - FOUND!
|
||||
4. Call turn_off(entity_id="light.study") # This controls all study lights
|
||||
|
||||
## Critical Rules
|
||||
Example for "Turn off hallway lights" (no room group):
|
||||
1. Domain is "light"
|
||||
2. Call list_devices(domain="light")
|
||||
3. Look for room group: `light.hallway` - NOT FOUND
|
||||
4. Find all with "hallway": light.hallway_spot_1, light.hallway_spot_2
|
||||
5. Call turn_off for each
|
||||
|
||||
**NEVER GUESS ENTITY IDs.** You do not know what devices exist. Entity IDs vary between
|
||||
installations. You MUST call list_devices() FIRST to discover actual entity_ids before
|
||||
ANY control action (turn_on, turn_off, toggle).
|
||||
## Tool Parameter Names
|
||||
|
||||
Wrong approach:
|
||||
User: "Turn off the study lights"
|
||||
You: turn_off("light.study") ← WRONG! You guessed the entity_id
|
||||
- turn_on, turn_off, toggle: Use `entity_id` (NOT device_id, NOT id)
|
||||
- activate_scene: Use `scene_id`
|
||||
- run_script: Use `script_id`
|
||||
|
||||
Correct approach:
|
||||
User: "Turn off the study lights"
|
||||
You: list_devices(domain="light") ← First discover what exists
|
||||
You: [See results like light.study_main, light.study]
|
||||
You: turn_off("light.study_main"), turn_off("light.study") ← Use actual IDs
|
||||
## What NOT To Do
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **ALWAYS list_devices first** before any control action. No exceptions.
|
||||
Filter by domain and/or area to narrow results.
|
||||
|
||||
2. **Use exact entity_ids** from list_devices results. Never construct or guess them.
|
||||
|
||||
3. **Area-aware filtering**: Use area parameter when users mention a room.
|
||||
Note: Some devices may have area=None but contain the room name in entity_id.
|
||||
|
||||
4. **Verify after actions**: Use get_device_state to confirm state changes if needed.
|
||||
|
||||
5. **Safety for bulk actions**: When affecting multiple devices, summarize first.
|
||||
|
||||
## Common Patterns
|
||||
|
||||
- "Turn on the lights" → list_devices(domain="light"), then turn_on each
|
||||
- "What's on?" → list_devices() and filter for state="on"
|
||||
- "Movie time" → Either activate_scene("scene.movie_night") or run_script if available
|
||||
- "Dim the bedroom" → turn_on("light.bedroom", brightness=64)
|
||||
- NEVER guess an entity_id
|
||||
- NEVER construct an entity_id from the room name
|
||||
- NEVER drop the domain prefix (light., switch., etc.)
|
||||
- NEVER use "device_id" - the parameter is called "entity_id"
|
||||
- NEVER provide an answer without calling list_devices first
|
||||
|
||||
## Response Format
|
||||
Your responses are returned to Tatlock (the butler) who will synthesize them into
|
||||
a final answer for the user. Keep this in mind:
|
||||
- Lead with confirmation of what you did or found
|
||||
- Be specific about which devices were affected
|
||||
- Include relevant state information
|
||||
- Note any issues or failures
|
||||
- Be concise - Tatlock will format the final response
|
||||
|
||||
After completing actions, briefly confirm:
|
||||
- Which devices were affected (list the entity_ids)
|
||||
- Whether each action succeeded or failed
|
||||
"""
|
||||
|
||||
# Lazy initialization to avoid connection issues during imports
|
||||
@@ -127,16 +103,10 @@ _housekeeper_agent: Optional[Agent[None, str]] = None
|
||||
|
||||
def _create_housekeeper_agent() -> Agent[None, str]:
|
||||
"""Create the Housekeeper PydanticAI agent."""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
# Create Ollama model with sanitized provider
|
||||
# (fixes 'content: null' issue with tool calls)
|
||||
model = OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
agent: Agent[None, str] = Agent(
|
||||
model=model,
|
||||
@@ -169,9 +139,12 @@ def _create_housekeeper_agent() -> Agent[None, str]:
|
||||
# Register history tools
|
||||
agent.tool_plain(get_history)
|
||||
|
||||
from src.anthropic.model_selector import get_model_info
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"housekeeper_agent_created",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
tool_count=13,
|
||||
)
|
||||
|
||||
@@ -231,9 +204,13 @@ async def run_housekeeper(
|
||||
)
|
||||
|
||||
try:
|
||||
# Use temperature 0.1 for slight exploration
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
result = await agent.run(
|
||||
prompt,
|
||||
message_history=message_history,
|
||||
model_settings=ModelSettings(temperature=0.1),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -289,9 +266,13 @@ async def run_housekeeper_stream(
|
||||
)
|
||||
|
||||
try:
|
||||
# Use temperature 0.1 for slight exploration
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
async with agent.run_stream(
|
||||
prompt,
|
||||
message_history=message_history,
|
||||
model_settings=ModelSettings(temperature=0.1),
|
||||
) as response:
|
||||
async for delta in response.stream_text(delta=True):
|
||||
yield delta
|
||||
|
||||
@@ -59,10 +59,27 @@ async def list_devices(
|
||||
|
||||
for dom, dom_devices in sorted(by_domain.items()):
|
||||
output_parts.append(f"### {dom.title()}s")
|
||||
for device in dom_devices:
|
||||
|
||||
# Sort devices: room groups first (using Home Assistant's is_hue_group attribute)
|
||||
def is_room_group(d: object) -> bool:
|
||||
"""Check if device is a room group based on HA attributes."""
|
||||
attrs = getattr(d, "attributes", {})
|
||||
# Check for Hue room groups
|
||||
if attrs.get("is_hue_group") and attrs.get("hue_type") == "room":
|
||||
return True
|
||||
# Check for other group indicators (icon or entity_id list)
|
||||
if "entity_id" in attrs and isinstance(attrs["entity_id"], list):
|
||||
return True
|
||||
return False
|
||||
|
||||
sorted_devices = sorted(dom_devices, key=lambda d: (not is_room_group(d), d.entity_id))
|
||||
|
||||
for device in sorted_devices:
|
||||
state_icon = "on" if device.state == "on" else "off" if device.state == "off" else device.state
|
||||
area_str = f" ({device.area})" if device.area else ""
|
||||
output_parts.append(f"- **{device.name}**{area_str}: {state_icon}")
|
||||
# Mark room groups clearly using actual HA data
|
||||
group_marker = " [ROOM GROUP]" if is_room_group(device) else ""
|
||||
output_parts.append(f"- **{device.name}**{area_str}{group_marker}: {state_icon}")
|
||||
output_parts.append(f" ID: `{device.entity_id}`")
|
||||
output_parts.append("")
|
||||
|
||||
@@ -164,12 +181,12 @@ async def turn_on(
|
||||
color_temp: int | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Turn on a device.
|
||||
Turn on a device. Use the entity_id parameter with the EXACT value from list_devices.
|
||||
|
||||
For lights, can optionally set brightness and color temperature.
|
||||
|
||||
Args:
|
||||
entity_id: Device to turn on (e.g., light.living_room, switch.coffee_maker)
|
||||
entity_id: The EXACT entity ID from list_devices including domain prefix.
|
||||
brightness: Optional brightness for lights (0-255, where 255 is full brightness)
|
||||
color_temp: Optional color temperature in Kelvin (2700=warm, 6500=cool)
|
||||
|
||||
@@ -177,10 +194,9 @@ async def turn_on(
|
||||
Confirmation of the action
|
||||
|
||||
Examples:
|
||||
turn_on("light.living_room") # Turn on at current brightness
|
||||
turn_on("light.bedroom", brightness=128) # Turn on at 50% brightness
|
||||
turn_on("light.office", brightness=255, color_temp=4000) # Full, neutral white
|
||||
turn_on("switch.coffee_maker") # Turn on a switch
|
||||
turn_on(entity_id="light.living_room")
|
||||
turn_on(entity_id="light.bedroom", brightness=128)
|
||||
turn_on(entity_id="switch.coffee_maker")
|
||||
"""
|
||||
try:
|
||||
async with CoreAPIClient() as client:
|
||||
@@ -209,17 +225,18 @@ async def turn_on(
|
||||
|
||||
async def turn_off(entity_id: str) -> str:
|
||||
"""
|
||||
Turn off a device.
|
||||
Turn off a device. Use the entity_id parameter with the EXACT value from list_devices.
|
||||
|
||||
Args:
|
||||
entity_id: Device to turn off (e.g., light.living_room, switch.coffee_maker)
|
||||
entity_id: The EXACT entity ID from list_devices including domain prefix.
|
||||
|
||||
Returns:
|
||||
Confirmation of the action
|
||||
|
||||
Examples:
|
||||
turn_off("light.living_room")
|
||||
turn_off("switch.coffee_maker")
|
||||
turn_off(entity_id="light.living_room")
|
||||
turn_off(entity_id="switch.coffee_maker")
|
||||
turn_off(entity_id="light.kitchen")
|
||||
"""
|
||||
try:
|
||||
async with CoreAPIClient() as client:
|
||||
@@ -239,15 +256,17 @@ async def toggle(entity_id: str) -> str:
|
||||
"""
|
||||
Toggle a device's state (on becomes off, off becomes on).
|
||||
|
||||
Use the entity_id parameter with the EXACT value from list_devices.
|
||||
|
||||
Args:
|
||||
entity_id: Device to toggle
|
||||
entity_id: The EXACT entity ID from list_devices including domain prefix.
|
||||
|
||||
Returns:
|
||||
Confirmation with the new state
|
||||
|
||||
Examples:
|
||||
toggle("light.living_room")
|
||||
toggle("switch.fan")
|
||||
toggle(entity_id="light.living_room")
|
||||
toggle(entity_id="switch.fan")
|
||||
"""
|
||||
try:
|
||||
async with CoreAPIClient() as client:
|
||||
|
||||
@@ -39,7 +39,14 @@ Your role is to help users find, understand, synthesize, and manage information
|
||||
- The personal wiki (Wiki.js) containing documentation and notes
|
||||
- The knowledge graph (Neo4j) with entities and relationships
|
||||
- Vector embeddings (Qdrant) for semantic search
|
||||
- Web search (SearXNG) for current information
|
||||
- Paperless documents (📑) - indexed PDFs, scanned documents, invoices, receipts from the user's document archive
|
||||
- Volatile cache (⚡) - pre-fetched real-time data for user-relevant locations and items:
|
||||
- weather/forecast: conditions and forecasts for user's configured cities
|
||||
- news: headlines from user's preferred sources
|
||||
- stock/crypto: quotes for user's watched symbols
|
||||
- sun/air_quality: data for user's locations
|
||||
- Note: volatile data may not exist for arbitrary queries - falls back to web search
|
||||
- Web search (SearXNG) for current information not available in cache
|
||||
|
||||
## Your Personality
|
||||
- Scholarly and thorough in your research
|
||||
@@ -60,7 +67,13 @@ Your role is to help users find, understand, synthesize, and manage information
|
||||
- Use for: comparing multiple sources, gathering info from several pages
|
||||
|
||||
### Internal Research Tools
|
||||
- **hybrid_search**: Your primary research tool - searches wiki, graph, and web at once
|
||||
- **hybrid_search**: Your primary research tool - searches ALL sources at once:
|
||||
- Wiki pages (vector similarity)
|
||||
- Knowledge graph (entity relationships)
|
||||
- Paperless documents (📑 indexed PDFs, scans)
|
||||
- Volatile cache (⚡ weather, news, stocks - when available)
|
||||
- Web search (current information)
|
||||
Results are fused and re-ranked by relevance. Volatile data gets priority when fresh.
|
||||
- **search_wiki**: Find specific wiki pages by keyword
|
||||
- **semantic_search**: Find conceptually similar content
|
||||
- **explore_knowledge_graph** / **find_related_entities**: Discover connections
|
||||
@@ -114,6 +127,14 @@ Your responses are returned to Tatlock (the butler) who will synthesize them int
|
||||
- Note any gaps in available information
|
||||
- Be concise but thorough - Tatlock will format the final response
|
||||
- Structure your findings clearly so they can be easily integrated with other responses
|
||||
|
||||
## CRITICAL: Never Fabricate Information
|
||||
If a tool fails or you cannot access a data source:
|
||||
- Say "I was unable to retrieve [information type]" - be specific about what failed
|
||||
- Do NOT provide placeholder, template, or made-up data
|
||||
- Do NOT say "Here's what I would have said" or "Here's a sample response"
|
||||
- Do NOT invent specific numbers, dates, or facts when the actual data is unavailable
|
||||
- It is better to return no information than to return fabricated information
|
||||
"""
|
||||
|
||||
# Lazy initialization to avoid connection issues during imports
|
||||
@@ -122,16 +143,10 @@ _librarian_agent: Optional[Agent[None, str]] = None
|
||||
|
||||
def _create_librarian_agent() -> Agent[None, str]:
|
||||
"""Create the Librarian PydanticAI agent."""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
# Create Ollama model with sanitized provider
|
||||
# (fixes 'content: null' issue with tool calls)
|
||||
model = OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
agent: Agent[None, str] = Agent(
|
||||
model=model,
|
||||
@@ -161,9 +176,12 @@ def _create_librarian_agent() -> Agent[None, str]:
|
||||
agent.tool_plain(update_wiki_page)
|
||||
agent.tool_plain(smart_create_wiki_page)
|
||||
|
||||
from src.anthropic.model_selector import get_model_info
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"librarian_agent_created",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
tool_count=14, # 7 research + 3 web + 1 wiki read + 3 wiki write
|
||||
)
|
||||
|
||||
|
||||
@@ -225,18 +225,22 @@ class LibraryDeskClient:
|
||||
vector_limit: int = 10,
|
||||
graph_limit: int = 10,
|
||||
web_limit: int = 5,
|
||||
document_limit: int = 5,
|
||||
volatile_limit: int = 3,
|
||||
enable_reranking: bool = True,
|
||||
final_result_count: int = 10,
|
||||
) -> HybridRAGResponse:
|
||||
"""
|
||||
Execute HybridRAG search combining vector, graph, and web results.
|
||||
Execute HybridRAG search combining vector, graph, documents, volatile, and web.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
user: User identifier for multi-tenancy (defaults to request context)
|
||||
vector_limit: Max results from vector search
|
||||
vector_limit: Max results from vector search (wiki pages)
|
||||
graph_limit: Max results from graph search
|
||||
web_limit: Max results from web search
|
||||
web_limit: Max results from web search (0 to disable)
|
||||
document_limit: Max results from Paperless documents (0 to disable)
|
||||
volatile_limit: Max results from volatile cache (0 to disable)
|
||||
enable_reranking: Whether to rerank with LLM
|
||||
final_result_count: Number of final results after fusion
|
||||
|
||||
@@ -252,6 +256,11 @@ class LibraryDeskClient:
|
||||
"vector_limit": vector_limit,
|
||||
"graph_limit": graph_limit,
|
||||
"web_limit": web_limit,
|
||||
"document_limit": document_limit,
|
||||
"volatile_limit": volatile_limit,
|
||||
"enable_documents": document_limit > 0,
|
||||
"enable_volatile": volatile_limit > 0,
|
||||
"enable_web": web_limit > 0,
|
||||
"enable_reranking": enable_reranking,
|
||||
"final_result_count": final_result_count,
|
||||
},
|
||||
|
||||
@@ -17,20 +17,26 @@ logger = get_logger(__name__)
|
||||
async def hybrid_search(
|
||||
query: str,
|
||||
include_web: bool = True,
|
||||
include_documents: bool = True,
|
||||
include_volatile: bool = True,
|
||||
) -> str:
|
||||
"""
|
||||
Search across all knowledge sources using HybridRAG.
|
||||
|
||||
This is the primary research tool, combining:
|
||||
- Vector search (semantic similarity over documents)
|
||||
- Vector search (semantic similarity over wiki pages)
|
||||
- Knowledge graph (entities and relationships)
|
||||
- Paperless documents (📑 indexed PDFs, scans, invoices)
|
||||
- Volatile cache (⚡ weather, news, stocks - for user's configured items)
|
||||
- Web search (current information from SearXNG)
|
||||
|
||||
Results are fused and re-ranked by relevance.
|
||||
Results are fused and re-ranked by relevance. Volatile data gets priority when fresh.
|
||||
|
||||
Args:
|
||||
query: Natural language research query
|
||||
include_web: Whether to include web results (default: True)
|
||||
include_documents: Whether to include Paperless documents (default: True)
|
||||
include_volatile: Whether to include volatile cache data (default: True)
|
||||
|
||||
Returns:
|
||||
Formatted search results with sources and context
|
||||
@@ -38,12 +44,16 @@ async def hybrid_search(
|
||||
Examples:
|
||||
hybrid_search("How does Docker orchestration work with Kubernetes?")
|
||||
hybrid_search("What projects use Neo4j?", include_web=False)
|
||||
hybrid_search("Find my electricity invoices", include_web=False, include_volatile=False)
|
||||
hybrid_search("What's the weather in Rotterdam?") # May hit volatile cache
|
||||
"""
|
||||
try:
|
||||
async with LibraryDeskClient() as client:
|
||||
response = await client.hybrid_search(
|
||||
query=query,
|
||||
web_limit=5 if include_web else 0,
|
||||
document_limit=5 if include_documents else 0,
|
||||
volatile_limit=3 if include_volatile else 0,
|
||||
)
|
||||
|
||||
if not response.results:
|
||||
@@ -70,6 +80,8 @@ async def hybrid_search(
|
||||
"vector": "📄",
|
||||
"graph": "🔗",
|
||||
"web": "🌐",
|
||||
"document": "📑",
|
||||
"volatile": "⚡",
|
||||
}.get(result.source, "•")
|
||||
|
||||
output_parts.append(
|
||||
|
||||
+110
-29
@@ -5,11 +5,13 @@ The Steward analyzes incoming requests, identifies relevant household
|
||||
capabilities, and provides focused recommendations to Tatlock (the Butler).
|
||||
This creates a two-tier architecture that prevents cognitive overload.
|
||||
|
||||
Uses plain text output (not JSON) for reliability with Ollama models.
|
||||
Uses plain text output (not JSON) for reliability. Supports both Claude
|
||||
(preferred) and Ollama (fallback) backends via direct API calls.
|
||||
"""
|
||||
import httpx
|
||||
from typing import Optional
|
||||
|
||||
from src.anthropic.model_selector import is_claude_available, get_model_info
|
||||
from src.core.config import config
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
@@ -56,14 +58,22 @@ USER QUERY: {query}
|
||||
GUIDELINES:
|
||||
- Be conservative - only recommend truly necessary capabilities
|
||||
- Simple greetings/chat → no capabilities needed (conversational response only)
|
||||
- Questions about prior conversation ("what did I say", "my name", "what we discussed") → no capabilities (Tatlock has full history)
|
||||
- Questions about prior conversation ("what did I say", "what we discussed") → no capabilities (Tatlock has full history)
|
||||
- Math/calculations → tatlock_core
|
||||
- Time/date queries → tatlock_core
|
||||
- PERSONAL MEMORY queries → biographer to recall (ALWAYS use for questions about the user themselves):
|
||||
- "where do I live", "what's my location", "my address" → biographer to recall location
|
||||
- "what's my name", "who am I" → biographer to recall name
|
||||
- "what car do I drive", "my vehicle" → biographer to recall car
|
||||
- "what do you know about me", "what have I told you" → biographer to recall or list_memories
|
||||
- "remember that I...", "store that..." → biographer to store_insight
|
||||
- "forget my...", "delete..." → biographer to forget_memory
|
||||
- "my timezone", "my preferences" → biographer to recall preferences
|
||||
- Web searches, weather, news, current information → librarian with search_web
|
||||
- Read a URL or article → librarian with read_url
|
||||
- Wiki creation ("create a page about X", "add X to wiki") → librarian with smart_create
|
||||
- Wiki updates ("update the page", "add to dossier") → librarian with update
|
||||
- Research queries ("find info", "what do we know about", "search for") → librarian with hybrid_search
|
||||
- Research queries about TOPICS (not about the user) → librarian with hybrid_search
|
||||
- In-depth research, knowledge synthesis, document lookup → librarian with hybrid_search
|
||||
- If conversation history is relevant, note which previous turns matter
|
||||
- Assess complexity: simple (1 tool), moderate (2-3 tools), complex (multiple steps)
|
||||
@@ -75,6 +85,10 @@ COMPLEXITY: [simple/moderate/complex]
|
||||
CONTEXT: [any relevant conversation context, or "none"]
|
||||
|
||||
EXAMPLES:
|
||||
- "DELEGATE: biographer to recall the user's location" (for "where do I live?")
|
||||
- "DELEGATE: biographer to recall the user's car" (for "what car do I drive?")
|
||||
- "DELEGATE: biographer to list_memories about the user" (for "what do you know about me?")
|
||||
- "DELEGATE: biographer to store_insight about user's pet" (for "remember that I have a dog named Max")
|
||||
- "DELEGATE: librarian to search_web for tomorrow's weather forecast"
|
||||
- "DELEGATE: librarian to create a wiki page about CI/CD pipelines"
|
||||
- "DELEGATE: librarian to hybrid_search for information about Docker networking"
|
||||
@@ -93,22 +107,74 @@ class StewardAgent:
|
||||
Analyzes requests with full conversation context and recommends
|
||||
which household capabilities the Butler should use.
|
||||
|
||||
Uses plain text output for reliability with Ollama models.
|
||||
Uses plain text output for reliability. Supports both Claude
|
||||
(preferred) and Ollama (fallback) backends via direct API calls.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Steward with Ollama model (same as Tatlock for VRAM efficiency)."""
|
||||
"""Initialize Steward with backend selection based on availability."""
|
||||
# Ollama config (fallback)
|
||||
self.ollama_host = str(config.OLLAMA_HOST).rstrip('/')
|
||||
self.model_name = config.OLLAMA_DEFAULT_MODEL
|
||||
self.ollama_model = config.OLLAMA_DEFAULT_MODEL
|
||||
|
||||
# Claude config (preferred)
|
||||
self.claude_model = config.ANTHROPIC_MODEL
|
||||
self._anthropic_client = None
|
||||
|
||||
# Determine which backend to use
|
||||
self._use_claude = config.PREFER_CLOUD_BACKEND and is_claude_available()
|
||||
|
||||
self.timeout = 30.0 # 30 second timeout for analysis
|
||||
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"steward_agent_created",
|
||||
ollama_host=self.ollama_host,
|
||||
model=self.model_name,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
def _get_anthropic_client(self):
|
||||
"""Get or create Anthropic client (lazy initialization)."""
|
||||
if self._anthropic_client is None:
|
||||
from anthropic import AsyncAnthropic
|
||||
self._anthropic_client = AsyncAnthropic(api_key=config.ANTHROPIC_API_KEY)
|
||||
return self._anthropic_client
|
||||
|
||||
async def _call_claude(self, system_prompt: str, user_message: str) -> str:
|
||||
"""Call Claude API directly for plain text generation."""
|
||||
client = self._get_anthropic_client()
|
||||
|
||||
response = await client.messages.create(
|
||||
model=self.claude_model,
|
||||
max_tokens=1024,
|
||||
system=system_prompt,
|
||||
messages=[{"role": "user", "content": user_message}],
|
||||
temperature=0.3, # Lower = more consistent
|
||||
)
|
||||
|
||||
return response.content[0].text.strip()
|
||||
|
||||
async def _call_ollama(self, prompt: str) -> str:
|
||||
"""Call Ollama API directly for plain text generation."""
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.ollama_host}/api/generate",
|
||||
json={
|
||||
"model": self.ollama_model,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.3, # Lower = more consistent
|
||||
"top_p": 0.9
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
return result["response"].strip()
|
||||
|
||||
async def analyze(
|
||||
self,
|
||||
query: str,
|
||||
@@ -117,6 +183,8 @@ class StewardAgent:
|
||||
"""
|
||||
Analyze query and return plain text recommendation.
|
||||
|
||||
Uses Claude if available, falls back to Ollama.
|
||||
|
||||
Args:
|
||||
query: User's query to analyze
|
||||
conversation_history: Previous conversation turns
|
||||
@@ -132,35 +200,48 @@ class StewardAgent:
|
||||
history = conversation_history or []
|
||||
prompt = build_steward_prompt(query, history)
|
||||
|
||||
logger.debug("steward_calling_ollama", query_preview=query[:100])
|
||||
backend = "claude" if self._use_claude else "ollama"
|
||||
logger.debug(
|
||||
"steward_calling_llm",
|
||||
backend=backend,
|
||||
query_preview=query[:100],
|
||||
)
|
||||
|
||||
# Call Ollama API directly (more reliable than PydanticAI for plain text)
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.ollama_host}/api/generate",
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.3, # Lower = more consistent
|
||||
"top_p": 0.9
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
|
||||
analysis_text = result["response"].strip()
|
||||
try:
|
||||
if self._use_claude:
|
||||
# For Claude, split into system + user message
|
||||
# The prompt contains both, but Claude prefers explicit system
|
||||
analysis_text = await self._call_claude(
|
||||
system_prompt="You are the Steward of the household, advising the Butler (Tatlock) on which capabilities to use. Be concise and specific.",
|
||||
user_message=prompt,
|
||||
)
|
||||
else:
|
||||
analysis_text = await self._call_ollama(prompt)
|
||||
|
||||
logger.debug(
|
||||
"steward_analysis_received",
|
||||
text_preview=analysis_text[:150]
|
||||
backend=backend,
|
||||
text_preview=analysis_text[:150],
|
||||
)
|
||||
|
||||
return analysis_text
|
||||
|
||||
except Exception as e:
|
||||
# If Claude fails, try Ollama as fallback
|
||||
if self._use_claude:
|
||||
logger.warning(
|
||||
"steward_claude_fallback",
|
||||
error=str(e),
|
||||
)
|
||||
analysis_text = await self._call_ollama(prompt)
|
||||
logger.debug(
|
||||
"steward_analysis_received",
|
||||
backend="ollama_fallback",
|
||||
text_preview=analysis_text[:150],
|
||||
)
|
||||
return analysis_text
|
||||
raise
|
||||
|
||||
|
||||
# Global Steward instance
|
||||
_steward_agent = None
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""
|
||||
Steward service layer.
|
||||
|
||||
Provides high-level interface for request analysis with logging,
|
||||
benchmarking, and error handling.
|
||||
Provides high-level interface for request analysis with logging
|
||||
and error handling.
|
||||
|
||||
Parses plain text recommendations into structured data.
|
||||
Includes memory pre-fetch for user context injection.
|
||||
@@ -10,7 +10,6 @@ Includes memory pre-fetch for user context injection.
|
||||
import re
|
||||
from typing import Any, Optional
|
||||
|
||||
from src.core.benchmarks import PerformanceBenchmark, get_benchmark_store
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger, log_operation
|
||||
from src.core.memory_service import memory_service
|
||||
@@ -237,7 +236,9 @@ async def _prefetch_memory_context(user_request: str) -> dict[str, Any]:
|
||||
# Location-related queries
|
||||
if any(word in request_lower for word in [
|
||||
"weather", "temperature", "forecast", "nearby", "local",
|
||||
"directions", "distance", "map", "here"
|
||||
"directions", "distance", "map", "here",
|
||||
# Direct location questions
|
||||
"live", "where", "home", "reside", "location", "address",
|
||||
]):
|
||||
profile_keys.append("location")
|
||||
|
||||
@@ -285,8 +286,7 @@ async def analyze_request(
|
||||
This is the main entry point for Steward analysis. It:
|
||||
1. Calls the Steward agent with full conversation history
|
||||
2. Logs the operation with timing
|
||||
3. Records performance benchmarks to Redis
|
||||
4. Returns structured recommendations
|
||||
3. Returns structured recommendations
|
||||
|
||||
Args:
|
||||
user_request: The current user message to analyze
|
||||
@@ -365,23 +365,6 @@ async def analyze_request(
|
||||
reasoning=analysis_text[:200], # First 200 chars
|
||||
)
|
||||
|
||||
# Record performance benchmark
|
||||
if log_ctx.get("duration_seconds"):
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="steward_analysis",
|
||||
duration_seconds=log_ctx["duration_seconds"],
|
||||
success=True,
|
||||
recommendation_count=len(recommendation.recommended_capabilities),
|
||||
confidence=None, # Could add confidence scoring in future
|
||||
conversation_id=conversation_id,
|
||||
metadata={
|
||||
"complexity": recommendation.estimated_complexity,
|
||||
"has_context": recommendation.conversation_context.has_previous_context,
|
||||
"missing_capabilities": recommendation.missing_capabilities is not None,
|
||||
},
|
||||
)
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
return recommendation
|
||||
|
||||
except Exception as e:
|
||||
|
||||
+91
-68
@@ -20,6 +20,11 @@ from src.agents.tatlock_core.tools import (
|
||||
)
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import (
|
||||
start_span, end_span, get_current_span,
|
||||
add_tool_spans_from_messages,
|
||||
SpanType, SpanStatus,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -42,7 +47,16 @@ def generate_id() -> str:
|
||||
# System prompt defining Tatlock's personality
|
||||
TATLOCK_SYSTEM_PROMPT = """You are Tatlock, a helpful personal assistant with the demeanor of a British butler.
|
||||
|
||||
Address users as "sir" and maintain a formal yet personable tone. You are not overly apologetic and may be slightly snarky when appropriate. If an opportunity for a pun presents itself, you cannot resist.
|
||||
## Personality
|
||||
|
||||
Address users as "sir". Be confident, direct, and efficient - you are an unflappable English butler who gets things done. Dry wit and puns are encouraged.
|
||||
|
||||
**CRITICAL - Do NOT:**
|
||||
- Apologize unless you genuinely made an error
|
||||
- Say "Apologies for any confusion" or "Allow me to rectify" when nothing went wrong
|
||||
- Preface successful results with caveats or apologies
|
||||
|
||||
When presenting findings: lead with the answer, be concise, skip the preamble.
|
||||
|
||||
You coordinate with various household staff (expert agents) to provide comprehensive assistance across:
|
||||
- Research and knowledge work
|
||||
@@ -124,10 +138,7 @@ class TatlockAgent(AgentInterface):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Tatlock configuration (lazy agent creation)."""
|
||||
# Store Ollama configuration
|
||||
self.ollama_host = str(config.OLLAMA_HOST)
|
||||
self.model_name = config.OLLAMA_DEFAULT_MODEL
|
||||
"""Initialize Tatlock (lazy agent creation)."""
|
||||
self._agent = None # Lazy initialization
|
||||
|
||||
def _ensure_agent(self):
|
||||
@@ -135,30 +146,21 @@ class TatlockAgent(AgentInterface):
|
||||
if self._agent is not None:
|
||||
return
|
||||
|
||||
from src.anthropic.model_selector import get_model, get_model_info
|
||||
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"tatlock_agent_initializing",
|
||||
ollama_host=self.ollama_host,
|
||||
model=self.model_name,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
)
|
||||
|
||||
# Import required classes for Ollama configuration
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
# PydanticAI expects Ollama base URL to end with /v1
|
||||
# Remove trailing slash from ollama_host if present
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
# Create Ollama model with provider
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
|
||||
# Create PydanticAI agent with Ollama model
|
||||
# Create PydanticAI agent
|
||||
self._agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
)
|
||||
|
||||
@@ -433,7 +435,7 @@ class TatlockAgent(AgentInterface):
|
||||
steward_note: Note from Steward (prepended to request, invisible to user)
|
||||
scoped_tools: List of tool definitions from household registry
|
||||
message_history: Conversation history in PydanticAI format
|
||||
tool_tracker: Optional tool call tracker for benchmarking
|
||||
tool_tracker: Optional tool call tracker for analysis
|
||||
|
||||
Returns:
|
||||
str: Tatlock's response text
|
||||
@@ -447,8 +449,7 @@ class TatlockAgent(AgentInterface):
|
||||
... tool_tracker=tracker,
|
||||
... )
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_run_with_scoped_tools",
|
||||
@@ -459,18 +460,12 @@ class TatlockAgent(AgentInterface):
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
# This ensures Tatlock can ONLY use tools recommended by the Steward
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools
|
||||
# Tools from household registry are already PydanticAI Tool objects
|
||||
scoped_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
tools=scoped_tools, # Pass tools directly to Agent constructor
|
||||
)
|
||||
@@ -541,8 +536,7 @@ class TatlockAgent(AgentInterface):
|
||||
Yields:
|
||||
Text chunks from the streaming response
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_run_with_scoped_tools_stream",
|
||||
@@ -552,17 +546,11 @@ class TatlockAgent(AgentInterface):
|
||||
)
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools
|
||||
scoped_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
tools=scoped_tools,
|
||||
)
|
||||
@@ -627,7 +615,7 @@ class TatlockAgent(AgentInterface):
|
||||
steward_note: Note from Steward (invisible to user)
|
||||
scoped_tools: List of tool definitions from household registry
|
||||
message_history: Conversation history
|
||||
tool_tracker: Optional tool call tracker for benchmarking
|
||||
tool_tracker: Optional tool call tracker for analysis
|
||||
|
||||
Returns:
|
||||
dict with:
|
||||
@@ -636,8 +624,6 @@ class TatlockAgent(AgentInterface):
|
||||
- tool_outputs: Dict mapping tool names to their outputs
|
||||
- raw_output: The agent's raw text output
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
from pydantic_ai.messages import (
|
||||
ModelRequest,
|
||||
@@ -647,6 +633,7 @@ class TatlockAgent(AgentInterface):
|
||||
ToolCallPart,
|
||||
ToolReturnPart,
|
||||
)
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_orchestrate_tool_calls",
|
||||
@@ -655,18 +642,22 @@ class TatlockAgent(AgentInterface):
|
||||
history_length=len(message_history),
|
||||
)
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
# Start tracing span for orchestration phase
|
||||
orchestrate_span = start_span(
|
||||
"tatlock_orchestrate",
|
||||
SpanType.TATLOCK,
|
||||
metadata={
|
||||
"scoped_tool_count": len(scoped_tools),
|
||||
"tool_names": [getattr(t, '__name__', str(t)) for t in scoped_tools[:5]],
|
||||
},
|
||||
)
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools
|
||||
scoped_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
tools=scoped_tools,
|
||||
)
|
||||
@@ -731,6 +722,23 @@ class TatlockAgent(AgentInterface):
|
||||
tool_output_count=len(tool_outputs),
|
||||
)
|
||||
|
||||
# Add tool-level spans from result messages
|
||||
if orchestrate_span:
|
||||
add_tool_spans_from_messages(result.new_messages(), orchestrate_span)
|
||||
|
||||
# End orchestration span with results
|
||||
end_span(
|
||||
orchestrate_span,
|
||||
metadata_update={
|
||||
"tools_called": tools_called,
|
||||
"expert_count": len(expert_results),
|
||||
"tool_output_count": len(tool_outputs),
|
||||
},
|
||||
details_update={
|
||||
"steward_note_preview": steward_note[:500] if steward_note else None,
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"tools_called": tools_called,
|
||||
"expert_results": expert_results,
|
||||
@@ -758,9 +766,8 @@ class TatlockAgent(AgentInterface):
|
||||
Returns:
|
||||
str: Butler-toned response synthesized from all results
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from pydantic_ai.messages import ModelRequest, ModelResponse, UserPromptPart, TextPart
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_synthesize_from_results",
|
||||
@@ -769,6 +776,16 @@ class TatlockAgent(AgentInterface):
|
||||
tool_count=len(orchestration_results.get("tool_outputs", {})),
|
||||
)
|
||||
|
||||
# Start tracing span for synthesis phase
|
||||
synthesize_span = start_span(
|
||||
"tatlock_synthesize",
|
||||
SpanType.TATLOCK,
|
||||
metadata={
|
||||
"expert_count": len(orchestration_results.get("expert_results", {})),
|
||||
"tool_output_count": len(orchestration_results.get("tool_outputs", {})),
|
||||
},
|
||||
)
|
||||
|
||||
# Build synthesis prompt with all available information
|
||||
synthesis_parts = []
|
||||
synthesis_parts.append(f"The user asked: {user_message}")
|
||||
@@ -789,25 +806,19 @@ class TatlockAgent(AgentInterface):
|
||||
synthesis_parts.append("")
|
||||
|
||||
synthesis_parts.append(
|
||||
"Based on this information, provide a response to the user. "
|
||||
"Maintain your butler personality - address them as 'sir', "
|
||||
"use formal but personable language, and be helpful."
|
||||
"Synthesize a response for the user. Be direct and confident. "
|
||||
"Lead with the answer - no apologies, no caveats, no 'mix-ups'. "
|
||||
"Address them as 'sir', be concise, add dry wit if appropriate."
|
||||
)
|
||||
|
||||
synthesis_prompt = "\n".join(synthesis_parts)
|
||||
|
||||
# Create synthesis agent (no tools needed)
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
model = get_model()
|
||||
|
||||
# Synthesis agent uses butler prompt but no tools
|
||||
synthesis_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
# No tools for synthesis phase
|
||||
)
|
||||
@@ -841,6 +852,18 @@ class TatlockAgent(AgentInterface):
|
||||
response_preview=result.output[:100],
|
||||
)
|
||||
|
||||
# End synthesis span with result
|
||||
end_span(
|
||||
synthesize_span,
|
||||
metadata_update={
|
||||
"response_length": len(result.output),
|
||||
},
|
||||
details_update={
|
||||
"synthesis_prompt": synthesis_prompt[:1000],
|
||||
"response_preview": result.output[:500],
|
||||
},
|
||||
)
|
||||
|
||||
return result.output
|
||||
|
||||
async def get_capabilities(self) -> dict:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""
|
||||
Anthropic/Claude integration module.
|
||||
|
||||
Provides model selection with automatic fallback between Claude and Ollama.
|
||||
"""
|
||||
|
||||
from src.anthropic.model_selector import (
|
||||
check_claude_health,
|
||||
get_model,
|
||||
is_claude_available,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"check_claude_health",
|
||||
"get_model",
|
||||
"is_claude_available",
|
||||
]
|
||||
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
Model selector for Claude/Ollama backend switching.
|
||||
|
||||
Provides automatic model selection with Claude as preferred backend
|
||||
and Ollama as offline fallback.
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from pydantic_ai.providers.anthropic import AnthropicProvider
|
||||
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Cached health check result (set once at startup)
|
||||
_claude_available: bool | None = None
|
||||
|
||||
|
||||
async def check_claude_health() -> bool:
|
||||
"""
|
||||
Check if Claude API is reachable and working.
|
||||
|
||||
This should be called once at application startup.
|
||||
The result is cached in `_claude_available`.
|
||||
|
||||
Returns:
|
||||
True if Claude API is accessible, False otherwise.
|
||||
"""
|
||||
global _claude_available
|
||||
|
||||
# No API key configured - Claude not available
|
||||
if not config.ANTHROPIC_API_KEY:
|
||||
logger.info(
|
||||
"claude_health_check_skipped",
|
||||
reason="no_api_key",
|
||||
)
|
||||
_claude_available = False
|
||||
return False
|
||||
|
||||
try:
|
||||
from anthropic import AsyncAnthropic
|
||||
|
||||
client = AsyncAnthropic(api_key=config.ANTHROPIC_API_KEY)
|
||||
|
||||
# Minimal API call to verify connectivity
|
||||
# Using a tiny max_tokens to minimize cost
|
||||
await client.messages.create(
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
max_tokens=1,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
_claude_available = True
|
||||
logger.info(
|
||||
"claude_health_check_passed",
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
_claude_available = False
|
||||
logger.warning(
|
||||
"claude_health_check_failed",
|
||||
error=str(e),
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def is_claude_available() -> bool:
|
||||
"""
|
||||
Check if Claude is available (from cached health check result).
|
||||
|
||||
Returns:
|
||||
True if Claude API was reachable at startup, False otherwise.
|
||||
|
||||
Note:
|
||||
Returns False if health check hasn't been run yet.
|
||||
Call `check_claude_health()` at startup first.
|
||||
"""
|
||||
return _claude_available is True
|
||||
|
||||
|
||||
def get_model(prefer_cloud: bool | None = None) -> Union[AnthropicModel, OpenAIChatModel]:
|
||||
"""
|
||||
Get the best available model.
|
||||
|
||||
Returns Claude if available and preferred, otherwise Ollama.
|
||||
|
||||
Args:
|
||||
prefer_cloud: Override config.PREFER_CLOUD_BACKEND for this call.
|
||||
If None, uses the config value.
|
||||
|
||||
Returns:
|
||||
PydanticAI model instance (AnthropicModel or OpenAIChatModel).
|
||||
|
||||
Example:
|
||||
>>> model = get_model()
|
||||
>>> agent = Agent(model, system_prompt="...")
|
||||
"""
|
||||
# Determine preference
|
||||
use_cloud = prefer_cloud if prefer_cloud is not None else config.PREFER_CLOUD_BACKEND
|
||||
|
||||
# Use Claude if available and preferred
|
||||
if use_cloud and is_claude_available():
|
||||
logger.debug(
|
||||
"model_selected",
|
||||
backend="claude",
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return AnthropicModel(
|
||||
model_name=config.ANTHROPIC_MODEL,
|
||||
provider=AnthropicProvider(api_key=config.ANTHROPIC_API_KEY),
|
||||
)
|
||||
|
||||
# Fall back to Ollama
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
logger.debug(
|
||||
"model_selected",
|
||||
backend="ollama",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
reason="fallback" if use_cloud else "preferred_local",
|
||||
)
|
||||
return OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
|
||||
|
||||
def get_model_info() -> dict:
|
||||
"""
|
||||
Get information about the current model configuration.
|
||||
|
||||
Useful for health checks and debugging.
|
||||
|
||||
Returns:
|
||||
Dict with backend, model name, and availability info.
|
||||
"""
|
||||
use_cloud = config.PREFER_CLOUD_BACKEND and is_claude_available()
|
||||
|
||||
return {
|
||||
"backend": "claude" if use_cloud else "ollama",
|
||||
"model": config.ANTHROPIC_MODEL if use_cloud else config.OLLAMA_DEFAULT_MODEL,
|
||||
"claude_available": is_claude_available(),
|
||||
"claude_configured": bool(config.ANTHROPIC_API_KEY),
|
||||
"prefer_cloud": config.PREFER_CLOUD_BACKEND,
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
"""
|
||||
Performance benchmark storage using Redis.
|
||||
|
||||
Tracks operation timing, tool usage, and recommendation accuracy across sessions.
|
||||
Provides time-series data for performance analysis and optimization.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
import redis.asyncio as redis
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .config import config
|
||||
from .logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class PerformanceBenchmark(BaseModel):
|
||||
"""
|
||||
Performance benchmark record.
|
||||
|
||||
Stores timing and metadata for operations like Steward analysis,
|
||||
tool calls, and agent execution.
|
||||
"""
|
||||
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
operation: str # "steward_analysis", "tool_call", "tatlock_execution"
|
||||
duration_seconds: float
|
||||
success: bool
|
||||
|
||||
# Steward-specific fields
|
||||
recommendation_count: Optional[int] = None
|
||||
confidence: Optional[float] = None
|
||||
|
||||
# Tool-specific fields
|
||||
tool_name: Optional[str] = None
|
||||
was_recommended: Optional[bool] = None
|
||||
was_actually_used: Optional[bool] = None
|
||||
|
||||
# Context
|
||||
conversation_id: Optional[str] = None
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
def to_redis_dict(self) -> dict[str, Any]:
|
||||
"""Convert to dict suitable for Redis storage."""
|
||||
data = self.model_dump()
|
||||
data["timestamp"] = self.timestamp.isoformat()
|
||||
data["metadata"] = json.dumps(self.metadata)
|
||||
# Convert booleans to strings (Redis doesn't accept bool type)
|
||||
for key, value in data.items():
|
||||
if isinstance(value, bool):
|
||||
data[key] = str(value)
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def from_redis_dict(cls, data: dict[str, Any]) -> "PerformanceBenchmark":
|
||||
"""Reconstruct from Redis dict."""
|
||||
data["timestamp"] = datetime.fromisoformat(data["timestamp"])
|
||||
data["metadata"] = json.loads(data.get("metadata", "{}"))
|
||||
# Convert string booleans back to bool
|
||||
for key in ["success", "was_recommended", "was_actually_used"]:
|
||||
if key in data and isinstance(data[key], str):
|
||||
data[key] = data[key] == "True"
|
||||
return cls(**data)
|
||||
|
||||
|
||||
class BenchmarkStore:
|
||||
"""
|
||||
Redis-backed benchmark storage with automatic expiry.
|
||||
|
||||
Stores performance metrics in time-series format with 30-day retention.
|
||||
Provides querying capabilities for analysis and reporting.
|
||||
"""
|
||||
|
||||
def __init__(self, redis_client: Optional[redis.Redis] = None):
|
||||
"""
|
||||
Initialize benchmark store.
|
||||
|
||||
Args:
|
||||
redis_client: Optional Redis client. If None, creates from config.
|
||||
"""
|
||||
self._client = redis_client
|
||||
self._ttl_days = 30 # 30-day retention
|
||||
|
||||
async def _get_client(self) -> redis.Redis:
|
||||
"""Get or create Redis client."""
|
||||
if self._client is None:
|
||||
self._client = redis.from_url(
|
||||
config.redis_url,
|
||||
encoding="utf-8",
|
||||
decode_responses=True,
|
||||
socket_timeout=config.REDIS_TIMEOUT,
|
||||
socket_connect_timeout=config.REDIS_TIMEOUT,
|
||||
)
|
||||
return self._client
|
||||
|
||||
async def record(self, benchmark: PerformanceBenchmark) -> None:
|
||||
"""
|
||||
Record a performance benchmark.
|
||||
|
||||
Args:
|
||||
benchmark: Performance benchmark to record
|
||||
|
||||
Example:
|
||||
>>> await store.record(PerformanceBenchmark(
|
||||
... operation="steward_analysis",
|
||||
... duration_seconds=1.23,
|
||||
... success=True,
|
||||
... recommendation_count=3,
|
||||
... ))
|
||||
"""
|
||||
if not config.ENABLE_BENCHMARKS:
|
||||
return
|
||||
|
||||
try:
|
||||
client = await self._get_client()
|
||||
|
||||
# Generate key: benchmark:{operation}:{timestamp_ms}
|
||||
timestamp_ms = int(benchmark.timestamp.timestamp() * 1000)
|
||||
key = f"benchmark:{benchmark.operation}:{timestamp_ms}"
|
||||
|
||||
# Store as hash
|
||||
await client.hset(key, mapping=benchmark.to_redis_dict())
|
||||
|
||||
# Set expiry
|
||||
await client.expire(key, self._ttl_days * 24 * 60 * 60)
|
||||
|
||||
# Add to sorted set for time-based queries
|
||||
index_key = f"benchmark_index:{benchmark.operation}"
|
||||
await client.zadd(index_key, {key: timestamp_ms})
|
||||
await client.expire(index_key, self._ttl_days * 24 * 60 * 60)
|
||||
|
||||
logger.debug(
|
||||
"benchmark_recorded",
|
||||
operation=benchmark.operation,
|
||||
duration=benchmark.duration_seconds,
|
||||
success=benchmark.success,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"benchmark_recording_failed",
|
||||
error=str(e),
|
||||
operation=benchmark.operation,
|
||||
)
|
||||
# Don't fail the request if benchmarking fails
|
||||
|
||||
async def query(
|
||||
self,
|
||||
operation: str,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
limit: int = 100,
|
||||
) -> list[PerformanceBenchmark]:
|
||||
"""
|
||||
Query benchmarks by operation and time range.
|
||||
|
||||
Args:
|
||||
operation: Operation name to filter by
|
||||
start_time: Start of time range (inclusive)
|
||||
end_time: End of time range (inclusive)
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of benchmarks matching the query
|
||||
|
||||
Example:
|
||||
>>> from datetime import timedelta
|
||||
>>> now = datetime.now(timezone.utc)
|
||||
>>> yesterday = now - timedelta(days=1)
|
||||
>>> benchmarks = await store.query(
|
||||
... "steward_analysis",
|
||||
... start_time=yesterday,
|
||||
... limit=50
|
||||
... )
|
||||
"""
|
||||
if not config.ENABLE_BENCHMARKS:
|
||||
return []
|
||||
|
||||
try:
|
||||
client = await self._get_client()
|
||||
index_key = f"benchmark_index:{operation}"
|
||||
|
||||
# Convert time range to timestamps
|
||||
min_score = (
|
||||
int(start_time.timestamp() * 1000)
|
||||
if start_time
|
||||
else "-inf"
|
||||
)
|
||||
max_score = (
|
||||
int(end_time.timestamp() * 1000)
|
||||
if end_time
|
||||
else "+inf"
|
||||
)
|
||||
|
||||
# Query sorted set
|
||||
keys = await client.zrevrangebyscore(
|
||||
index_key,
|
||||
max_score,
|
||||
min_score,
|
||||
start=0,
|
||||
num=limit,
|
||||
)
|
||||
|
||||
# Fetch benchmark data
|
||||
benchmarks = []
|
||||
for key in keys:
|
||||
data = await client.hgetall(key)
|
||||
if data:
|
||||
benchmarks.append(PerformanceBenchmark.from_redis_dict(data))
|
||||
|
||||
return benchmarks
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"benchmark_query_failed",
|
||||
error=str(e),
|
||||
operation=operation,
|
||||
)
|
||||
return []
|
||||
|
||||
async def get_statistics(
|
||||
self,
|
||||
operation: str,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Get aggregate statistics for an operation.
|
||||
|
||||
Args:
|
||||
operation: Operation name
|
||||
start_time: Start of time range
|
||||
end_time: End of time range
|
||||
|
||||
Returns:
|
||||
Dictionary with statistics (count, avg_duration, success_rate, etc.)
|
||||
|
||||
Example:
|
||||
>>> stats = await store.get_statistics("steward_analysis")
|
||||
>>> print(f"Average duration: {stats['avg_duration']}s")
|
||||
>>> print(f"Success rate: {stats['success_rate']}%")
|
||||
"""
|
||||
benchmarks = await self.query(operation, start_time, end_time, limit=1000)
|
||||
|
||||
if not benchmarks:
|
||||
return {
|
||||
"count": 0,
|
||||
"avg_duration": 0.0,
|
||||
"min_duration": 0.0,
|
||||
"max_duration": 0.0,
|
||||
"success_rate": 0.0,
|
||||
}
|
||||
|
||||
durations = [b.duration_seconds for b in benchmarks]
|
||||
successes = sum(1 for b in benchmarks if b.success)
|
||||
|
||||
return {
|
||||
"count": len(benchmarks),
|
||||
"avg_duration": sum(durations) / len(durations),
|
||||
"min_duration": min(durations),
|
||||
"max_duration": max(durations),
|
||||
"success_rate": (successes / len(benchmarks)) * 100,
|
||||
"total_successes": successes,
|
||||
"total_failures": len(benchmarks) - successes,
|
||||
}
|
||||
|
||||
async def get_tool_accuracy(
|
||||
self,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Analyze tool recommendation accuracy.
|
||||
|
||||
Compares recommended tools vs actually used tools to measure
|
||||
Steward's recommendation precision.
|
||||
|
||||
Args:
|
||||
start_time: Start of time range
|
||||
end_time: End of time range
|
||||
|
||||
Returns:
|
||||
Dictionary with accuracy metrics
|
||||
|
||||
Example:
|
||||
>>> accuracy = await store.get_tool_accuracy()
|
||||
>>> print(f"Precision: {accuracy['precision']}%")
|
||||
"""
|
||||
tool_calls = await self.query("tool_call", start_time, end_time, limit=1000)
|
||||
|
||||
if not tool_calls:
|
||||
return {
|
||||
"total_calls": 0,
|
||||
"recommended_and_used": 0,
|
||||
"recommended_not_used": 0,
|
||||
"not_recommended_but_used": 0,
|
||||
"precision": 0.0,
|
||||
}
|
||||
|
||||
recommended_and_used = sum(
|
||||
1 for b in tool_calls
|
||||
if b.was_recommended and b.was_actually_used
|
||||
)
|
||||
not_recommended_but_used = sum(
|
||||
1 for b in tool_calls
|
||||
if not b.was_recommended and b.was_actually_used
|
||||
)
|
||||
|
||||
total_used = sum(1 for b in tool_calls if b.was_actually_used)
|
||||
precision = (
|
||||
(recommended_and_used / total_used * 100) if total_used > 0 else 0.0
|
||||
)
|
||||
|
||||
return {
|
||||
"total_calls": len(tool_calls),
|
||||
"total_used": total_used,
|
||||
"recommended_and_used": recommended_and_used,
|
||||
"not_recommended_but_used": not_recommended_but_used,
|
||||
"precision": precision,
|
||||
}
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close Redis connection."""
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
|
||||
# Global benchmark store instance
|
||||
_benchmark_store: Optional[BenchmarkStore] = None
|
||||
|
||||
|
||||
def get_benchmark_store() -> BenchmarkStore:
|
||||
"""
|
||||
Get global benchmark store instance.
|
||||
|
||||
Returns:
|
||||
BenchmarkStore instance
|
||||
"""
|
||||
global _benchmark_store
|
||||
if _benchmark_store is None:
|
||||
_benchmark_store = BenchmarkStore()
|
||||
return _benchmark_store
|
||||
+16
-12
@@ -64,7 +64,21 @@ class Config(BaseSettings):
|
||||
API_PORT: int = Field(default=8000, description="API port")
|
||||
API_PREFIX: str = Field(default="/v1", description="API route prefix")
|
||||
|
||||
# Ollama Configuration
|
||||
# Anthropic Configuration (Claude - preferred backend)
|
||||
ANTHROPIC_API_KEY: str | None = Field(
|
||||
default=None,
|
||||
description="Anthropic API key for Claude access"
|
||||
)
|
||||
ANTHROPIC_MODEL: str = Field(
|
||||
default="claude-sonnet-4-20250514",
|
||||
description="Claude model to use"
|
||||
)
|
||||
PREFER_CLOUD_BACKEND: bool = Field(
|
||||
default=True,
|
||||
description="Prefer Claude over Ollama when available"
|
||||
)
|
||||
|
||||
# Ollama Configuration (local fallback)
|
||||
OLLAMA_HOST: HttpUrl = Field(
|
||||
default="http://localhost:11434",
|
||||
description="Ollama server URL"
|
||||
@@ -101,10 +115,6 @@ class Config(BaseSettings):
|
||||
default=6379,
|
||||
description="Redis server port"
|
||||
)
|
||||
REDIS_BENCHMARK_DB: int = Field(
|
||||
default=6,
|
||||
description="Redis database number for benchmarks"
|
||||
)
|
||||
REDIS_TIMEOUT: int = Field(
|
||||
default=5,
|
||||
description="Redis connection timeout in seconds"
|
||||
@@ -158,7 +168,7 @@ class Config(BaseSettings):
|
||||
description="Ollama model for embeddings"
|
||||
)
|
||||
|
||||
# Redis Memory Database (separate from benchmarks)
|
||||
# Redis Memory Database
|
||||
REDIS_MEMORY_DB: int = Field(
|
||||
default=1,
|
||||
description="Redis database number for memory cache"
|
||||
@@ -173,7 +183,6 @@ class Config(BaseSettings):
|
||||
default=None,
|
||||
description="Logging level (auto-set based on environment if not specified)"
|
||||
)
|
||||
ENABLE_BENCHMARKS: bool = Field(default=True, description="Enable performance benchmarking")
|
||||
|
||||
# User Configuration
|
||||
DEFAULT_USER: str | None = Field(
|
||||
@@ -190,11 +199,6 @@ class Config(BaseSettings):
|
||||
CORS_ALLOW_METHODS: list[str] = ["*"]
|
||||
CORS_ALLOW_HEADERS: list[str] = ["*"]
|
||||
|
||||
@property
|
||||
def redis_url(self) -> str:
|
||||
"""Construct Redis connection URL for benchmarks."""
|
||||
return f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/{self.REDIS_BENCHMARK_DB}"
|
||||
|
||||
@property
|
||||
def redis_memory_url(self) -> str:
|
||||
"""Construct Redis connection URL for memory cache."""
|
||||
|
||||
@@ -6,7 +6,7 @@ Provides short-term memory storage with TTL:
|
||||
- Recent entities mentioned in conversation
|
||||
- User-scoped with conversation isolation
|
||||
|
||||
Uses Redis DB 2 (separate from benchmarks in DB 1).
|
||||
Uses Redis DB 1.
|
||||
"""
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
@@ -11,6 +11,7 @@ from src.agents.steward import analyze_request, format_steward_note
|
||||
from src.agents.steward.schemas import StewardRecommendation
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import trace_span, SpanType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -93,12 +94,32 @@ async def preprocess_request(
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Call Steward with full conversation history
|
||||
recommendation = await analyze_request(
|
||||
enriched_request,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
# Call Steward with full conversation history (traced)
|
||||
async with trace_span(
|
||||
"steward_analysis",
|
||||
SpanType.STEWARD,
|
||||
metadata={
|
||||
"request_preview": user_request[:100],
|
||||
"history_length": len(conversation_history),
|
||||
},
|
||||
) as span:
|
||||
recommendation = await analyze_request(
|
||||
enriched_request,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Update span with results
|
||||
if span:
|
||||
span.metadata.update({
|
||||
"recommended_capabilities": recommendation.recommended_capabilities,
|
||||
"complexity": recommendation.estimated_complexity,
|
||||
"has_memory_context": bool(recommendation.memory_context),
|
||||
"has_conversation_context": recommendation.conversation_context.has_previous_context,
|
||||
})
|
||||
span.details["reasoning"] = recommendation.reasoning
|
||||
if recommendation.enriched_query:
|
||||
span.details["enriched_query"] = recommendation.enriched_query
|
||||
|
||||
# Format note for Tatlock (includes conversation context)
|
||||
steward_note = await format_steward_note(recommendation)
|
||||
|
||||
+15
-4
@@ -9,6 +9,7 @@ from src.agents.biographer import register_biographer
|
||||
from src.agents.housekeeper import register_housekeeper
|
||||
from src.agents.librarian import register_librarian
|
||||
from src.agents.tatlock_core import TATLOCK_CORE_CAPABILITY, tatlock_core_tools
|
||||
from src.anthropic.model_selector import check_claude_health, get_model_info
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
@@ -81,19 +82,29 @@ def register_household_members():
|
||||
)
|
||||
|
||||
|
||||
def initialize_application():
|
||||
async def initialize_application():
|
||||
"""
|
||||
Initialize the application.
|
||||
|
||||
Performs all startup tasks:
|
||||
1. Register household members
|
||||
2. (Future) Initialize connections
|
||||
3. (Future) Load configuration
|
||||
1. Check Claude API health (for backend selection)
|
||||
2. Register household members
|
||||
3. (Future) Initialize connections
|
||||
|
||||
This should be called once during application startup.
|
||||
"""
|
||||
logger.info("application_initialization_starting")
|
||||
|
||||
# Check Claude API health for backend selection
|
||||
await check_claude_health()
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"model_backend_configured",
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
claude_available=model_info["claude_available"],
|
||||
)
|
||||
|
||||
# Register household members
|
||||
register_household_members()
|
||||
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
"""
|
||||
Tool call tracking and benchmarking.
|
||||
Tool call tracking.
|
||||
|
||||
Tracks which tools are recommended by the Steward versus which tools
|
||||
are actually used by Tatlock, recording benchmarks for analysis.
|
||||
are actually used by Tatlock for debugging and analysis.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from src.core.benchmarks import PerformanceBenchmark, get_benchmark_store
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -15,7 +13,7 @@ logger = get_logger(__name__)
|
||||
|
||||
class ToolCallTracker:
|
||||
"""
|
||||
Tracks tool calls for benchmarking and accuracy analysis.
|
||||
Tracks tool calls for accuracy analysis.
|
||||
|
||||
Compares Steward's recommendations with Tatlock's actual tool usage
|
||||
to measure recommendation accuracy.
|
||||
@@ -53,6 +51,10 @@ class ToolCallTracker:
|
||||
return tool_name.replace("delegate_to_", "")
|
||||
return tool_name
|
||||
|
||||
def log_call(self, message: str):
|
||||
"""Log a tool call message (for UI display)."""
|
||||
logger.debug("tool_call_message", message=message)
|
||||
|
||||
async def track_call(self, tool_name: str, duration: float):
|
||||
"""
|
||||
Record a tool call with timing.
|
||||
@@ -78,23 +80,6 @@ class ToolCallTracker:
|
||||
recommended=list(self.recommended_capabilities),
|
||||
)
|
||||
|
||||
# Record benchmark to Redis
|
||||
benchmark = PerformanceBenchmark(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
operation="tool_call",
|
||||
duration_seconds=duration,
|
||||
success=True, # If we got here, the call succeeded
|
||||
tool_name=tool_name,
|
||||
was_recommended=was_recommended,
|
||||
was_actually_used=True,
|
||||
conversation_id=self.conversation_id,
|
||||
metadata={
|
||||
"recommended_capabilities": list(self.recommended_capabilities),
|
||||
},
|
||||
)
|
||||
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
logger.debug(
|
||||
"tool_call_tracked",
|
||||
tool_name=tool_name,
|
||||
@@ -124,24 +109,6 @@ class ToolCallTracker:
|
||||
conversation_id=self.conversation_id,
|
||||
)
|
||||
|
||||
# Record benchmarks for unused recommendations
|
||||
for tool_name in unused_tools:
|
||||
benchmark = PerformanceBenchmark(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
operation="tool_call",
|
||||
duration_seconds=0.0, # Not used
|
||||
success=True,
|
||||
tool_name=tool_name,
|
||||
was_recommended=True,
|
||||
was_actually_used=False,
|
||||
conversation_id=self.conversation_id,
|
||||
metadata={
|
||||
"recommended_capabilities": list(self.recommended_capabilities),
|
||||
"reason": "recommended_but_unused",
|
||||
},
|
||||
)
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
# Log summary
|
||||
total_calls = sum(len(durations) for durations in self.actual_calls.values())
|
||||
logger.info(
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
"""
|
||||
Lightweight request tracing for local development.
|
||||
|
||||
Captures the full request flow through Tatlock's multi-agent architecture
|
||||
as structured JSON traces for debugging and optimization.
|
||||
|
||||
Enable via DEBUG=true environment variable.
|
||||
|
||||
Traces are written to logs/traces/{trace_id}.json
|
||||
View with logs/traces/viewer.html
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import json
|
||||
import secrets
|
||||
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class SpanType(str, Enum):
|
||||
"""Types of traced operations."""
|
||||
ROUTER = "router"
|
||||
STEWARD = "steward"
|
||||
TATLOCK = "tatlock"
|
||||
EXPERT = "expert"
|
||||
TOOL = "tool"
|
||||
|
||||
|
||||
class SpanStatus(str, Enum):
|
||||
"""Span completion status."""
|
||||
OK = "ok"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Span:
|
||||
"""A single traced operation."""
|
||||
span_id: str
|
||||
name: str
|
||||
type: SpanType
|
||||
start_time: datetime
|
||||
parent_id: str | None = None
|
||||
end_time: datetime | None = None
|
||||
status: SpanStatus = SpanStatus.OK
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
children: list[str] = field(default_factory=list)
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def duration_ms(self) -> float | None:
|
||||
"""Calculate duration in milliseconds."""
|
||||
if self.end_time and self.start_time:
|
||||
return (self.end_time - self.start_time).total_seconds() * 1000
|
||||
return None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert span to dictionary for JSON serialization."""
|
||||
result = {
|
||||
"span_id": self.span_id,
|
||||
"parent_id": self.parent_id,
|
||||
"name": self.name,
|
||||
"type": self.type.value,
|
||||
"start_time": self.start_time.isoformat(),
|
||||
"end_time": self.end_time.isoformat() if self.end_time else None,
|
||||
"duration_ms": round(self.duration_ms, 2) if self.duration_ms else None,
|
||||
"status": self.status.value,
|
||||
"metadata": self.metadata if self.metadata else None,
|
||||
}
|
||||
# Only include non-empty optional fields
|
||||
if self.details:
|
||||
result["details"] = self.details
|
||||
if self.children:
|
||||
result["children"] = self.children
|
||||
if self.error:
|
||||
result["error"] = self.error
|
||||
return {k: v for k, v in result.items() if v is not None}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Trace:
|
||||
"""Complete trace of a request."""
|
||||
trace_id: str
|
||||
conversation_id: str | None
|
||||
user: str
|
||||
timestamp: datetime
|
||||
request: dict[str, Any]
|
||||
spans: list[Span] = field(default_factory=list)
|
||||
response: dict[str, Any] | None = None
|
||||
status: str = "in_progress"
|
||||
|
||||
@property
|
||||
def total_duration_ms(self) -> float | None:
|
||||
"""Calculate total trace duration from span timings."""
|
||||
if not self.spans:
|
||||
return None
|
||||
start = min(s.start_time for s in self.spans)
|
||||
ends = [s.end_time for s in self.spans if s.end_time]
|
||||
if not ends:
|
||||
return None
|
||||
end = max(ends)
|
||||
return (end - start).total_seconds() * 1000
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert trace to dictionary for JSON serialization."""
|
||||
return {
|
||||
"trace_id": self.trace_id,
|
||||
"conversation_id": self.conversation_id,
|
||||
"user": self.user,
|
||||
"timestamp": self.timestamp.isoformat(),
|
||||
"total_duration_ms": round(self.total_duration_ms, 2) if self.total_duration_ms else None,
|
||||
"status": self.status,
|
||||
"request": self.request,
|
||||
"response": self.response,
|
||||
"spans": [s.to_dict() for s in self.spans],
|
||||
}
|
||||
|
||||
|
||||
# ContextVar for async-safe trace propagation
|
||||
_current_trace: ContextVar[Trace | None] = ContextVar("current_trace", default=None)
|
||||
_current_span: ContextVar[Span | None] = ContextVar("current_span", default=None)
|
||||
|
||||
|
||||
def tracing_enabled() -> bool:
|
||||
"""Check if tracing is enabled (requires DEBUG=true)."""
|
||||
from src.core.config import config
|
||||
return config.DEBUG
|
||||
|
||||
|
||||
def _generate_id(prefix: str = "") -> str:
|
||||
"""Generate unique ID with optional prefix."""
|
||||
return f"{prefix}{secrets.token_hex(8)}"
|
||||
|
||||
|
||||
def start_trace(
|
||||
conversation_id: str | None,
|
||||
user: str,
|
||||
request: dict[str, Any],
|
||||
) -> Trace | None:
|
||||
"""
|
||||
Start a new trace for a request.
|
||||
|
||||
Args:
|
||||
conversation_id: Conversation identifier
|
||||
user: User identifier
|
||||
request: Request data (should include preview and full)
|
||||
|
||||
Returns:
|
||||
Trace object if tracing enabled, None otherwise
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
return None
|
||||
|
||||
trace = Trace(
|
||||
trace_id=_generate_id("trace_"),
|
||||
conversation_id=conversation_id,
|
||||
user=user,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
request=request,
|
||||
)
|
||||
_current_trace.set(trace)
|
||||
|
||||
logger.debug("trace_started", trace_id=trace.trace_id, user=user)
|
||||
return trace
|
||||
|
||||
|
||||
def get_current_trace() -> Trace | None:
|
||||
"""Get the current trace from context."""
|
||||
return _current_trace.get()
|
||||
|
||||
|
||||
def get_current_span() -> Span | None:
|
||||
"""Get the current span from context."""
|
||||
return _current_span.get()
|
||||
|
||||
|
||||
def start_span(
|
||||
name: str,
|
||||
span_type: SpanType,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> Span | None:
|
||||
"""
|
||||
Start a new span within the current trace.
|
||||
|
||||
Args:
|
||||
name: Span name (e.g., "steward_analysis")
|
||||
span_type: Type of operation
|
||||
metadata: Quick-access metadata (shown in timeline)
|
||||
details: Expandable details (prompts, full responses)
|
||||
|
||||
Returns:
|
||||
Span object if tracing enabled, None otherwise
|
||||
"""
|
||||
trace = get_current_trace()
|
||||
if not trace:
|
||||
return None
|
||||
|
||||
parent = get_current_span()
|
||||
span = Span(
|
||||
span_id=_generate_id("span_"),
|
||||
name=name,
|
||||
type=span_type,
|
||||
start_time=datetime.now(timezone.utc),
|
||||
parent_id=parent.span_id if parent else None,
|
||||
metadata=metadata or {},
|
||||
details=details or {},
|
||||
)
|
||||
|
||||
# Add to parent's children list
|
||||
if parent:
|
||||
parent.children.append(span.span_id)
|
||||
|
||||
trace.spans.append(span)
|
||||
_current_span.set(span)
|
||||
|
||||
logger.debug(
|
||||
"span_started",
|
||||
span_id=span.span_id,
|
||||
name=name,
|
||||
type=span_type.value,
|
||||
parent_id=span.parent_id,
|
||||
)
|
||||
return span
|
||||
|
||||
|
||||
def end_span(
|
||||
span: Span | None = None,
|
||||
status: SpanStatus = SpanStatus.OK,
|
||||
metadata_update: dict[str, Any] | None = None,
|
||||
details_update: dict[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
End a span and restore parent as current.
|
||||
|
||||
Args:
|
||||
span: Span to end (defaults to current span)
|
||||
status: Completion status
|
||||
metadata_update: Additional metadata to merge
|
||||
details_update: Additional details to merge
|
||||
error: Error message if failed
|
||||
"""
|
||||
if span is None:
|
||||
span = get_current_span()
|
||||
if not span:
|
||||
return
|
||||
|
||||
span.end_time = datetime.now(timezone.utc)
|
||||
span.status = status
|
||||
if error:
|
||||
span.error = error
|
||||
span.status = SpanStatus.ERROR
|
||||
if metadata_update:
|
||||
span.metadata.update(metadata_update)
|
||||
if details_update:
|
||||
span.details.update(details_update)
|
||||
|
||||
# Restore parent span as current
|
||||
trace = get_current_trace()
|
||||
if trace and span.parent_id:
|
||||
parent = next((s for s in trace.spans if s.span_id == span.parent_id), None)
|
||||
_current_span.set(parent)
|
||||
else:
|
||||
_current_span.set(None)
|
||||
|
||||
logger.debug(
|
||||
"span_ended",
|
||||
span_id=span.span_id,
|
||||
duration_ms=span.duration_ms,
|
||||
status=status.value,
|
||||
)
|
||||
|
||||
|
||||
def end_trace(
|
||||
response: dict[str, Any] | None = None,
|
||||
status: str = "completed",
|
||||
) -> str | None:
|
||||
"""
|
||||
End the current trace and write to file.
|
||||
|
||||
Args:
|
||||
response: Response data to include
|
||||
status: Final trace status ("completed" or "error")
|
||||
|
||||
Returns:
|
||||
Path to trace file if written, None otherwise
|
||||
"""
|
||||
trace = get_current_trace()
|
||||
if not trace:
|
||||
return None
|
||||
|
||||
trace.response = response
|
||||
trace.status = status
|
||||
|
||||
# Write trace to file
|
||||
trace_path = _write_trace(trace)
|
||||
|
||||
# Clear context
|
||||
_current_trace.set(None)
|
||||
_current_span.set(None)
|
||||
|
||||
logger.info(
|
||||
"trace_completed",
|
||||
trace_id=trace.trace_id,
|
||||
total_duration_ms=round(trace.total_duration_ms, 2) if trace.total_duration_ms else None,
|
||||
span_count=len(trace.spans),
|
||||
path=str(trace_path) if trace_path else None,
|
||||
)
|
||||
|
||||
return str(trace_path) if trace_path else None
|
||||
|
||||
|
||||
def _write_trace(trace: Trace) -> Path | None:
|
||||
"""Write trace to JSON file."""
|
||||
try:
|
||||
# Ensure traces directory exists
|
||||
traces_dir = Path("logs/traces")
|
||||
traces_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write trace file
|
||||
trace_path = traces_dir / f"{trace.trace_id}.json"
|
||||
with open(trace_path, "w") as f:
|
||||
json.dump(trace.to_dict(), f, indent=2, default=str)
|
||||
|
||||
return trace_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error("trace_write_failed", error=str(e), trace_id=trace.trace_id)
|
||||
return None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def trace_span(
|
||||
name: str,
|
||||
span_type: SpanType,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
):
|
||||
"""
|
||||
Async context manager for tracing a span.
|
||||
|
||||
Automatically handles start/end timing and error capture.
|
||||
|
||||
Usage:
|
||||
async with trace_span("steward_analysis", SpanType.STEWARD) as span:
|
||||
result = await analyze_request(...)
|
||||
if span:
|
||||
span.metadata["result_count"] = len(result)
|
||||
|
||||
Args:
|
||||
name: Span name
|
||||
span_type: Type of operation
|
||||
metadata: Initial metadata
|
||||
details: Initial details (expandable in viewer)
|
||||
|
||||
Yields:
|
||||
Span object or None if tracing disabled
|
||||
"""
|
||||
span = start_span(name, span_type, metadata, details)
|
||||
try:
|
||||
yield span
|
||||
except Exception as e:
|
||||
end_span(span, SpanStatus.ERROR, error=str(e))
|
||||
raise
|
||||
else:
|
||||
end_span(span, SpanStatus.OK)
|
||||
|
||||
|
||||
def add_tool_spans_from_messages(messages: list[Any], parent_span: Span | None = None) -> None:
|
||||
"""
|
||||
Extract tool calls from PydanticAI result messages and add as child spans.
|
||||
|
||||
Call this after an agent.run() to capture tool-level timing retroactively.
|
||||
Note: Since we don't have actual timing, we estimate based on sequence.
|
||||
|
||||
Args:
|
||||
messages: List from result.new_messages()
|
||||
parent_span: Parent span to attach tool spans to
|
||||
"""
|
||||
trace = get_current_trace()
|
||||
if not trace or not parent_span:
|
||||
return
|
||||
|
||||
# Import PydanticAI message types
|
||||
try:
|
||||
from pydantic_ai.messages import ModelRequest, ModelResponse, ToolCallPart, ToolReturnPart
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
# Track tool calls and their returns
|
||||
tool_calls: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for msg in messages:
|
||||
if isinstance(msg, ModelResponse):
|
||||
for part in msg.parts:
|
||||
if isinstance(part, ToolCallPart):
|
||||
tool_calls[part.tool_call_id] = {
|
||||
"name": part.tool_name,
|
||||
"args": part.args if hasattr(part, 'args') else {},
|
||||
}
|
||||
elif isinstance(msg, ModelRequest):
|
||||
for part in msg.parts:
|
||||
if isinstance(part, ToolReturnPart):
|
||||
if part.tool_call_id in tool_calls:
|
||||
tool_info = tool_calls[part.tool_call_id]
|
||||
# Create a span for this tool call
|
||||
span = Span(
|
||||
span_id=_generate_id("span_"),
|
||||
name=tool_info["name"],
|
||||
type=SpanType.TOOL,
|
||||
start_time=parent_span.start_time, # Approximate
|
||||
end_time=parent_span.end_time or datetime.now(timezone.utc),
|
||||
parent_id=parent_span.span_id,
|
||||
status=SpanStatus.OK,
|
||||
metadata={
|
||||
"tool_name": tool_info["name"],
|
||||
"args_preview": str(tool_info.get("args", {}))[:100],
|
||||
},
|
||||
details={
|
||||
"args": tool_info.get("args", {}),
|
||||
"result": part.content[:2000] if isinstance(part.content, str) else str(part.content)[:2000],
|
||||
},
|
||||
)
|
||||
parent_span.children.append(span.span_id)
|
||||
trace.spans.append(span)
|
||||
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
Trace viewer router.
|
||||
|
||||
Serves the trace viewer UI and trace files when tracing is enabled.
|
||||
Only available when DEBUG=true.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/traces", tags=["traces"])
|
||||
|
||||
TRACES_DIR = Path("logs/traces")
|
||||
VIEWER_PATH = TRACES_DIR / "viewer.html"
|
||||
|
||||
|
||||
def tracing_enabled() -> bool:
|
||||
"""Check if tracing is enabled."""
|
||||
return config.DEBUG
|
||||
|
||||
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def get_trace_viewer():
|
||||
"""
|
||||
Serve the trace viewer UI.
|
||||
|
||||
Returns the standalone HTML viewer for browsing traces.
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
raise HTTPException(status_code=404, detail="Tracing not enabled")
|
||||
|
||||
if not VIEWER_PATH.exists():
|
||||
raise HTTPException(status_code=404, detail="Viewer not found")
|
||||
|
||||
return HTMLResponse(content=VIEWER_PATH.read_text())
|
||||
|
||||
|
||||
@router.get("/list")
|
||||
async def list_traces(
|
||||
limit: int = 50,
|
||||
since_minutes: int | None = None,
|
||||
status: str | None = None,
|
||||
search: str | None = None,
|
||||
):
|
||||
"""
|
||||
List available trace files.
|
||||
|
||||
Returns most recent traces first, with basic metadata.
|
||||
|
||||
Args:
|
||||
limit: Maximum number of traces to return (default 50)
|
||||
since_minutes: Only return traces from the last N minutes
|
||||
status: Filter by status (completed, error, streaming)
|
||||
search: Search in request preview text
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
raise HTTPException(status_code=404, detail="Tracing not enabled")
|
||||
|
||||
if not TRACES_DIR.exists():
|
||||
return {"traces": [], "total": 0}
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
# Calculate cutoff time if filtering by time
|
||||
cutoff_time = None
|
||||
if since_minutes:
|
||||
cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
|
||||
|
||||
# Get all trace files, sorted by modification time (newest first)
|
||||
trace_files = sorted(
|
||||
TRACES_DIR.glob("trace_*.json"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
traces = []
|
||||
for path in trace_files:
|
||||
if len(traces) >= limit:
|
||||
break
|
||||
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Parse timestamp for filtering
|
||||
trace_timestamp = data.get("timestamp")
|
||||
if cutoff_time and trace_timestamp:
|
||||
try:
|
||||
ts = datetime.fromisoformat(trace_timestamp.replace('Z', '+00:00'))
|
||||
if ts < cutoff_time:
|
||||
continue
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Filter by status
|
||||
trace_status = data.get("status", "")
|
||||
if status and trace_status != status:
|
||||
continue
|
||||
|
||||
# Filter by search text
|
||||
request_preview = data.get("request", {}).get("input_preview", "")
|
||||
if search and search.lower() not in request_preview.lower():
|
||||
continue
|
||||
|
||||
traces.append({
|
||||
"trace_id": data.get("trace_id"),
|
||||
"timestamp": trace_timestamp,
|
||||
"user": data.get("user"),
|
||||
"status": trace_status,
|
||||
"total_duration_ms": data.get("total_duration_ms"),
|
||||
"span_count": len(data.get("spans", [])),
|
||||
"request_preview": request_preview[:100],
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning("trace_list_parse_error", path=str(path), error=str(e))
|
||||
|
||||
return {"traces": traces, "total": len(traces)}
|
||||
|
||||
|
||||
@router.get("/{trace_id}")
|
||||
async def get_trace(trace_id: str):
|
||||
"""
|
||||
Get a specific trace by ID.
|
||||
|
||||
Returns the full trace JSON.
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
raise HTTPException(status_code=404, detail="Tracing not enabled")
|
||||
|
||||
# Sanitize trace_id to prevent path traversal
|
||||
if not trace_id.startswith("trace_") or "/" in trace_id or "\\" in trace_id:
|
||||
raise HTTPException(status_code=400, detail="Invalid trace ID")
|
||||
|
||||
trace_path = TRACES_DIR / f"{trace_id}.json"
|
||||
|
||||
if not trace_path.exists():
|
||||
raise HTTPException(status_code=404, detail="Trace not found")
|
||||
|
||||
try:
|
||||
import json
|
||||
with open(trace_path) as f:
|
||||
data = json.load(f)
|
||||
return JSONResponse(content=data)
|
||||
except Exception as e:
|
||||
logger.error("trace_read_error", trace_id=trace_id, error=str(e))
|
||||
raise HTTPException(status_code=500, detail="Failed to read trace")
|
||||
+12
-4
@@ -23,6 +23,7 @@ from src.core.exceptions import AppException
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.router import router as core_router
|
||||
from src.core.startup import initialize_application
|
||||
from src.core.tracing_router import router as tracing_router
|
||||
from src.models.router import router as models_router
|
||||
from src.responses.router import router as responses_router
|
||||
|
||||
@@ -43,14 +44,16 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app_name=config.APP_NAME,
|
||||
version=config.APP_VERSION,
|
||||
environment=config.ENVIRONMENT.value,
|
||||
prefer_cloud=config.PREFER_CLOUD_BACKEND,
|
||||
anthropic_model=config.ANTHROPIC_MODEL,
|
||||
ollama_host=str(config.OLLAMA_HOST),
|
||||
ollama_model=config.OLLAMA_DEFAULT_MODEL,
|
||||
redis_url=config.redis_url,
|
||||
redis_url=config.redis_memory_url,
|
||||
log_format=config.log_format,
|
||||
)
|
||||
|
||||
# Initialize application (register household members, etc.)
|
||||
initialize_application()
|
||||
# Initialize application (check Claude health, register household members, etc.)
|
||||
await initialize_application()
|
||||
|
||||
yield
|
||||
|
||||
@@ -90,7 +93,12 @@ def create_application() -> FastAPI:
|
||||
application.include_router(chat_router, prefix=config.API_PREFIX)
|
||||
application.include_router(models_router, prefix=config.API_PREFIX)
|
||||
application.include_router(responses_router, prefix=config.API_PREFIX) # Responses API
|
||||
|
||||
|
||||
# Conditionally include tracing router (only in debug mode)
|
||||
if config.DEBUG:
|
||||
application.include_router(tracing_router)
|
||||
logger.info("tracing_router_enabled")
|
||||
|
||||
return application
|
||||
|
||||
|
||||
|
||||
+5
-73
@@ -10,7 +10,6 @@ from sse_starlette.sse import EventSourceResponse
|
||||
from src.responses import service
|
||||
from src.responses.schemas import ResponseRequest, Response
|
||||
from src.core.exceptions import ModelNotFoundError, AppException
|
||||
from src.core.context import current_user, current_conversation, get_default_user
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -37,77 +36,16 @@ async def create_response(
|
||||
|
||||
Returns:
|
||||
Response object or SSE stream
|
||||
|
||||
Example non-streaming request:
|
||||
POST /v1/responses
|
||||
{
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Hello"}],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
"stream": false
|
||||
}
|
||||
|
||||
Example streaming request:
|
||||
POST /v1/responses
|
||||
{
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Hello"}],
|
||||
"stream": true
|
||||
}
|
||||
|
||||
Response format (non-streaming):
|
||||
{
|
||||
"id": "resp_...",
|
||||
"object": "response",
|
||||
"created_at": 1733529600,
|
||||
"model": "lorem-tester",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_...",
|
||||
"summary": ["Analyzing...", "Considering..."]
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_...",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Lorem ipsum..."}]
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 50,
|
||||
"reasoning_tokens": 20,
|
||||
"total_tokens": 80
|
||||
}
|
||||
}
|
||||
|
||||
Streaming format (SSE):
|
||||
event: response.reasoning_summary_text.delta
|
||||
data: {"delta": "Analyzing..."}
|
||||
|
||||
event: response.output_text.delta
|
||||
data: {"delta": "Lorem"}
|
||||
|
||||
event: response.done
|
||||
data: {"response": {...}}
|
||||
"""
|
||||
# Set request context (propagates through all async calls)
|
||||
effective_user = request.user or get_default_user()
|
||||
user_token = current_user.set(effective_user)
|
||||
conv_id = request.metadata.get("conversation_id") if request.metadata else None
|
||||
conv_token = current_conversation.set(conv_id)
|
||||
|
||||
logger.info(
|
||||
"response_request_received",
|
||||
model=request.model,
|
||||
user=effective_user,
|
||||
conversation_id=conv_id,
|
||||
user=request.user,
|
||||
streaming=request.stream,
|
||||
)
|
||||
|
||||
try:
|
||||
# Check if this is a Tatlock request - use Steward preprocessing (Phase 2)
|
||||
# Check if this is a Tatlock request - use Steward preprocessing
|
||||
model_id = request.model
|
||||
if "." in model_id:
|
||||
model_id = model_id.split(".", 1)[1]
|
||||
@@ -116,21 +54,20 @@ async def create_response(
|
||||
|
||||
if request.stream:
|
||||
logger.info("Streaming response requested")
|
||||
|
||||
if use_steward:
|
||||
logger.info("Streaming with Steward preprocessing for Tatlock request")
|
||||
# Use Steward + Tatlock streaming (Milestone 3.5)
|
||||
from src.responses.streaming import StreamingCoordinator
|
||||
coordinator = StreamingCoordinator()
|
||||
return EventSourceResponse(
|
||||
coordinator.stream_response_with_steward(request)
|
||||
)
|
||||
else:
|
||||
# Regular streaming for non-Tatlock models
|
||||
return EventSourceResponse(
|
||||
service.create_response_stream(request)
|
||||
)
|
||||
|
||||
# Use appropriate service method
|
||||
# Non-streaming response
|
||||
if use_steward:
|
||||
logger.info("Using Steward preprocessing for Tatlock request")
|
||||
return await service.create_response_with_steward(request)
|
||||
@@ -148,8 +85,3 @@ async def create_response(
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
finally:
|
||||
# Reset context (important for connection reuse)
|
||||
current_user.reset(user_token)
|
||||
current_conversation.reset(conv_token)
|
||||
|
||||
+251
-140
@@ -26,6 +26,8 @@ from src.responses.context import ContextWindow
|
||||
from src.core.preprocessing import preprocess_request
|
||||
from src.core.tool_tracking import ToolCallTracker
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import start_trace, end_trace, start_span, SpanType
|
||||
from src.core.context import current_user, current_conversation, get_default_user
|
||||
from src.agents.steward.schemas import StewardRecommendation
|
||||
|
||||
import re
|
||||
@@ -34,6 +36,29 @@ import asyncio
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def _extract_user_input(input_data) -> str:
|
||||
"""Extract user input text from request input for tracing."""
|
||||
if isinstance(input_data, str):
|
||||
return input_data
|
||||
elif isinstance(input_data, list) and input_data:
|
||||
last_msg = input_data[-1]
|
||||
if isinstance(last_msg, dict):
|
||||
return last_msg.get("content", str(last_msg))
|
||||
return str(last_msg)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_response_preview(response: Response) -> str:
|
||||
"""Extract response preview text for tracing."""
|
||||
if response.output:
|
||||
for item in response.output:
|
||||
if hasattr(item, 'content'):
|
||||
for content in item.content:
|
||||
if hasattr(content, 'text'):
|
||||
return content.text[:200]
|
||||
return ""
|
||||
|
||||
|
||||
async def _execute_single_delegation(
|
||||
agent_name: str,
|
||||
task: str,
|
||||
@@ -405,45 +430,88 @@ async def create_response(request: ResponseRequest) -> Response:
|
||||
# Get or generate conversation ID
|
||||
conversation_id = await _conversation_history.get_conversation_id(request)
|
||||
|
||||
# Strip pipeline prefix if present (e.g., "pipeline.model" -> "model")
|
||||
model_id = request.model
|
||||
if "." in model_id:
|
||||
model_id = model_id.split(".", 1)[1]
|
||||
# Set context for tracing
|
||||
effective_user = request.user or get_default_user()
|
||||
current_user.set(effective_user)
|
||||
current_conversation.set(conversation_id)
|
||||
|
||||
# Get agent for model
|
||||
agent = ModelRegistry.get_agent(model_id)
|
||||
# Extract user input for tracing
|
||||
user_input = _extract_user_input(request.input)
|
||||
|
||||
# Collect all output items from agent
|
||||
output_items = []
|
||||
async for item in agent.generate_response(
|
||||
messages=request.input,
|
||||
reasoning=request.reasoning,
|
||||
tools=request.tools,
|
||||
temperature=request.temperature,
|
||||
max_tokens=request.max_output_tokens,
|
||||
stop=request.stop,
|
||||
):
|
||||
output_items.append(item)
|
||||
|
||||
# Convert agent OutputItems to schema OutputItems
|
||||
converted_items = _convert_output_items(output_items)
|
||||
|
||||
# Calculate token usage
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=converted_items,
|
||||
usage=usage
|
||||
# Start trace
|
||||
trace = start_trace(
|
||||
conversation_id=conversation_id,
|
||||
user=effective_user,
|
||||
request={
|
||||
"model": request.model,
|
||||
"input_preview": user_input[:200] if user_input else "",
|
||||
"full_input": request.input,
|
||||
"streaming": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Track conversation history (for analytics and future vector memory)
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
# Start service span
|
||||
service_span = start_span(
|
||||
"create_response",
|
||||
SpanType.ROUTER,
|
||||
metadata={"model": request.model, "user": effective_user},
|
||||
)
|
||||
|
||||
return response
|
||||
try:
|
||||
# Strip pipeline prefix if present (e.g., "pipeline.model" -> "model")
|
||||
model_id = request.model
|
||||
if "." in model_id:
|
||||
model_id = model_id.split(".", 1)[1]
|
||||
|
||||
# Get agent for model
|
||||
agent = ModelRegistry.get_agent(model_id)
|
||||
|
||||
# Collect all output items from agent
|
||||
output_items = []
|
||||
async for item in agent.generate_response(
|
||||
messages=request.input,
|
||||
reasoning=request.reasoning,
|
||||
tools=request.tools,
|
||||
temperature=request.temperature,
|
||||
max_tokens=request.max_output_tokens,
|
||||
stop=request.stop,
|
||||
):
|
||||
output_items.append(item)
|
||||
|
||||
# Convert agent OutputItems to schema OutputItems
|
||||
converted_items = _convert_output_items(output_items)
|
||||
|
||||
# Calculate token usage
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=converted_items,
|
||||
usage=usage
|
||||
)
|
||||
|
||||
# Track conversation history (for analytics and future vector memory)
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
|
||||
# End trace with response info
|
||||
response_preview = _extract_response_preview(response)
|
||||
end_trace(
|
||||
response={
|
||||
"output_preview": response_preview,
|
||||
"output_count": len(response.output) if response.output else 0,
|
||||
"status": response.status,
|
||||
},
|
||||
status="completed",
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
end_trace(status="error")
|
||||
raise
|
||||
|
||||
|
||||
async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
@@ -454,7 +522,7 @@ async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
1. Steward analyzes the request and recommends capabilities
|
||||
2. Phase 1: Tatlock orchestrates tool calls and expert delegations
|
||||
3. Phase 2: Tatlock synthesizes butler-toned response from results
|
||||
4. Tool usage is tracked for benchmarking
|
||||
4. Tool usage is tracked for analysis
|
||||
|
||||
Args:
|
||||
request: Response request
|
||||
@@ -473,133 +541,176 @@ async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
# Get or generate conversation ID
|
||||
conversation_id = await _conversation_history.get_conversation_id(request)
|
||||
|
||||
# Extract user message and conversation history
|
||||
user_message = ""
|
||||
for msg in reversed(request.input):
|
||||
if msg.get("role") == "user":
|
||||
user_message = msg.get("content", "")
|
||||
break
|
||||
# Set context for tracing
|
||||
effective_user = request.user or get_default_user()
|
||||
current_user.set(effective_user)
|
||||
current_conversation.set(conversation_id)
|
||||
|
||||
# Conversation history is all messages except the current one
|
||||
conversation_history = request.input[:-1] if len(request.input) > 1 else []
|
||||
# Extract user input for tracing
|
||||
user_input = _extract_user_input(request.input)
|
||||
|
||||
logger.info(
|
||||
"creating_response_with_steward",
|
||||
user_message_preview=user_message[:100],
|
||||
history_length=len(conversation_history),
|
||||
# Start trace
|
||||
trace = start_trace(
|
||||
conversation_id=conversation_id,
|
||||
user=effective_user,
|
||||
request={
|
||||
"model": request.model,
|
||||
"input_preview": user_input[:200] if user_input else "",
|
||||
"full_input": request.input,
|
||||
"streaming": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Steward preprocessing
|
||||
enriched = await preprocess_request(
|
||||
user_message,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
# Start service span
|
||||
service_span = start_span(
|
||||
"create_response_with_steward",
|
||||
SpanType.ROUTER,
|
||||
metadata={"model": request.model, "user": effective_user},
|
||||
)
|
||||
|
||||
# Initialize tool tracker
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
try:
|
||||
# Extract user message and conversation history
|
||||
user_message = ""
|
||||
for msg in reversed(request.input):
|
||||
if msg.get("role") == "user":
|
||||
user_message = msg.get("content", "")
|
||||
break
|
||||
|
||||
# Check if direct delegation is recommended
|
||||
# If Steward recommends ONLY delegation agents (biographer/librarian/housekeeper),
|
||||
# we still use two-phase but delegate directly in Phase 1
|
||||
delegation_agents = {"biographer", "librarian", "housekeeper"}
|
||||
delegation_only = all(
|
||||
cap in delegation_agents
|
||||
for cap in enriched.recommendation.recommended_capabilities
|
||||
) and enriched.recommendation.recommended_capabilities
|
||||
# Conversation history is all messages except the current one
|
||||
conversation_history = request.input[:-1] if len(request.input) > 1 else []
|
||||
|
||||
from src.agents.tatlock import TatlockAgent
|
||||
tatlock = TatlockAgent()
|
||||
|
||||
# Use enriched query (with location/timezone context) if available
|
||||
effective_query = enriched.recommendation.enriched_query or user_message
|
||||
|
||||
if delegation_only:
|
||||
# Direct delegation path - collect results then synthesize
|
||||
orchestration_results = await _direct_delegation_with_results(
|
||||
effective_query, enriched.recommendation, tracker, conversation_id
|
||||
)
|
||||
else:
|
||||
# Phase 1: Orchestrate tool calls
|
||||
orchestration_results = await tatlock.orchestrate_tool_calls(
|
||||
user_message=effective_query,
|
||||
steward_note=enriched.steward_note,
|
||||
scoped_tools=enriched.scoped_tools,
|
||||
message_history=conversation_history,
|
||||
tool_tracker=tracker,
|
||||
logger.info(
|
||||
"creating_response_with_steward",
|
||||
user_message_preview=user_message[:100],
|
||||
history_length=len(conversation_history),
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Handle text-based delegation fallback if present
|
||||
if "[DELEGATE:" in orchestration_results.get("raw_output", ""):
|
||||
text_delegation_results = await _handle_text_delegation(
|
||||
orchestration_results["raw_output"], tracker, conversation_id
|
||||
# Steward preprocessing
|
||||
enriched = await preprocess_request(
|
||||
user_message,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Initialize tool tracker
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Check if direct delegation is recommended
|
||||
# If Steward recommends ONLY delegation agents (biographer/librarian/housekeeper),
|
||||
# we still use two-phase but delegate directly in Phase 1
|
||||
delegation_agents = {"biographer", "librarian", "housekeeper"}
|
||||
delegation_only = all(
|
||||
cap in delegation_agents
|
||||
for cap in enriched.recommendation.recommended_capabilities
|
||||
) and enriched.recommendation.recommended_capabilities
|
||||
|
||||
from src.agents.tatlock import TatlockAgent
|
||||
tatlock = TatlockAgent()
|
||||
|
||||
# Use enriched query (with location/timezone context) if available
|
||||
effective_query = enriched.recommendation.enriched_query or user_message
|
||||
|
||||
if delegation_only:
|
||||
# Direct delegation path - collect results then synthesize
|
||||
orchestration_results = await _direct_delegation_with_results(
|
||||
effective_query, enriched.recommendation, tracker, conversation_id
|
||||
)
|
||||
else:
|
||||
# Phase 1: Orchestrate tool calls
|
||||
orchestration_results = await tatlock.orchestrate_tool_calls(
|
||||
user_message=effective_query,
|
||||
steward_note=enriched.steward_note,
|
||||
scoped_tools=enriched.scoped_tools,
|
||||
message_history=conversation_history,
|
||||
tool_tracker=tracker,
|
||||
)
|
||||
# Add text delegation results to expert_results
|
||||
if text_delegation_results != orchestration_results["raw_output"]:
|
||||
orchestration_results["expert_results"]["text_delegation"] = text_delegation_results
|
||||
|
||||
# Phase 2: Synthesize butler-toned response from all results
|
||||
tatlock_response = await tatlock.synthesize_from_results(
|
||||
user_message=user_message,
|
||||
orchestration_results=orchestration_results,
|
||||
message_history=conversation_history,
|
||||
)
|
||||
# Handle text-based delegation fallback if present
|
||||
if "[DELEGATE:" in orchestration_results.get("raw_output", ""):
|
||||
text_delegation_results = await _handle_text_delegation(
|
||||
orchestration_results["raw_output"], tracker, conversation_id
|
||||
)
|
||||
# Add text delegation results to expert_results
|
||||
if text_delegation_results != orchestration_results["raw_output"]:
|
||||
orchestration_results["expert_results"]["text_delegation"] = text_delegation_results
|
||||
|
||||
# Finalize tool tracking
|
||||
await tracker.finalize()
|
||||
# Phase 2: Synthesize butler-toned response from all results
|
||||
tatlock_response = await tatlock.synthesize_from_results(
|
||||
user_message=user_message,
|
||||
orchestration_results=orchestration_results,
|
||||
message_history=conversation_history,
|
||||
)
|
||||
|
||||
# Build response output items
|
||||
output_items = []
|
||||
# Finalize tool tracking
|
||||
await tracker.finalize()
|
||||
|
||||
# Add Steward reasoning as a reasoning output item
|
||||
output_items.append(ReasoningOutputItem(
|
||||
id=f"reasoning_{generate_id()}",
|
||||
summary=[
|
||||
"🎩 Steward's Analysis:",
|
||||
enriched.steward_reasoning,
|
||||
],
|
||||
status="completed"
|
||||
))
|
||||
# Build response output items
|
||||
output_items = []
|
||||
|
||||
# Add Tatlock's message
|
||||
output_items.append(MessageOutputItem(
|
||||
id=f"msg_{generate_id()}",
|
||||
role="assistant",
|
||||
content=[OutputTextContent(
|
||||
type="output_text",
|
||||
text=tatlock_response,
|
||||
annotations=[]
|
||||
)],
|
||||
status="completed"
|
||||
))
|
||||
# Add Steward reasoning as a reasoning output item
|
||||
output_items.append(ReasoningOutputItem(
|
||||
id=f"reasoning_{generate_id()}",
|
||||
summary=[
|
||||
"🎩 Steward's Analysis:",
|
||||
enriched.steward_reasoning,
|
||||
],
|
||||
status="completed"
|
||||
))
|
||||
|
||||
# Calculate usage (approximate)
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
# Add Tatlock's message
|
||||
output_items.append(MessageOutputItem(
|
||||
id=f"msg_{generate_id()}",
|
||||
role="assistant",
|
||||
content=[OutputTextContent(
|
||||
type="output_text",
|
||||
text=tatlock_response,
|
||||
annotations=[]
|
||||
)],
|
||||
status="completed"
|
||||
))
|
||||
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=output_items,
|
||||
usage=usage
|
||||
)
|
||||
# Calculate usage (approximate)
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
|
||||
# Track conversation history
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=output_items,
|
||||
usage=usage
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"response_with_steward_complete",
|
||||
response_id=response.id,
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
tool_summary=tracker.get_summary(),
|
||||
)
|
||||
# Track conversation history
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
|
||||
return response
|
||||
logger.info(
|
||||
"response_with_steward_complete",
|
||||
response_id=response.id,
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
tool_summary=tracker.get_summary(),
|
||||
)
|
||||
|
||||
# End trace with response info
|
||||
response_preview = _extract_response_preview(response)
|
||||
end_trace(
|
||||
response={
|
||||
"output_preview": response_preview,
|
||||
"output_count": len(response.output) if response.output else 0,
|
||||
"status": response.status,
|
||||
},
|
||||
status="completed",
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
end_trace(status="error")
|
||||
raise
|
||||
|
||||
|
||||
async def create_response_stream(
|
||||
|
||||
@@ -1,352 +0,0 @@
|
||||
"""
|
||||
Tests for benchmark storage.
|
||||
|
||||
Tests performance tracking, Redis storage, and analytics features.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.benchmarks import (
|
||||
BenchmarkStore,
|
||||
PerformanceBenchmark,
|
||||
get_benchmark_store,
|
||||
)
|
||||
|
||||
|
||||
class TestPerformanceBenchmark:
|
||||
"""Test PerformanceBenchmark model."""
|
||||
|
||||
def test_benchmark_creation(self):
|
||||
"""Test creating a performance benchmark."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="steward_analysis",
|
||||
duration_seconds=1.23,
|
||||
success=True,
|
||||
recommendation_count=3,
|
||||
)
|
||||
|
||||
assert benchmark.operation == "steward_analysis"
|
||||
assert benchmark.duration_seconds == 1.23
|
||||
assert benchmark.success is True
|
||||
assert benchmark.recommendation_count == 3
|
||||
assert isinstance(benchmark.timestamp, datetime)
|
||||
|
||||
def test_benchmark_with_tool_fields(self):
|
||||
"""Test benchmark with tool-specific fields."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="tool_call",
|
||||
duration_seconds=0.5,
|
||||
success=True,
|
||||
tool_name="calculate",
|
||||
was_recommended=True,
|
||||
was_actually_used=True,
|
||||
)
|
||||
|
||||
assert benchmark.tool_name == "calculate"
|
||||
assert benchmark.was_recommended is True
|
||||
assert benchmark.was_actually_used is True
|
||||
|
||||
def test_benchmark_to_redis_dict(self):
|
||||
"""Test conversion to Redis dict."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
metadata={"key": "value"},
|
||||
)
|
||||
|
||||
redis_dict = benchmark.to_redis_dict()
|
||||
assert redis_dict["operation"] == "test_op"
|
||||
assert redis_dict["duration_seconds"] == 1.0
|
||||
assert redis_dict["success"] == "True" # Booleans stored as strings in Redis
|
||||
assert isinstance(redis_dict["timestamp"], str)
|
||||
assert isinstance(redis_dict["metadata"], str)
|
||||
|
||||
def test_benchmark_from_redis_dict(self):
|
||||
"""Test reconstruction from Redis dict."""
|
||||
now = datetime.now(timezone.utc)
|
||||
redis_dict = {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": 1.5,
|
||||
"success": "True", # Booleans stored as strings in Redis
|
||||
"metadata": json.dumps({"test": "data"}),
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
benchmark = PerformanceBenchmark.from_redis_dict(redis_dict)
|
||||
assert benchmark.operation == "test_op"
|
||||
assert benchmark.duration_seconds == 1.5
|
||||
assert benchmark.success is True # Converted back to bool
|
||||
assert benchmark.metadata == {"test": "data"}
|
||||
|
||||
|
||||
class TestBenchmarkStore:
|
||||
"""Test BenchmarkStore functionality."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_redis(self):
|
||||
"""Create mock Redis client."""
|
||||
mock = AsyncMock()
|
||||
mock.hset = AsyncMock()
|
||||
mock.expire = AsyncMock()
|
||||
mock.zadd = AsyncMock()
|
||||
mock.zrevrangebyscore = AsyncMock(return_value=[])
|
||||
mock.hgetall = AsyncMock(return_value={})
|
||||
mock.aclose = AsyncMock()
|
||||
return mock
|
||||
|
||||
@pytest.fixture
|
||||
def store(self, mock_redis):
|
||||
"""Create benchmark store with mock Redis."""
|
||||
return BenchmarkStore(redis_client=mock_redis)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark(self, store, mock_redis):
|
||||
"""Test recording a benchmark."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
await store.record(benchmark)
|
||||
|
||||
# Verify Redis calls
|
||||
mock_redis.hset.assert_called_once()
|
||||
mock_redis.expire.assert_called()
|
||||
mock_redis.zadd.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark_disabled(self, mock_redis):
|
||||
"""Test recording when benchmarks are disabled."""
|
||||
with patch("src.core.benchmarks.config.ENABLE_BENCHMARKS", False):
|
||||
store = BenchmarkStore(redis_client=mock_redis)
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
await store.record(benchmark)
|
||||
|
||||
# Should not call Redis
|
||||
mock_redis.hset.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark_handles_errors(self, store, mock_redis):
|
||||
"""Test recording handles Redis errors gracefully."""
|
||||
mock_redis.hset.side_effect = Exception("Redis error")
|
||||
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
# Should not raise exception
|
||||
await store.record(benchmark)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_benchmarks(self, store, mock_redis):
|
||||
"""Test querying benchmarks."""
|
||||
# Setup mock data
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_key = f"benchmark:test_op:{int(now.timestamp() * 1000)}"
|
||||
mock_redis.zrevrangebyscore.return_value = [mock_key]
|
||||
|
||||
# Mock hgetall to return proper data (booleans as strings, like Redis)
|
||||
mock_redis.hgetall.return_value = {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": 1.5, # Numeric, not string
|
||||
"success": "True", # Booleans stored as strings in Redis
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
results = await store.query("test_op", limit=10)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].operation == "test_op"
|
||||
mock_redis.zrevrangebyscore.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_with_time_range(self, store, mock_redis):
|
||||
"""Test querying with time range."""
|
||||
now = datetime.now(timezone.utc)
|
||||
start_time = now - timedelta(hours=1)
|
||||
end_time = now
|
||||
|
||||
await store.query("test_op", start_time=start_time, end_time=end_time)
|
||||
|
||||
# Verify time range was converted to timestamps
|
||||
call_args = mock_redis.zrevrangebyscore.call_args
|
||||
assert call_args is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_disabled_benchmarks(self, mock_redis):
|
||||
"""Test querying when benchmarks are disabled."""
|
||||
with patch("src.core.benchmarks.config.ENABLE_BENCHMARKS", False):
|
||||
store = BenchmarkStore(redis_client=mock_redis)
|
||||
results = await store.query("test_op")
|
||||
assert results == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_handles_errors(self, store, mock_redis):
|
||||
"""Test query handles errors gracefully."""
|
||||
mock_redis.zrevrangebyscore.side_effect = Exception("Redis error")
|
||||
|
||||
results = await store.query("test_op")
|
||||
assert results == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_statistics(self, store, mock_redis):
|
||||
"""Test getting statistics."""
|
||||
# Setup mock data with multiple benchmarks
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_keys = [
|
||||
f"benchmark:test_op:{int((now - timedelta(seconds=i)).timestamp() * 1000)}"
|
||||
for i in range(3)
|
||||
]
|
||||
mock_redis.zrevrangebyscore.return_value = mock_keys
|
||||
|
||||
# Return different durations and success values
|
||||
benchmarks_data = [
|
||||
{"duration_seconds": "1.0", "success": "True"},
|
||||
{"duration_seconds": "2.0", "success": "True"},
|
||||
{"duration_seconds": "3.0", "success": "False"},
|
||||
]
|
||||
|
||||
async def mock_hgetall(key):
|
||||
idx = mock_keys.index(key)
|
||||
data = benchmarks_data[idx]
|
||||
return {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": float(data["duration_seconds"]),
|
||||
"success": data["success"], # Pass string through, from_redis_dict converts
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
mock_redis.hgetall.side_effect = mock_hgetall
|
||||
|
||||
stats = await store.get_statistics("test_op")
|
||||
|
||||
assert stats["count"] == 3
|
||||
assert stats["avg_duration"] == 2.0 # (1 + 2 + 3) / 3
|
||||
assert stats["min_duration"] == 1.0
|
||||
assert stats["max_duration"] == 3.0
|
||||
assert stats["success_rate"] == pytest.approx(66.67, rel=0.01)
|
||||
assert stats["total_successes"] == 2
|
||||
assert stats["total_failures"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_statistics_empty(self, store, mock_redis):
|
||||
"""Test statistics with no data."""
|
||||
mock_redis.zrevrangebyscore.return_value = []
|
||||
|
||||
stats = await store.get_statistics("test_op")
|
||||
|
||||
assert stats["count"] == 0
|
||||
assert stats["avg_duration"] == 0.0
|
||||
assert stats["success_rate"] == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_tool_accuracy(self, store, mock_redis):
|
||||
"""Test tool accuracy calculation."""
|
||||
# Setup mock data
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_keys = [
|
||||
f"benchmark:tool_call:{int((now - timedelta(seconds=i)).timestamp() * 1000)}"
|
||||
for i in range(4)
|
||||
]
|
||||
mock_redis.zrevrangebyscore.return_value = mock_keys
|
||||
|
||||
# Different combinations of recommended/used
|
||||
tool_data = [
|
||||
{"was_recommended": "True", "was_actually_used": "True"}, # Good
|
||||
{"was_recommended": "True", "was_actually_used": "True"}, # Good
|
||||
{"was_recommended": "False", "was_actually_used": "True"}, # Missed
|
||||
{"was_recommended": "True", "was_actually_used": "False"}, # Not used
|
||||
]
|
||||
|
||||
async def mock_hgetall(key):
|
||||
idx = mock_keys.index(key)
|
||||
data = tool_data[idx]
|
||||
return {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "tool_call",
|
||||
"duration_seconds": 1.0,
|
||||
"success": "True", # Booleans stored as strings in Redis
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": "test_tool",
|
||||
"conversation_id": None,
|
||||
"was_recommended": data["was_recommended"], # Already strings
|
||||
"was_actually_used": data["was_actually_used"], # Already strings
|
||||
}
|
||||
|
||||
mock_redis.hgetall.side_effect = mock_hgetall
|
||||
|
||||
accuracy = await store.get_tool_accuracy()
|
||||
|
||||
assert accuracy["total_calls"] == 4
|
||||
assert accuracy["total_used"] == 3
|
||||
assert accuracy["recommended_and_used"] == 2
|
||||
assert accuracy["not_recommended_but_used"] == 1
|
||||
assert accuracy["precision"] == pytest.approx(66.67, rel=0.01)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_tool_accuracy_empty(self, store, mock_redis):
|
||||
"""Test tool accuracy with no data."""
|
||||
mock_redis.zrevrangebyscore.return_value = []
|
||||
|
||||
accuracy = await store.get_tool_accuracy()
|
||||
|
||||
assert accuracy["total_calls"] == 0
|
||||
assert accuracy["precision"] == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close(self, store, mock_redis):
|
||||
"""Test closing the store."""
|
||||
await store.close()
|
||||
mock_redis.aclose.assert_called_once()
|
||||
|
||||
# Client should be None after close
|
||||
assert store._client is None
|
||||
|
||||
|
||||
class TestGlobalBenchmarkStore:
|
||||
"""Test global benchmark store instance."""
|
||||
|
||||
def test_get_benchmark_store(self):
|
||||
"""Test getting global store instance."""
|
||||
store = get_benchmark_store()
|
||||
assert isinstance(store, BenchmarkStore)
|
||||
|
||||
def test_get_benchmark_store_singleton(self):
|
||||
"""Test store is singleton."""
|
||||
store1 = get_benchmark_store()
|
||||
store2 = get_benchmark_store()
|
||||
assert store1 is store2
|
||||
@@ -43,7 +43,7 @@ LOG_FILE="$LOGS_DIR/server.log"
|
||||
echo -e "${YELLOW}Logs will be written to: ${LOG_FILE}${NC}"
|
||||
|
||||
# Start the server
|
||||
echo -e "${GREEN}Starting uvicorn server on http://localhost:8777${NC}"
|
||||
echo -e "${GREEN}Starting uvicorn server on http://tower-of-joy:8777${NC}"
|
||||
echo -e "${YELLOW}Press Ctrl+C to stop the server${NC}"
|
||||
echo ""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user