Compare commits
+9
-4
@@ -8,9 +8,16 @@ API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
API_PREFIX=/v1
|
||||
|
||||
# Ollama Configuration
|
||||
# Anthropic Configuration (Claude - preferred backend)
|
||||
# Set ANTHROPIC_API_KEY to enable Claude as the default backend
|
||||
# Without an API key, Tatlock uses Ollama exclusively
|
||||
# ANTHROPIC_API_KEY=sk-ant-api03-your-key-here
|
||||
ANTHROPIC_MODEL=claude-sonnet-4-20250514
|
||||
PREFER_CLOUD_BACKEND=true
|
||||
|
||||
# Ollama Configuration (local fallback when Claude unavailable)
|
||||
OLLAMA_HOST=http://localhost:11434
|
||||
OLLAMA_DEFAULT_MODEL=mistral-nemo:latest
|
||||
OLLAMA_DEFAULT_MODEL=gemma4:e2b
|
||||
OLLAMA_TIMEOUT=120
|
||||
|
||||
# SearXNG Configuration
|
||||
@@ -21,7 +28,6 @@ SEARXNG_TIMEOUT=30
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
REDIS_MEMORY_DB=1
|
||||
REDIS_BENCHMARK_DB=6
|
||||
REDIS_TIMEOUT=5
|
||||
|
||||
# Qdrant Configuration
|
||||
@@ -33,7 +39,6 @@ QDRANT_PORT=6333
|
||||
# - development: DEBUG (maximum verbosity)
|
||||
# - production: WARNING (minimal noise)
|
||||
# Uncomment to override: LOG_LEVEL=INFO
|
||||
ENABLE_BENCHMARKS=true
|
||||
# Note: Log format is auto-selected based on ENVIRONMENT (console for dev, json for production)
|
||||
|
||||
# User Configuration
|
||||
|
||||
@@ -1,10 +1,22 @@
|
||||
name: Build and Push
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
push:
|
||||
tags:
|
||||
- 'v[0-9]*'
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create Gitea Release
|
||||
run: |
|
||||
curl -sf -X POST \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"tag_name": "${{ github.ref_name }}", "name": "Release ${{ github.ref_name }}", "body": "Automated release for ${{ github.ref_name }}"}' \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases"
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
+15
-7
@@ -46,29 +46,37 @@ ENV/
|
||||
.ipynb_checkpoints/
|
||||
*.ipynb
|
||||
|
||||
# Testing & Coverage
|
||||
# Caches (pytest, mypy, ruff)
|
||||
.cache/
|
||||
|
||||
# Build output (coverage, logs)
|
||||
build/
|
||||
|
||||
# Legacy cache/output locations (in case tools fall back)
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
.coverage.*
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
|
||||
# Testing
|
||||
.tox/
|
||||
.nox/
|
||||
*.cover
|
||||
.hypothesis/
|
||||
|
||||
# Type checking
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
.pyre/
|
||||
.pytype/
|
||||
|
||||
# Linting
|
||||
.ruff_cache/
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
logs/*
|
||||
!logs/traces/
|
||||
logs/traces/*
|
||||
!logs/traces/viewer.html
|
||||
*.log
|
||||
|
||||
# Database
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
This document contains instructions and documentation references for AI assistants working with this codebase.
|
||||
|
||||
> **📖 Important**: Before working on this project, read [PHILOSOPHY.md](PHILOSOPHY.md) to understand the system vision, architectural patterns, and design goals. All development should work towards realizing those patterns.
|
||||
> **📖 Important**: Before working on this project, read [docs/philosophy.md](docs/philosophy.md) to understand the system vision, architectural patterns, and design goals. All development should work towards realizing those patterns.
|
||||
# AGENTS.md
|
||||
|
||||
> **Start every session by reading this file.**
|
||||
|
||||
+193
-1
@@ -7,6 +7,184 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.2.0] - 2026-04-04
|
||||
|
||||
### Changed
|
||||
|
||||
- **Switch default Ollama model to gemma4:e2b** - Replaces mistral-nemo as the local LLM backend; gemma4:e2b has native function calling support, faster tool calling (2-4s vs 15-20s), better parameter accuracy on word problems, and uses less VRAM (8GB vs 9.2GB)
|
||||
|
||||
### Added
|
||||
|
||||
- **Tool calling benchmark script** (`scripts/benchmark_tool_calling.py`) - Compares tool calling accuracy and latency across Ollama models via the Tatlock API
|
||||
|
||||
## [2.1.0] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Streaming SSE compatibility with Open WebUI** - Switch from `exclude_none=True` to `exclude_unset=True` for SSE chunk serialization; `exclude_none` was too aggressive — it stripped `finish_reason: null` from intermediate chunks (which OpenAI includes), while `exclude_unset` correctly omits only fields never passed to the constructor (like `reasoning_content` on content-only chunks) while preserving explicitly-set `finish_reason: null`
|
||||
|
||||
### Changed
|
||||
|
||||
- **Project structure consolidation** - Moved documentation to `docs/`, consolidated all config into `pyproject.toml`, replaced `wakeup.sh`/`pytest.ini`/`requirements*.txt` with `Makefile` + `pyproject.toml`
|
||||
- **CI test gate** - Unit tests now gate release and build jobs in Gitea Actions workflow
|
||||
- **Build output organization** - Tool caches in `.cache/`, generated output (coverage, logs) in `build/`
|
||||
|
||||
## [2.0.5] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Streaming JSON compatibility** - Exclude null fields from streaming chunks using `exclude_none=True`; OpenAI's API omits null fields entirely, and including them (e.g., `content: null`, `reasoning_content: null`) caused parsing issues in Open WebUI
|
||||
|
||||
## [2.0.4] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Open WebUI streaming compatibility** - Replaced `sse_starlette` `EventSourceResponse` with plain `StreamingResponse` for chat completions; `sse_starlette` added `\r\n` line endings and extra SSE fields that Open WebUI couldn't parse
|
||||
|
||||
## [2.0.3] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Steward analysis leaking into responses** - Removed internal routing analysis (`DELEGATE: tatlock_core...`) from user-visible reasoning in both streaming and non-streaming paths
|
||||
|
||||
## [2.0.2] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **tool_choice format incompatibility** - Removed `extra_body` tool_choice hack for Claude backend; PydanticAI handles tool_choice natively for Anthropic, preventing infinite tool call loops
|
||||
- **CI trigger** - Changed workflow trigger from `release:published` to `push:tags:v[0-9]*`
|
||||
|
||||
## [2.0.1] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Expert agent registration failure** - `AnthropicModel` does not accept `api_key` directly; now passes it via `AnthropicProvider`
|
||||
|
||||
## [2.0.0] - 2026-02-05
|
||||
|
||||
### Added
|
||||
|
||||
- **Claude backend support (Claudification Phase 1)** - All agents now prefer Claude over Ollama
|
||||
- New `src/anthropic/` module with model selector and health check
|
||||
- `get_model()` factory returns Claude if available, Ollama as fallback
|
||||
- Startup health check caches Claude API availability
|
||||
- Configuration: `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `PREFER_CLOUD_BACKEND`
|
||||
- 200k token context when using Claude backend
|
||||
|
||||
- **Steward dual-backend support** - Direct API calls to Claude or Ollama
|
||||
- `_call_claude()`: Anthropic Messages API path
|
||||
- `_call_ollama()`: Existing Ollama generate API path (preserved)
|
||||
- Automatic fallback: if Claude call fails mid-request, retries with Ollama
|
||||
|
||||
- **Claudification project tracking** - `PROJECT_CLAUDIFICATION.md` with Phase 1/2 roadmap
|
||||
|
||||
### Changed
|
||||
|
||||
- **All PydanticAI agents refactored to use `get_model()`**:
|
||||
- Tatlock (6 instantiation locations)
|
||||
- Librarian
|
||||
- Biographer
|
||||
- Housekeeper
|
||||
- **`initialize_application()` is now async** - Supports async Claude health check at startup
|
||||
- **Dependencies**: `pydantic-ai-slim[openai,anthropic]` replaces `pydantic-ai-slim[openai]`
|
||||
- **Startup logging** now includes backend selection info (claude/ollama)
|
||||
- **Agent creation logging** now includes backend and model info
|
||||
|
||||
### Removed
|
||||
|
||||
- Stale `tests/core/test_benchmarks.py` (benchmark system was removed in v1.10.0)
|
||||
|
||||
## [1.11.0] - 2025-12-30
|
||||
|
||||
### Added
|
||||
|
||||
- **Paperless document integration** - HybridRAG now includes indexed PDFs and scanned documents from Paperless-ngx
|
||||
- New `include_documents` parameter in `hybrid_search` tool
|
||||
- 📑 icon for document sources in search results
|
||||
- Librarian prompt updated with document awareness
|
||||
|
||||
- **Volatile cache integration** - HybridRAG now includes pre-fetched real-time data
|
||||
- New `include_volatile` parameter in `hybrid_search` tool
|
||||
- ⚡ icon for volatile sources in search results
|
||||
- Supports weather, forecast, news, stock, crypto, sun, air_quality namespaces
|
||||
- Librarian prompt updated with volatile cache awareness (user-configured items only)
|
||||
|
||||
- **Biographer routing in Steward** - Personal memory queries now correctly route to The Biographer
|
||||
- Added explicit routing rules for "where do I live", "what car do I drive", etc.
|
||||
- Added biographer delegation examples to Steward prompt
|
||||
- Location keywords ("live", "where", "home") now trigger profile pre-fetch
|
||||
|
||||
### Changed
|
||||
|
||||
- **LibraryDeskClient.hybrid_search** - Now passes full config including `document_limit`, `volatile_limit`, and enable flags
|
||||
- **Steward guidelines** - Clarified that research queries about TOPICS go to Librarian, queries about USER go to Biographer
|
||||
|
||||
## [1.10.1] - 2025-12-23
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Tatlock's excessive apologizing** - Strengthened personality prompt to prevent unnecessary apologies after successful Librarian delegations. Added explicit "do NOT apologize" instructions to both system prompt and synthesis prompt.
|
||||
|
||||
## [1.10.0] - 2025-12-22
|
||||
|
||||
### Added
|
||||
|
||||
#### Lightweight Request Tracing
|
||||
- **JSON-based tracing system** for local development debugging
|
||||
- Captures full request flow through multi-agent architecture
|
||||
- `Trace` and `Span` dataclasses with automatic timing and nesting
|
||||
- ContextVar-based propagation for async-safe tracing
|
||||
- `trace_span` async context manager for clean instrumentation
|
||||
- Traces written to `logs/traces/{trace_id}.json`
|
||||
- Enabled via `DEBUG=true` environment variable
|
||||
- **Trace Viewer UI** (`logs/traces/viewer.html`)
|
||||
- Standalone HTML viewer with timeline visualization
|
||||
- Filter by status, search by request text
|
||||
- Expandable span details with prompts and responses
|
||||
- **Tracing REST API** (`/traces`)
|
||||
- `GET /traces` - Serve trace viewer UI
|
||||
- `GET /traces/list` - List available traces with filtering
|
||||
- `GET /traces/{trace_id}` - Retrieve specific trace JSON
|
||||
- Only available when `DEBUG=true`
|
||||
- **Full pipeline instrumentation**
|
||||
- Router-level trace start/end with context management
|
||||
- Steward analysis spans in preprocessing
|
||||
- Tatlock orchestrate/synthesize spans
|
||||
- Expert delegation spans (librarian/biographer/housekeeper)
|
||||
- Tool-level spans extracted from PydanticAI messages
|
||||
|
||||
### Changed
|
||||
|
||||
- **Replaced Redis benchmarks with file-based tracing** - Simpler, more useful for debugging
|
||||
- **Context management moved to service layer** - Router simplified, context set in response service
|
||||
- **Server binds to all interfaces** - `wakeup.sh` now uses `0.0.0.0` for network access
|
||||
|
||||
### Removed
|
||||
|
||||
- **Redis benchmark system** (`src/core/benchmarks.py`)
|
||||
- `ENABLE_BENCHMARKS` config setting
|
||||
- `REDIS_BENCHMARK_DB` config setting
|
||||
- `redis_url` property (kept `redis_memory_url`)
|
||||
- Benchmark recording in Steward service and tool tracking
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Librarian fabrication prevention** - Added explicit instructions to never invent data when tools fail or sources are unavailable
|
||||
|
||||
## [1.9.0] - 2025-12-18
|
||||
|
||||
### Changed
|
||||
|
||||
- **Housekeeper prompt optimization** - Rewrote system prompt for Mistral-Nemo function calling with negative constraints, step-by-step process, and explicit entity ID format guidance
|
||||
- **Housekeeper temperature setting** - Set temperature to 0.1 for deterministic tool calling behavior
|
||||
- **Device list room group priority** - Room groups now appear first in `list_devices` output with `[ROOM GROUP]` marker to address positional bias
|
||||
- **Tool docstring improvements** - Updated turn_on/turn_off/toggle with explicit `entity_id=` parameter examples
|
||||
|
||||
### Added
|
||||
|
||||
- **Housekeeper optimization findings** - Added `docs/housekeeper-optimization-findings.md` documenting the experiment journey from 0% to 100% success rate
|
||||
- **Housekeeper test script** - Added `scripts/test_housekeeper.sh` for room group detection regression testing
|
||||
|
||||
## [1.8.6] - 2025-12-17
|
||||
|
||||
### Fixed
|
||||
@@ -765,7 +943,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- CORS middleware
|
||||
- Exception handlers (OpenAI-compatible error format)
|
||||
|
||||
[Unreleased]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.6.0...main
|
||||
[Unreleased]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.1.0...main
|
||||
[2.1.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.0.5...v2.1.0
|
||||
[2.0.5]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.0.0...v2.0.5
|
||||
[2.0.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.11.0...v2.0.0
|
||||
[1.11.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.10.0...v1.11.0
|
||||
[1.10.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.9.0...v1.10.0
|
||||
[1.9.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.6...v1.9.0
|
||||
[1.8.6]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.5...v1.8.6
|
||||
[1.8.5]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.4...v1.8.5
|
||||
[1.8.4]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.3...v1.8.4
|
||||
[1.8.3]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.2...v1.8.3
|
||||
[1.8.2]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.1...v1.8.2
|
||||
[1.8.1]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.0...v1.8.1
|
||||
[1.8.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.7.0...v1.8.0
|
||||
[1.7.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.6.0...v1.7.0
|
||||
[1.6.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.5.0...v1.6.0
|
||||
[1.5.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.4.0...v1.5.0
|
||||
[1.4.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.3.3...v1.4.0
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# CLAUDE.md
|
||||
|
||||
Claude Code-specific notes for this project. For general development instructions, architecture, coding standards, and deployment — see [AGENTS.md](AGENTS.md).
|
||||
|
||||
## Setup & Commands
|
||||
|
||||
```bash
|
||||
make setup # Create venv and install all dependencies
|
||||
make test # Unit tests (no external services)
|
||||
make test-integration # Integration tests (needs Claude/Ollama)
|
||||
make run # Start dev server on port 8777
|
||||
make lint # Ruff linter + formatter check
|
||||
make typecheck # Mypy
|
||||
make clean # Remove caches and build artifacts
|
||||
```
|
||||
|
||||
Dependencies are in `pyproject.toml` (`[project.dependencies]` and `[project.optional-dependencies.dev]`).
|
||||
|
||||
## Critical Gotchas
|
||||
|
||||
**ASGITransport does NOT trigger FastAPI lifespan events.** The session-scoped `_initialize_app` fixture in `tests/conftest.py` calls `initialize_application()` explicitly via `asyncio.run()`. Without this, `check_claude_health()` never runs and `_claude_available` stays `None`, causing all tests to silently fall back to Ollama.
|
||||
|
||||
**AsyncIO scope mismatch.** `asyncio_default_fixture_loop_scope = function` is set in `pyproject.toml`. Session-scoped async fixtures cause `ScopeMismatch` errors. The fix is to use a sync fixture with `asyncio.run()` for session-scoped initialization.
|
||||
|
||||
**Ollama is unreliable for tool calling.** `mistral-nemo` on Ollama often does mental math instead of calling calculator tools, and frequently gets wrong answers. Claude reliably calls tools. If integration tests give wrong math answers, check which backend is actually being used.
|
||||
|
||||
**Integration test timeouts.** Set to 120s to match `OLLAMA_TIMEOUT` config. Ollama on tower-of-joy can be slow, especially on first request.
|
||||
|
||||
**`get_benchmark_store` does not exist.** The benchmarking module (`src/core/benchmarks.py`) was never implemented. `scripts/benchmark_analysis.py` also references it and is broken. Do not add mocks for it in tests.
|
||||
|
||||
**Steward tests need household registry.** Use `register_household_members()` (sync) in fixtures, not `initialize_application()` (async). The steward extracts capabilities from the registry.
|
||||
+2
-2
@@ -5,8 +5,8 @@ WORKDIR /app
|
||||
RUN apt-get update && apt-get install -y curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt pyproject.toml ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY pyproject.toml ./
|
||||
RUN pip install --no-cache-dir .
|
||||
|
||||
COPY src/ ./src/
|
||||
|
||||
|
||||
@@ -1,920 +0,0 @@
|
||||
# Tatlock Implementation Roadmap
|
||||
|
||||
> **Reference**: See [PHILOSOPHY.md](PHILOSOPHY.md) for the target architecture and vision
|
||||
|
||||
This document outlines the phased implementation plan to transform the current OpenAI-compatible API into the full Tatlock household butler system.
|
||||
|
||||
## Current State (v1.2.0 - Phase F Complete)
|
||||
|
||||
**What we have**:
|
||||
- ✅ **The Orchestrator** - FastAPI infrastructure layer
|
||||
- OpenAI-compatible API endpoints (Responses API + Chat Completions)
|
||||
- Streaming coordination and conversation management
|
||||
- Response format with reasoning support
|
||||
- Test infrastructure (~400 tests)
|
||||
- ✅ **Two-Tier Architecture**
|
||||
- The Steward analyzes requests and recommends capabilities
|
||||
- Tatlock coordinates execution with scoped tools
|
||||
- Real-time streaming of analysis and reasoning
|
||||
- ✅ **Household Staff**
|
||||
- **Tatlock** (Butler): Primary interface with witty personality
|
||||
- **The Steward**: Request analysis and capability recommendation
|
||||
- **The Librarian**: Research via library-desk HybridRAG + wiki
|
||||
- **The Biographer**: User memory, profiles, preferences, semantic recall
|
||||
- ✅ **Core Tools**
|
||||
- Calculator, Date/Time toolkit, Web search (SearXNG)
|
||||
- ✅ **Memory System**
|
||||
- Direct access layer (memory_service) for fast lookups
|
||||
- Vector storage (Qdrant) for semantic recall
|
||||
- Session cache (Redis) with 24h TTL
|
||||
- Multi-tenancy via ContextVar
|
||||
- ✅ Mock agent (lorem-tester for testing)
|
||||
|
||||
**What we need**:
|
||||
- More household staff (Developer, Secretary, Handyman, Housekeeper)
|
||||
- MCP (Model Context Protocol) integration
|
||||
- Dynamic model switching for specialized tasks
|
||||
- Full multi-tenant database (PostgreSQL)
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Real LLM Integration - PydanticAI + Tools
|
||||
|
||||
**Goal**: Connect to actual language models and establish the base plumbing
|
||||
|
||||
**Note**: Ollama is an external service dependency (already running separately)
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **PydanticAI Integration** ✅
|
||||
- PydanticAI → Ollama connection ✅
|
||||
- Agent creation patterns ✅
|
||||
- Streaming response handling ✅
|
||||
- Error handling and retries ✅
|
||||
|
||||
2. **Convert Tatlock Agent** ✅
|
||||
- Convert Tatlock agent from mock to PydanticAI ✅
|
||||
- British butler personality prompt ✅
|
||||
- Research-oriented mindset ✅
|
||||
- Streaming to reasoning output ✅
|
||||
- Tool calling framework setup ✅
|
||||
|
||||
3. **Permanent Tools** ✅
|
||||
- Calculator: Safe mathematical expression evaluation ✅
|
||||
- Date/Time toolkit: Current time, relative dates, time differences ✅
|
||||
- Web search: SearXNG integration (external service) ✅
|
||||
- Tool registration with PydanticAI ✅
|
||||
|
||||
4. **Testing Infrastructure** ✅
|
||||
- Integration tests with real LLM ✅
|
||||
- Tool functionality tests ✅
|
||||
- Response quality validation ✅
|
||||
- 131 tests, 81.78% coverage ✅
|
||||
|
||||
### Success Criteria
|
||||
- [x] **PydanticAI agents can call Ollama** (mistral-nemo:latest)
|
||||
- [x] **Streaming works end-to-end**
|
||||
- [x] **Tool calling framework functional**
|
||||
- [x] **Permanent tools working** (calculator, date/time, search)
|
||||
- [x] **Tests pass with real LLM**
|
||||
- [ ] Can switch models dynamically (e.g., Codestral for code)
|
||||
|
||||
### Status
|
||||
**✅ MOSTLY COMPLETE** - Tatlock agent functional with permanent tools
|
||||
|
||||
### Remaining Work
|
||||
- Dynamic model switching for specialized tasks (e.g., Codestral for coding)
|
||||
|
||||
### Why First?
|
||||
Without real LLM integration, we can't meaningfully implement the Steward/Butler pattern. Everything else depends on having actual AI agents working.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Orchestration Layer - The Steward
|
||||
|
||||
**Goal**: Implement the first-tier LLM call for tool/agent selection
|
||||
|
||||
**Purpose**: The Steward performs crucial preparatory work before Tatlock engages with a request. By analyzing incoming requests and determining which tools, services, and household staff members will be needed, the Steward creates a curated recommendation that streamlines Tatlock's work and prevents cognitive overload.
|
||||
|
||||
### Core Architecture
|
||||
|
||||
The Steward operates as the first tier in the two-tier request flow:
|
||||
|
||||
```
|
||||
User Request → Orchestrator → Steward Analysis → Recommendations → Tatlock (with scoped tools/agents)
|
||||
```
|
||||
|
||||
**Key Principle**: The Steward narrows the scope to only relevant capabilities, making Tatlock's decision-making cleaner and more focused.
|
||||
|
||||
### Deliverables
|
||||
|
||||
#### 1. Tool & Agent Registry System
|
||||
|
||||
**Purpose**: Centralized catalog of all available capabilities for the Steward to recommend
|
||||
|
||||
**Implementation Details**:
|
||||
- **Registry Module** (`src/core/registry.py`)
|
||||
- Tool registration decorator pattern
|
||||
- Agent registration with capability metadata
|
||||
- Category-based organization (computation, information, automation, communication)
|
||||
- Dynamic tool/agent discovery and loading
|
||||
|
||||
- **Tool Metadata Schema**
|
||||
```python
|
||||
{
|
||||
"name": "calculator",
|
||||
"category": "computation",
|
||||
"description": "Safe mathematical expression evaluation",
|
||||
"capabilities": ["arithmetic", "algebra", "trigonometry"],
|
||||
"cost": "low", # computational cost indicator
|
||||
"requires_network": false
|
||||
}
|
||||
```
|
||||
|
||||
- **Agent Metadata Schema**
|
||||
```python
|
||||
{
|
||||
"name": "developer",
|
||||
"role": "The Developer",
|
||||
"category": "technical",
|
||||
"description": "Software development assistance",
|
||||
"domains": ["code_generation", "debugging", "architecture"],
|
||||
"specialized_model": "codestral", # optional
|
||||
"cost": "high"
|
||||
}
|
||||
```
|
||||
|
||||
- **Registry API**
|
||||
- `get_all_tools()` - List all available tools
|
||||
- `get_all_agents()` - List all expert agents
|
||||
- `get_by_category(category)` - Filter by category
|
||||
- `search_by_capability(query)` - Semantic search (future: vector search)
|
||||
|
||||
**Testing**:
|
||||
- Unit tests for registration and retrieval
|
||||
- Test dynamic loading of new tools/agents
|
||||
- Validate metadata schemas
|
||||
|
||||
#### 2. Steward PydanticAI Agent
|
||||
|
||||
**Purpose**: First-tier LLM that analyzes requests and recommends relevant tools/agents
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Agent Module** (`src/agents/steward.py`)
|
||||
```python
|
||||
from pydantic_ai import Agent, RunContext
|
||||
from pydantic import BaseModel
|
||||
|
||||
class StewardRecommendation(BaseModel):
|
||||
"""Structured output from Steward analysis"""
|
||||
recommended_tools: list[str]
|
||||
recommended_agents: list[str]
|
||||
reasoning: str
|
||||
estimated_complexity: str # "simple", "moderate", "complex"
|
||||
requires_multi_step: bool
|
||||
|
||||
steward = Agent(
|
||||
'ollama:mistral-nemo', # Same base model as Tatlock
|
||||
result_type=StewardRecommendation,
|
||||
system_prompt="""..."""
|
||||
)
|
||||
```
|
||||
|
||||
- **System Prompt Engineering**
|
||||
- Role: Estate steward responsible for efficient household coordination
|
||||
- Task: Analyze requests to determine needed resources
|
||||
- Output: Structured recommendations with reasoning
|
||||
- Constraints: Be conservative (recommend only truly relevant capabilities)
|
||||
- Context: Full registry of available tools and agents
|
||||
|
||||
- **Steward Tools**
|
||||
```python
|
||||
@steward.tool
|
||||
def get_available_capabilities(ctx: RunContext) -> dict:
|
||||
"""Get catalog of all available tools and agents."""
|
||||
return {
|
||||
"tools": registry.get_all_tools(),
|
||||
"agents": registry.get_all_agents()
|
||||
}
|
||||
```
|
||||
|
||||
- **Request Analysis Flow**
|
||||
1. Receive user request
|
||||
2. Query capability registry via tool
|
||||
3. Analyze request for required capabilities
|
||||
4. Generate structured recommendation
|
||||
5. Format as note to Tatlock
|
||||
|
||||
**Testing**:
|
||||
- Test various request types (simple, complex, multi-domain)
|
||||
- Verify recommendations are relevant and not over-inclusive
|
||||
- Test structured output parsing
|
||||
- Validate reasoning quality
|
||||
|
||||
#### 3. Request Preprocessing Pipeline
|
||||
|
||||
**Purpose**: Integration layer that routes requests through Steward before Tatlock
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Preprocessing Module** (`src/core/preprocessing.py`)
|
||||
```python
|
||||
async def preprocess_request(user_request: str) -> EnrichedRequest:
|
||||
"""
|
||||
1. Call Steward for analysis
|
||||
2. Get recommendations
|
||||
3. Enrich original request
|
||||
4. Return scoped context for Tatlock
|
||||
"""
|
||||
# Get Steward analysis
|
||||
steward_result = await steward.run(user_request)
|
||||
recommendations = steward_result.data
|
||||
|
||||
# Create note to Tatlock
|
||||
steward_note = format_steward_note(recommendations)
|
||||
|
||||
# Build scoped tool/agent list
|
||||
scoped_tools = get_scoped_tools(recommendations.recommended_tools)
|
||||
scoped_agents = get_scoped_agents(recommendations.recommended_agents)
|
||||
|
||||
return EnrichedRequest(
|
||||
original_request=user_request,
|
||||
steward_note=steward_note,
|
||||
available_tools=scoped_tools,
|
||||
available_agents=scoped_agents,
|
||||
metadata=recommendations
|
||||
)
|
||||
```
|
||||
|
||||
- **Note Formatting**
|
||||
```
|
||||
=== Internal Note from the Steward ===
|
||||
|
||||
Request Analysis:
|
||||
{steward reasoning}
|
||||
|
||||
Recommended Tools:
|
||||
- calculator: For mathematical computations
|
||||
- web_search: To find current information
|
||||
|
||||
Recommended Household Staff:
|
||||
- The Developer: For code generation assistance
|
||||
|
||||
Estimated Complexity: moderate
|
||||
===================================
|
||||
|
||||
[Original User Request]
|
||||
```
|
||||
|
||||
- **Orchestrator Integration**
|
||||
- Modify `src/responses/service.py` to call preprocessing
|
||||
- Prepend Steward note to request before sending to Tatlock
|
||||
- Limit Tatlock's tool access to recommended tools only
|
||||
- Stream Steward's reasoning to output
|
||||
|
||||
**Testing**:
|
||||
- Integration tests for full preprocessing flow
|
||||
- Test request enrichment format
|
||||
- Verify tool scoping works correctly
|
||||
- Test streaming of Steward reasoning
|
||||
|
||||
#### 4. Real-Time Transparency
|
||||
|
||||
**Purpose**: Stream Steward's analysis to user's reasoning output
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Streaming Integration** (`src/responses/streaming.py`)
|
||||
- Add Steward analysis phase to stream
|
||||
- Format as reasoning item
|
||||
- Include recommendation summary
|
||||
|
||||
- **Example Output to User**:
|
||||
```
|
||||
[Reasoning]
|
||||
Consulting the Steward for resource planning...
|
||||
|
||||
The Steward's Analysis:
|
||||
- Request requires mathematical computation
|
||||
- Need to verify current information via web search
|
||||
- May benefit from Developer's code expertise
|
||||
|
||||
Recommended: calculator, web_search, The Developer
|
||||
|
||||
Proceeding with scoped resources...
|
||||
```
|
||||
|
||||
**Testing**:
|
||||
- Test streaming of Steward analysis
|
||||
- Verify formatting in Open WebUI
|
||||
- Test error handling if Steward fails
|
||||
|
||||
#### 5. Model Efficiency Optimization
|
||||
|
||||
**Purpose**: Ensure the base model stays loaded in VRAM
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Shared Model Configuration**
|
||||
- Both Steward and Tatlock use `ollama:mistral-nemo` by default
|
||||
- Sequential calls (Steward → Tatlock) keep model hot
|
||||
- No reload delays between tiers
|
||||
|
||||
- **Performance Monitoring**
|
||||
- Log response times for Steward calls
|
||||
- Track total request latency (Steward + Tatlock)
|
||||
- Identify optimization opportunities
|
||||
|
||||
**Testing**:
|
||||
- Benchmark Steward → Tatlock call latency
|
||||
- Verify model stays loaded between calls
|
||||
- Test performance under load
|
||||
|
||||
### Implementation Strategy
|
||||
|
||||
#### Week 1-2: Foundation
|
||||
- [ ] Design and implement registry system
|
||||
- [ ] Create tool/agent metadata schemas
|
||||
- [ ] Build registry API with tests
|
||||
- [ ] Migrate existing tools to registry
|
||||
|
||||
#### Week 3-4: Steward Agent
|
||||
- [ ] Create Steward PydanticAI agent
|
||||
- [ ] Engineer system prompt for analysis
|
||||
- [ ] Implement structured recommendation output
|
||||
- [ ] Add registry query tool
|
||||
- [ ] Test with various request types
|
||||
|
||||
#### Week 5-6: Integration
|
||||
- [ ] Build request preprocessing pipeline
|
||||
- [ ] Implement note formatting
|
||||
- [ ] Integrate with Orchestrator
|
||||
- [ ] Add streaming transparency
|
||||
- [ ] Tool scoping for Tatlock
|
||||
|
||||
#### Week 7: Testing & Refinement
|
||||
- [ ] End-to-end integration tests
|
||||
- [ ] Performance optimization
|
||||
- [ ] Prompt refinement based on results
|
||||
- [ ] Documentation and examples
|
||||
|
||||
### Success Criteria
|
||||
|
||||
- [x] **Steward analyzes incoming requests** using PydanticAI agent
|
||||
- [x] **Produces structured recommendations** (tools, agents, reasoning)
|
||||
- [x] **Recommendations formatted as prepended note** to Tatlock
|
||||
- [x] **Tool registry is queryable and extensible** via clean API
|
||||
- [x] **Steward output visible in reasoning stream** for transparency
|
||||
- [x] **Only recommended tools available** to Tatlock (scoped context)
|
||||
- [x] **Base model stays loaded** between Steward and Tatlock calls
|
||||
- [x] **Recommendations are accurate** (not over/under-inclusive)
|
||||
- [x] **Integration tests pass** for full Steward → Tatlock flow
|
||||
|
||||
### Status
|
||||
**✅ COMPLETE** (v0.2.5)
|
||||
|
||||
### Performance Targets
|
||||
|
||||
- **Steward Analysis Time**: < 2 seconds for typical requests
|
||||
- **Total Added Latency**: < 3 seconds including streaming
|
||||
- **Recommendation Accuracy**: > 90% relevance (manual evaluation)
|
||||
- **Model Reload Delay**: 0 seconds (model stays hot)
|
||||
|
||||
### Risk Mitigation
|
||||
|
||||
**Risk**: Steward recommendations too broad (defeats purpose)
|
||||
- Mitigation: Conservative prompt engineering, test with diverse requests, iterate
|
||||
|
||||
**Risk**: Added latency unacceptable to users
|
||||
- Mitigation: Stream Steward reasoning for transparency, optimize prompt, parallel processing where possible
|
||||
|
||||
**Risk**: Tool registry becomes unwieldy
|
||||
- Mitigation: Good categorization, semantic search (future), regular pruning
|
||||
|
||||
**Risk**: Steward and Tatlock models compete for VRAM
|
||||
- Mitigation: Use same base model, sequential calls, monitor memory
|
||||
|
||||
### Future Enhancements (Post-Phase 2)
|
||||
|
||||
- **Semantic Search**: Vector-based capability search instead of metadata lookup
|
||||
- **Learning from Usage**: Track which recommendations work well, adjust over time
|
||||
- **Confidence Scores**: Steward provides confidence for each recommendation
|
||||
- **Request Classification**: Cache classifications for similar requests
|
||||
- **Multi-Model Support**: Allow Steward to recommend specialized models for specific tasks
|
||||
|
||||
### Estimated Effort
|
||||
|
||||
**7-8 weeks** - Core intelligence routing with comprehensive implementation
|
||||
|
||||
### Why Second?
|
||||
|
||||
The Steward is the foundation of the household architecture. Without it, we'd need to expose all tools/agents to Tatlock, creating cognitive overload and poor decision-making. The Steward enables the focused expertise pattern that makes the whole system work.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: The Butler - Tatlock Agent
|
||||
|
||||
**Goal**: Implement the second-tier coordinator with personality within the existing Orchestrator infrastructure
|
||||
|
||||
**Context**: The Orchestrator (FastAPI infrastructure) already exists. This phase implements the real Tatlock PydanticAI agent to replace the current mock agent.
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Butler Agent (Tatlock)**
|
||||
- PydanticAI agent implementation within Orchestrator
|
||||
- Personality prompt engineering (witty British butler)
|
||||
- Tool calling framework
|
||||
- Multi-agent coordination logic
|
||||
|
||||
2. **Scoped Tool Access**
|
||||
- Filter tools based on Steward recommendations
|
||||
- Dynamic tool loading for Butler context
|
||||
- Tool execution framework
|
||||
- Result aggregation
|
||||
|
||||
3. **Real-Time Reasoning Output**
|
||||
- Stream all Butler activities to reasoning output
|
||||
- Tool call progress indicators
|
||||
- Expert agent consultation messages
|
||||
- Wait time transparency
|
||||
|
||||
### Success Criteria
|
||||
- [x] Tatlock receives enriched requests (user + Steward notes)
|
||||
- [x] Only recommended tools are available
|
||||
- [x] Tatlock coordinates multiple tool calls
|
||||
- [x] All actions streamed to reasoning output
|
||||
- [x] Responses have consistent personality
|
||||
- [x] Synthesizes multi-source results coherently
|
||||
|
||||
### Status
|
||||
**✅ COMPLETE** (v1.1.0)
|
||||
|
||||
### Estimated Effort
|
||||
**4-5 weeks** - Complex coordination logic
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Expert Household Staff - Core Agents
|
||||
|
||||
**Goal**: Implement the initial set of domain-specific expert agents
|
||||
|
||||
### Priority Expert Agents
|
||||
|
||||
1. **The Librarian** (Research & Knowledge Management) ✅ **COMPLETE** (v1.1.0)
|
||||
- Research assistance via library-desk HybridRAG
|
||||
- Wiki page management (search, create, update)
|
||||
- Semantic vector search
|
||||
- Knowledge graph queries
|
||||
- Dossier browsing
|
||||
|
||||
2. **The Biographer** (User Memory) ✅ **COMPLETE** (v1.2.0)
|
||||
- User profile management (name, location, timezone)
|
||||
- Preference storage (units, theme)
|
||||
- Semantic memory recall ("What car do I drive?")
|
||||
- Fact storage from conversations
|
||||
- Session context caching
|
||||
|
||||
3. **The Developer** (Software Development) 🔜 **Planned**
|
||||
- Code generation assistance
|
||||
- Debugging support
|
||||
- Documentation generation
|
||||
- Architecture guidance
|
||||
- *Rationale: Directly supports building the system itself*
|
||||
|
||||
4. **The Handyman** (System Maintenance) 🔜 **Planned**
|
||||
- System status queries
|
||||
- Log analysis
|
||||
- Basic troubleshooting
|
||||
- Infrastructure monitoring
|
||||
|
||||
5. **The Secretary** (Scheduling & Organization) 🔜 **Planned**
|
||||
- Calendar integration
|
||||
- Task management
|
||||
- Reminder system
|
||||
- Schedule conflict detection
|
||||
|
||||
6. **The Housekeeper** (Home Automation) 🔜 **Planned**
|
||||
- Home Assistant integration
|
||||
- Device control interface
|
||||
- Status queries
|
||||
- Automation triggers
|
||||
|
||||
### Each Agent Includes
|
||||
- Specialized prompt and personality
|
||||
- Domain-specific tools
|
||||
- MCP integration points (where applicable)
|
||||
- Integration with Butler orchestration
|
||||
|
||||
### Success Criteria
|
||||
- [x] Each agent implemented as separate module
|
||||
- [x] Agents callable via tool framework
|
||||
- [x] Agents use specialized prompts
|
||||
- [x] Results integrate cleanly with Butler
|
||||
- [ ] Can invoke specialized models (e.g., Codestral for Developer)
|
||||
|
||||
### Status
|
||||
**🔶 PARTIAL** - Librarian and Biographer complete, others planned
|
||||
|
||||
### Estimated Effort
|
||||
**6-8 weeks** - Parallel development possible
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: Persistence Layer - Database & Multi-Tenancy
|
||||
|
||||
**Goal**: Add persistent storage and multi-user support when needed
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **PostgreSQL Integration**
|
||||
- Docker compose configuration for PostgreSQL
|
||||
- Database schema design with tenant isolation
|
||||
- Alembic migrations setup
|
||||
- SQLAlchemy models
|
||||
|
||||
2. **Multi-Tenant Architecture**
|
||||
- Tenant identification middleware
|
||||
- Tenant-scoped database sessions
|
||||
- User authentication system (basic)
|
||||
- Per-tenant data isolation
|
||||
|
||||
3. **Core Data Models**
|
||||
- Users and tenants
|
||||
- Conversations and messages (migrate from in-memory)
|
||||
- Agent interactions log
|
||||
- System configuration and preferences
|
||||
|
||||
4. **Migration Strategy**
|
||||
- Gradual migration from in-memory to database
|
||||
- Backward compatibility during transition
|
||||
- Data export/import utilities
|
||||
|
||||
### Success Criteria
|
||||
- [ ] PostgreSQL container running
|
||||
- [ ] Multiple users can authenticate separately
|
||||
- [ ] Each user sees only their own data
|
||||
- [ ] Conversations persist across restarts
|
||||
- [ ] Database migrations work correctly
|
||||
- [ ] Tests verify tenant isolation
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Data layer foundation
|
||||
|
||||
### Why Later?
|
||||
The core orchestration (Steward → Butler → Experts) can work entirely with in-memory state. We only need database persistence when we want conversations to survive restarts and multiple users to have isolated experiences.
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Extended Services Integration
|
||||
|
||||
**Goal**: Connect to additional supporting services
|
||||
|
||||
### Services to Integrate
|
||||
|
||||
1. **Redis (Memory & Caching)** ✅ **COMPLETE** (v1.2.0)
|
||||
- Benchmark storage (db=1)
|
||||
- Memory cache for sessions (db=2)
|
||||
- 24h TTL for session context
|
||||
- Recent entities tracking
|
||||
|
||||
2. **Qdrant (Vector Storage)** ✅ **COMPLETE** (v1.2.0)
|
||||
- Per-user memory collections
|
||||
- 768-dim nomic-embed-text vectors
|
||||
- Semantic search for recall
|
||||
- Type-based filtering
|
||||
|
||||
3. **SearxNG (Web Search)** ✅ **COMPLETE** (v0.2.0)
|
||||
- Search tool integration
|
||||
- Result processing
|
||||
- Privacy-preserving queries
|
||||
|
||||
4. **library-desk (Research API)** ✅ **COMPLETE** (v1.1.0)
|
||||
- HybridRAG search
|
||||
- Wiki management
|
||||
- Knowledge graph queries
|
||||
|
||||
### Success Criteria
|
||||
- [x] Services communicate correctly
|
||||
- [x] Tatlock can invoke web search
|
||||
- [x] Redis used for session data
|
||||
- [x] Qdrant stores user memories
|
||||
- [x] Ollama serves the base model
|
||||
|
||||
### Status
|
||||
**✅ COMPLETE** - All core services integrated
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Infrastructure setup
|
||||
|
||||
---
|
||||
|
||||
## Phase 7: MCP (Model Context Protocol) Integration
|
||||
|
||||
**Goal**: Enable rich tool integrations via MCP
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **MCP Server Framework**
|
||||
- MCP server implementation
|
||||
- Tool registration via MCP
|
||||
- Schema validation
|
||||
- Error handling
|
||||
|
||||
2. **MCP Client in Agents**
|
||||
- PydanticAI MCP integration
|
||||
- Tool discovery from MCP servers
|
||||
- Dynamic tool loading
|
||||
- Result processing
|
||||
|
||||
3. **Initial MCP Tools**
|
||||
- File system operations
|
||||
- Database queries
|
||||
- API integrations
|
||||
- System commands
|
||||
|
||||
### Success Criteria
|
||||
- [ ] MCP server running
|
||||
- [ ] Tools exposed via MCP protocol
|
||||
- [ ] Agents can discover and use MCP tools
|
||||
- [ ] New tools addable without code changes
|
||||
- [ ] MCP tools visible in Steward recommendations
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Standards-based integration
|
||||
|
||||
---
|
||||
|
||||
## Phase 8: Advanced Memory & Context
|
||||
|
||||
**Goal**: Implement sophisticated memory and context management
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Long-Term Memory** ✅ **COMPLETE** (v1.2.0 - Phase F)
|
||||
- Memory service for direct key-based access
|
||||
- Qdrant vector storage for semantic recall
|
||||
- Embedding via nomic-embed-text
|
||||
- The Biographer agent for memory management
|
||||
|
||||
2. **Session Memory** ✅ **COMPLETE** (v1.2.0)
|
||||
- Redis session cache with 24h TTL
|
||||
- Recent entities tracking
|
||||
- Conversation context preservation
|
||||
- Multi-tenancy via ContextVar
|
||||
|
||||
3. **Steward Integration** ✅ **COMPLETE** (v1.2.0)
|
||||
- Memory pre-fetch during request analysis
|
||||
- Profile/preferences included in context
|
||||
- Keyword-based context determination
|
||||
|
||||
4. **Context Management** 🔜 **Future**
|
||||
- Smart context window trimming
|
||||
- Conversation branching
|
||||
- Topic tracking
|
||||
- Memory retrieval integration
|
||||
|
||||
5. **Personalization** 🔜 **Future**
|
||||
- User preference learning
|
||||
- Interaction pattern analysis
|
||||
- Adaptive responses
|
||||
- Custom agent personalities per user
|
||||
|
||||
### Success Criteria
|
||||
- [x] User facts stored in Qdrant with semantic search
|
||||
- [x] Profile and preferences accessible via memory_service
|
||||
- [x] Session context cached in Redis
|
||||
- [x] User preferences affect responses (via Steward pre-fetch)
|
||||
- [ ] Conversations automatically embedded to Qdrant
|
||||
- [ ] Memory improves over time (learning from interactions)
|
||||
|
||||
### Status
|
||||
**🔶 PARTIAL** - Core memory system complete, advanced features planned
|
||||
|
||||
### Estimated Effort
|
||||
**4-5 weeks** - AI/ML heavy (remaining work)
|
||||
|
||||
---
|
||||
|
||||
## Phase 9: Extended Household Staff
|
||||
|
||||
**Goal**: Add specialized agents for additional domains
|
||||
|
||||
### Future Agents
|
||||
|
||||
1. **The Librarian** (Knowledge Management)
|
||||
- Personal documentation indexing
|
||||
- Research assistance
|
||||
- Knowledge base queries
|
||||
- Reference management
|
||||
|
||||
2. **The Accountant** (Financial Tracking)
|
||||
- Expense tracking
|
||||
- Budget monitoring
|
||||
- Financial reports
|
||||
- Transaction categorization
|
||||
|
||||
3. **The Chef** (Meal Planning)
|
||||
- Recipe management
|
||||
- Meal planning
|
||||
- Nutrition tracking
|
||||
- Grocery lists
|
||||
|
||||
4. **Others as Needed**
|
||||
- Domain-specific as requirements emerge
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Each new agent follows household pattern
|
||||
- [ ] Integrates with Steward/Butler flow
|
||||
- [ ] Has appropriate specialized tools
|
||||
- [ ] Documented in PHILOSOPHY.md updates
|
||||
|
||||
### Estimated Effort
|
||||
**Ongoing** - Add as needed
|
||||
|
||||
---
|
||||
|
||||
## Phase 10: User Experience Refinement
|
||||
|
||||
**Goal**: Polish the interaction experience
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Personality Tuning**
|
||||
- Refine Tatlock's wit and tone
|
||||
- Consistent household character
|
||||
- Cultural references appropriate
|
||||
- Humor that doesn't annoy
|
||||
|
||||
2. **Transparency Improvements**
|
||||
- Better progress indicators
|
||||
- Clearer reasoning explanations
|
||||
- Informative wait messages
|
||||
- Error message clarity
|
||||
|
||||
3. **Performance Optimization**
|
||||
- Response time improvements
|
||||
- Model loading optimization
|
||||
- Caching strategies
|
||||
- Streaming smoothness
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Users find Tatlock engaging
|
||||
- [ ] Wait times feel reasonable
|
||||
- [ ] Errors are understandable
|
||||
- [ ] System feels responsive
|
||||
|
||||
### Estimated Effort
|
||||
**Ongoing** - Continuous improvement
|
||||
|
||||
---
|
||||
|
||||
## Phase 11: Production Hardening
|
||||
|
||||
**Goal**: Make the system production-ready for homelab deployment
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Deployment**
|
||||
- Complete docker-compose stack
|
||||
- Environment configuration
|
||||
- Backup strategies
|
||||
- Update procedures
|
||||
|
||||
2. **Monitoring**
|
||||
- Health checks
|
||||
- Performance metrics
|
||||
- Error tracking
|
||||
- Usage analytics
|
||||
|
||||
3. **Security**
|
||||
- Authentication hardening
|
||||
- Rate limiting
|
||||
- Input validation
|
||||
- Audit logging
|
||||
|
||||
4. **Documentation**
|
||||
- Installation guide
|
||||
- Configuration reference
|
||||
- Troubleshooting guide
|
||||
- Architecture documentation
|
||||
|
||||
### Success Criteria
|
||||
- [ ] One-command deployment
|
||||
- [ ] System health is monitorable
|
||||
- [ ] Secure for homelab use
|
||||
- [ ] Well documented
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Production polish
|
||||
|
||||
---
|
||||
|
||||
## Dependencies Between Phases
|
||||
|
||||
```
|
||||
Phase 1 (Ollama + PydanticAI) ← Foundation for all AI
|
||||
↓
|
||||
Phase 2 (Steward)
|
||||
↓
|
||||
Phase 3 (Butler/Tatlock)
|
||||
↓
|
||||
Phase 4 (Expert Agents) ← Phase 7 (MCP) can enhance
|
||||
↓
|
||||
Phase 5 (Database/Multi-Tenancy) ← Can be deferred
|
||||
↓
|
||||
Phase 6 (Extended Services) → Phase 8 (Advanced Memory)
|
||||
↓
|
||||
Phase 9 (Extended Staff) → Phase 10 (UX) → Phase 11 (Production)
|
||||
```
|
||||
|
||||
**Critical Path**: Phases 1 → 2 → 3 → 4 must be sequential
|
||||
**Can Be Deferred**: Phase 5 (Database) until you need persistence
|
||||
**Parallel Opportunities**: Phase 6 and 7 can overlap; Phase 9 and 10 ongoing
|
||||
|
||||
---
|
||||
|
||||
## Overall Timeline Estimate
|
||||
|
||||
**Minimum Viable Household** (Phases 1-4): **15-20 weeks**
|
||||
- Working Steward → Butler → Expert Agents with real LLM
|
||||
- In-memory state (no persistence needed yet)
|
||||
- Core household functional
|
||||
|
||||
**With Persistence** (Phases 1-5): **18-24 weeks**
|
||||
- Add database and multi-tenancy
|
||||
- Conversations survive restarts
|
||||
- Multiple users supported
|
||||
|
||||
**Full-Featured System** (Phases 1-9): **35-45 weeks**
|
||||
- All services integrated
|
||||
- Advanced memory and context
|
||||
- Extended household staff
|
||||
|
||||
**Production-Ready** (All phases): **40-50 weeks**
|
||||
- Polished UX
|
||||
- Hardened for homelab deployment
|
||||
- Fully documented
|
||||
|
||||
*Note: Timeline assumes consistent part-time development effort*
|
||||
|
||||
---
|
||||
|
||||
## Success Metrics
|
||||
|
||||
### Technical
|
||||
- System implements PHILOSOPHY.md patterns
|
||||
- All household roles functional
|
||||
- Multi-tenant isolation verified
|
||||
- Real-time reasoning transparency working
|
||||
- MCP integration complete
|
||||
|
||||
### User Experience
|
||||
- Tatlock feels like interacting with a butler
|
||||
- Wait times are transparent and acceptable
|
||||
- Expert agents provide value in their domains
|
||||
- System is reliable and trustworthy
|
||||
|
||||
### Architecture
|
||||
- Clean separation between household roles
|
||||
- Easy to add new agents/tools
|
||||
- Model efficiency (base model stays loaded)
|
||||
- Scales to household + friends usage
|
||||
|
||||
---
|
||||
|
||||
## Risk Management
|
||||
|
||||
### High Risk Items
|
||||
1. **PydanticAI + Ollama integration complexity**
|
||||
- Mitigation: Prototype early, iterate on connection layer
|
||||
|
||||
2. **Multi-agent coordination complexity**
|
||||
- Mitigation: Start simple, add coordination gradually
|
||||
|
||||
3. **Model performance on homelab hardware**
|
||||
- Mitigation: Model selection, quantization, optimization
|
||||
|
||||
4. **Prompt engineering for personality consistency**
|
||||
- Mitigation: Extensive testing, user feedback, iteration
|
||||
|
||||
### Medium Risk Items
|
||||
- MCP protocol adoption and tooling maturity
|
||||
- Vector embedding quality for memory
|
||||
- Home automation integration variability
|
||||
- User authentication security
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Priority**: Implement The Developer agent for code assistance
|
||||
2. **Integration**: Add Home Assistant integration for The Housekeeper
|
||||
3. **Calendar**: Integrate scheduling service for The Secretary
|
||||
4. **Ongoing**: Add more household staff as needed
|
||||
|
||||
---
|
||||
|
||||
**Document Status**: Active planning document
|
||||
**Created**: 2025-12-06
|
||||
**Last Updated**: 2025-12-13
|
||||
@@ -0,0 +1,48 @@
|
||||
.PHONY: help setup run test test-unit test-integration lint typecheck clean
|
||||
|
||||
VENV := .venv
|
||||
PYTHON := $(VENV)/bin/python
|
||||
PIP := $(VENV)/bin/pip
|
||||
PYTEST := $(VENV)/bin/pytest
|
||||
RUFF := $(VENV)/bin/ruff
|
||||
MYPY := $(VENV)/bin/mypy
|
||||
UVICORN := $(VENV)/bin/uvicorn
|
||||
|
||||
HOST := 0.0.0.0
|
||||
PORT := 8777
|
||||
|
||||
help: ## Show this help
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
setup: ## Create venv and install all dependencies
|
||||
python3 -m venv $(VENV)
|
||||
$(PIP) install --upgrade pip
|
||||
$(PIP) install -e ".[dev]"
|
||||
|
||||
run: ## Start the development server on port 8777
|
||||
@mkdir -p build/logs
|
||||
@if lsof -Pi :$(PORT) -sTCP:LISTEN -t >/dev/null 2>&1; then \
|
||||
echo "Error: Port $(PORT) is already in use"; \
|
||||
echo "Run: lsof -i :$(PORT) to see what's using it"; \
|
||||
exit 1; \
|
||||
fi
|
||||
$(UVICORN) src.main:app --reload --host $(HOST) --port $(PORT) 2>&1 | tee build/logs/server.log
|
||||
|
||||
test: ## Run unit tests (no external services needed)
|
||||
$(PYTEST) --ignore=tests/e2e --ignore=tests/integration
|
||||
|
||||
test-unit: test ## Alias for test
|
||||
|
||||
test-integration: ## Run integration tests (needs Claude/Ollama)
|
||||
$(PYTEST) tests/agents/test_tatlock_agent.py -v
|
||||
|
||||
lint: ## Run ruff linter and formatter check
|
||||
$(RUFF) check src tests
|
||||
$(RUFF) format --check src tests
|
||||
|
||||
typecheck: ## Run mypy type checking
|
||||
$(MYPY) src
|
||||
|
||||
clean: ## Remove build artifacts, caches, and coverage reports
|
||||
rm -rf .cache build
|
||||
find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
||||
@@ -1,6 +1,6 @@
|
||||
# Tatlock - Your Homelab Butler
|
||||
|
||||
> **📖 For the complete system vision and architectural philosophy, see [PHILOSOPHY.md](PHILOSOPHY.md)**
|
||||
> **📖 For the complete system vision and architectural philosophy, see [docs/philosophy.md](docs/philosophy.md)**
|
||||
|
||||
A privacy-first, offline-capable personal assistant system that coordinates specialized AI agents to help with research, development, home automation, and daily organization.
|
||||
|
||||
@@ -89,12 +89,8 @@ A privacy-first, offline-capable personal assistant system that coordinates spec
|
||||
git clone https://git.schweitz.net/jpmschweitzer/tatlock.git
|
||||
cd tatlock
|
||||
|
||||
# Create virtual environment
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
||||
|
||||
# Install dependencies
|
||||
pip install -r requirements.txt
|
||||
make setup
|
||||
```
|
||||
|
||||
### Run the Server
|
||||
@@ -396,8 +392,7 @@ tatlock/
|
||||
│ │ └── multi_tenancy.py # User isolation utilities
|
||||
│ └── main.py # Application entry point
|
||||
├── tests/ # Comprehensive test suite
|
||||
├── PHILOSOPHY.md # System vision and architecture
|
||||
├── IMPLEMENTATION_ROADMAP.md # Development phases
|
||||
├── docs/ # Project documentation
|
||||
├── CHANGELOG.md # Version history
|
||||
└── README.md # This file
|
||||
```
|
||||
@@ -416,8 +411,8 @@ For LLM agent development guidelines and architectural decisions, see [AGENTS.md
|
||||
|
||||
## Documentation
|
||||
|
||||
- **System Philosophy**: [PHILOSOPHY.md](PHILOSOPHY.md) - Vision, goals, and architectural patterns
|
||||
- **User Guide**: This file - Installation, usage, and examples
|
||||
- **System Philosophy**: [docs/philosophy.md](docs/philosophy.md) - Vision, goals, and architectural patterns
|
||||
- **Development Roadmap**: [docs/roadmap.md](docs/roadmap.md) - Open work and planned phases
|
||||
- **Developer Guidelines**: [AGENTS.md](AGENTS.md) - LLM agent development patterns
|
||||
- **Version History**: [CHANGELOG.md](CHANGELOG.md) - Changes and releases
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
# Claude Integration Plan
|
||||
|
||||
## Overview
|
||||
|
||||
Tatlock uses a bidirectional Claude architecture:
|
||||
- **Scenario A**: Tatlock powered by Claude backend (with Ollama fallback) — **COMPLETE**
|
||||
- **Scenario B**: Tatlock exposed as MCP server for external Claude instances — **OPEN**
|
||||
- **Scenario C**: Offline operation via Ollama — **COMPLETE**
|
||||
|
||||
---
|
||||
|
||||
## MCP Server (Expose Tools to Claude) — NOT STARTED
|
||||
|
||||
Create an MCP server that exposes Tatlock's household tools to external Claude instances.
|
||||
|
||||
### New Files
|
||||
|
||||
```
|
||||
src/mcp/
|
||||
├── __init__.py
|
||||
├── server.py # MCP server using mcp Python SDK
|
||||
├── tool_adapters.py # Convert PydanticAI tools → MCP schemas
|
||||
├── auth.py # API key authentication
|
||||
└── transport.py # Streamable HTTP transport
|
||||
```
|
||||
|
||||
### Docker Stack Addition
|
||||
|
||||
```yaml
|
||||
tatlock-mcp:
|
||||
image: git.schweitz.internal/jpmschweitzer/tatlock:latest
|
||||
command: ["python", "-m", "src.mcp.server"]
|
||||
ports:
|
||||
- "8002:8002"
|
||||
environment:
|
||||
- MCP_AUTH_TOKEN=${MCP_AUTH_TOKEN}
|
||||
networks:
|
||||
- docker-dataplane
|
||||
```
|
||||
|
||||
### Claude Desktop Configuration
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"tatlock": {
|
||||
"command": "npx",
|
||||
"args": ["mcp-remote", "https://mcp.schweitz.net/sse", "--header", "Authorization: Bearer ${MCP_AUTH_TOKEN}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Checklist
|
||||
|
||||
- [ ] Create `src/mcp/` module
|
||||
- [ ] Tool adapters (PydanticAI → MCP schema)
|
||||
- [ ] Authentication middleware
|
||||
- [ ] Streamable HTTP transport
|
||||
- [ ] Docker stack configuration
|
||||
|
||||
---
|
||||
|
||||
## Future Phases
|
||||
|
||||
- **LiteLLM Gateway** — Unified endpoint for all models, config-driven routing
|
||||
- **Multi-Provider** — Add OpenAI, Vertex AI, etc.
|
||||
- **Smart Routing** — Context-aware model selection, cost ceiling enforcement
|
||||
|
||||
---
|
||||
|
||||
## Offline Behavior
|
||||
|
||||
| Scenario | Behavior |
|
||||
|----------|----------|
|
||||
| No API key | Use Ollama exclusively |
|
||||
| API unreachable | Use Ollama, log warning |
|
||||
| API rate limited | Fallback to Ollama |
|
||||
|
||||
| Aspect | Claude | Ollama |
|
||||
|--------|--------|--------|
|
||||
| Context | 200k tokens | ~8k tokens |
|
||||
| Latency | 1-3s (network) | 0.5-1s (local) |
|
||||
| Personality | Preserved | Preserved |
|
||||
| Tools | All work | All work |
|
||||
| Cost | API charges | Free |
|
||||
|
||||
---
|
||||
|
||||
## Related Repo Handovers
|
||||
|
||||
Handover documents created in each repo: `PROJECT_CLAUDIFICATION_HANDOVER.md`
|
||||
|
||||
### Open Items
|
||||
|
||||
- **library-desk**: Review HybridRAG response size limits, smart_create endpoint, response formats
|
||||
- **core-api**: Review list_devices response format, error messages, rate limiting
|
||||
- **portainer-core**: Update stack with new env vars, configure secrets, update CONTAINERS.md
|
||||
- **webber**: Review content truncation limits, extraction quality
|
||||
- **tatlock-ui**: Test streaming with Claude backend, conversation history, tool call display
|
||||
@@ -0,0 +1,246 @@
|
||||
# Housekeeper Agent Optimization Findings
|
||||
|
||||
## Background
|
||||
|
||||
Research with Gemini identified key issues with mistral-nemo and tool calling:
|
||||
- "Pre-computation Hallucination" - model answers before using tools
|
||||
- High default temperature (0.7-0.8) causes wandering
|
||||
- Model is "chatty and confident" - needs explicit constraints
|
||||
|
||||
## Key Recommendations from Gemini Research
|
||||
|
||||
1. **Temperature 0.0** for tool-calling agents (deterministic, follows schema)
|
||||
2. **Chain of Thought (CoT)** - force step-by-step reasoning
|
||||
3. **Negative constraints** - tell model what NOT to do (Nemo responds better)
|
||||
4. **Explicit tool descriptions** - verbose docstrings with "never estimate yourself"
|
||||
5. **"Strictly tool-based assistant"** pattern - NO internal knowledge claim
|
||||
|
||||
---
|
||||
|
||||
## Experiment Log
|
||||
|
||||
### Baseline (v1.8.6)
|
||||
- **Date**: 2025-12-17
|
||||
- **Configuration**: Default temperature, improved prompt requiring list_devices first
|
||||
- **Results**:
|
||||
- Called list_devices first ✓
|
||||
- Still hallucinated `light.study_desk` despite seeing list with only `light.study` and `light.study_main`
|
||||
- Partial success: turned off `light.study_main`, failed on hallucinated entity
|
||||
- **Success rate**: ~50% (1 of 2 study lights controlled correctly)
|
||||
|
||||
---
|
||||
|
||||
### Experiment 1: Temperature 0.0
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Set `model_settings=ModelSettings(temperature=0.0)` for Housekeeper
|
||||
- **Hypothesis**: Deterministic output will force model to use exact entity IDs from tool results
|
||||
- **Results**:
|
||||
|
||||
**Study lights test:**
|
||||
- Called `list_devices()` first ✓ (but no domain filter)
|
||||
- Used wrong parameter `device_id` instead of `entity_id` (recovered after validation error)
|
||||
- Only identified `light.studeerlamp` as "study" related (Dutch name)
|
||||
- **Missed `light.study` and `light.study_main`** - didn't match English "study"
|
||||
- Turned off 1 wrong light, missed 2 actual study lights
|
||||
|
||||
**Kitchen lights test:**
|
||||
- Called `list_devices()` first ✓ (no domain filter)
|
||||
- Saw full device list including `light.kitchen`
|
||||
- Used wrong parameter `device_id` instead of `entity_id` (recovered after validation)
|
||||
- After correction, dropped domain prefix: used `kitchen` instead of `light.kitchen`
|
||||
- 404 error - device not found
|
||||
|
||||
- **Success rate**: 0% (no target lights successfully controlled)
|
||||
- **Observations**:
|
||||
- Temperature 0.0 alone is insufficient
|
||||
- Model consistently confuses `device_id` vs `entity_id` parameter name
|
||||
- After validation error correction, model truncates entity_id (drops domain prefix)
|
||||
- Semantic matching of room names to devices is weak
|
||||
- Model doesn't understand entity_id format: `domain.name`
|
||||
|
||||
---
|
||||
|
||||
### Experiment 2: Negative Constraints + CoT
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Complete prompt rewrite with:
|
||||
- "You have NO Internal Knowledge" - negative framing
|
||||
- Explicit entity_id format with WRONG/RIGHT examples
|
||||
- Step-by-step process (ALWAYS FOLLOW)
|
||||
- Explicit parameter names section
|
||||
- "What NOT To Do" negative constraints
|
||||
- **Hypothesis**: Negative constraints work better with Mistral-Nemo
|
||||
- **Results**:
|
||||
|
||||
**Study lights test:**
|
||||
- Called `list_devices(domain="light")` ✓ with domain filter (improvement!)
|
||||
- Still used `device_id` first, recovered to `entity_id` after validation error
|
||||
- After recovery, used correct full format: `light.studeerlamp`
|
||||
- **Still only matched `studeerlamp` not `light.study` or `light.study_main`**
|
||||
|
||||
**Kitchen lights test:**
|
||||
- Called `list_devices(domain="light")` ✓
|
||||
- Called `turn_off(entity_id="light.kitchen")` ✓ correct format!
|
||||
- All 4 kitchen lights turned off (light.kitchen is a group)
|
||||
- **100% success for kitchen!**
|
||||
|
||||
- **Success rate**:
|
||||
- Study: 0% (wrong semantic match)
|
||||
- Kitchen: 100% (4/4 lights off)
|
||||
- Combined: ~50% (1 of 2 tests successful)
|
||||
- **Observations**:
|
||||
- Domain filter now consistently used ✓
|
||||
- Entity_id format correct after recovery ✓
|
||||
- Semantic matching still fails for "study" → prefers Dutch "studeerlamp" over English "study"
|
||||
- Parameter name confusion persists (`device_id` vs `entity_id`)
|
||||
- Simple room names (kitchen) work; mixed language fails (study/studeerlamp)
|
||||
|
||||
---
|
||||
|
||||
### Experiment 3: Temperature 0.1 + Explicit Tool Docstrings
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**:
|
||||
- Temperature 0.1
|
||||
- Updated turn_on/turn_off docstrings with explicit `entity_id=` in examples
|
||||
- **Results**:
|
||||
- Still uses `device_id` first, recovers to `entity_id` after validation
|
||||
- Still picks wrong entity (studeerlamp over study)
|
||||
- **Success rate**: 0%
|
||||
|
||||
---
|
||||
|
||||
### Experiment 4: Room Group Priority (with explicit examples)
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Updated prompt with:
|
||||
- Explicit instruction: "Look for EXACT match `light.<room_name>` first!"
|
||||
- Concrete examples: "For 'study lights' → look for `light.study`"
|
||||
- Working example showing `turn_off(entity_id="light.study")`
|
||||
- **Hypothesis**: Explicit examples will guide model to use room groups
|
||||
- **Results**:
|
||||
|
||||
**Test 1 & 2 (consecutive):**
|
||||
- Called `list_devices(domain="light")` ✓
|
||||
- Device list clearly shows `light.study` at the bottom
|
||||
- First call: `turn_off({"devices":["studeerlamp"]})` - wrong param AND wrong device
|
||||
- After validation error: `turn_off(entity_id="light.studeerlamp")` - correct param, still wrong device
|
||||
- **Completely ignored `light.study` despite prompt explicitly saying to use it**
|
||||
|
||||
- **Success rate**: 0% (wrong device controlled)
|
||||
- **Observations**:
|
||||
- Model ignores explicit step-by-step instructions in favor of substring matching
|
||||
- Dutch "studeerlamp" contains "studer" which the model prefers over exact "study" match
|
||||
- Even when prompt has a literal example `turn_off(entity_id="light.study")`, model uses `light.studeerlamp`
|
||||
- Positional bias possible - `light.study` appears at end of 21-item list
|
||||
- **Fundamental limitation**: Mistral-Nemo cannot follow explicit matching rules
|
||||
|
||||
---
|
||||
|
||||
### Experiment 5: Room Groups First (Tool Output Ordering)
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Modified `list_devices` to sort room groups to top of list using HA attributes (`is_hue_group`, `hue_type="room"`)
|
||||
- **Hypothesis**: Positional bias - model focuses on items earlier in list
|
||||
- **Results**:
|
||||
- Room groups (`light.study`, `light.kitchen`, etc.) now appear first in device list
|
||||
- Combined with improved prompt, model now consistently uses room groups
|
||||
- **70% success rate** (7/10 tests) with default q4 quantization
|
||||
|
||||
---
|
||||
|
||||
### Experiment 6: Model Quantization (q5_1)
|
||||
- **Date**: 2025-12-18
|
||||
- **Change**: Upgraded from default Mistral-Nemo quantization (q4) to `mistral-nemo:12b-instruct-2407-q5_1`
|
||||
- **Hypothesis**: Higher precision weights improve tool calling accuracy
|
||||
- **Results**:
|
||||
|
||||
| Test | Action | Result |
|
||||
|------|--------|--------|
|
||||
| 1 | Turn off study | PASS |
|
||||
| 2 | Turn on study | PASS |
|
||||
| 3 | Toggle study | PASS |
|
||||
| 4 | Turn off kitchen | PASS |
|
||||
| 5 | Turn on kitchen | PASS |
|
||||
| 6 | Toggle kitchen | PASS |
|
||||
| 7 | Turn off bedroom | PASS |
|
||||
| 8 | Turn on bedroom | PASS |
|
||||
| 9 | Turn off living room | PASS |
|
||||
| 10 | Turn on living room | PASS |
|
||||
|
||||
- **Success rate**: **100%** (10/10 tests)
|
||||
- **Observations**:
|
||||
- q5_1 quantization dramatically improves tool calling accuracy
|
||||
- All room groups correctly identified and used
|
||||
- No parameter confusion (`entity_id` used correctly)
|
||||
- No entity_id truncation issues
|
||||
- Toggle operations now work reliably
|
||||
- Model fits within 10GB VRAM (q6 did not)
|
||||
|
||||
---
|
||||
|
||||
### Experiment 7: Device List in System Prompt (Context Injection)
|
||||
- **Date**: [PENDING]
|
||||
- **Change**: Store device list in database (per user/household) and inject into system prompt
|
||||
- **Approach**:
|
||||
1. Periodically sync device list from Home Assistant to PostgreSQL
|
||||
2. On each Housekeeper invocation, fetch device list and include in prompt
|
||||
3. Remove need for model to call list_devices() - just match from context
|
||||
- **Hypothesis**:
|
||||
- Eliminates tool call step where errors occur
|
||||
- Reduces context size by not returning full device list as tool output
|
||||
- Makes entity matching a language task (in prompt) rather than tool result parsing
|
||||
- **Trade-offs**:
|
||||
- Stale data if sync is infrequent
|
||||
- Prompt size increase (but less than tool call response)
|
||||
- Need sync mechanism and storage
|
||||
- **Results**: [TO BE RECORDED]
|
||||
- **Success rate**: [TO BE RECORDED]
|
||||
|
||||
---
|
||||
|
||||
## Key Problem Identified (Solved)
|
||||
|
||||
The model struggled with:
|
||||
1. **Parameter schema adherence** - uses `device_id` when schema requires `entity_id`
|
||||
2. **Value preservation** - truncates values after validation errors (drops `light.` prefix)
|
||||
3. **Semantic matching** - prefers substring matches ("studeerlamp" contains "studer") over exact matches (`light.study`)
|
||||
4. **Following explicit instructions** - ignores step-by-step processes even when examples are provided
|
||||
5. **Positional bias** - may not "see" items at the end of long lists
|
||||
|
||||
**Solution**: These issues were resolved by:
|
||||
1. Using q5_1 quantization instead of default q4 (higher precision weights)
|
||||
2. Sorting room groups to top of device list (address positional bias)
|
||||
3. Explicit prompt guidance with negative constraints and examples
|
||||
|
||||
---
|
||||
|
||||
## Potential Next Experiments
|
||||
|
||||
### Experiment 5: Room Groups First (List Ordering)
|
||||
- **Hypothesis**: Positional bias - model focuses on items earlier in list
|
||||
- **Change**: Sort device list to put room groups (entities matching `light.<single_word>`) at the TOP
|
||||
- **Effort**: Low - modify list_devices output formatting
|
||||
- **Risk**: May affect other use cases where individual devices are needed
|
||||
|
||||
### Experiment 6: Simplified Device List Format
|
||||
- **Hypothesis**: Markdown formatting adds noise that confuses the model
|
||||
- **Change**: Return simple list: `light.study (Study - GROUP), light.study_main (Ceiling light), ...`
|
||||
- **Effort**: Low - modify list_devices output
|
||||
- **Risk**: Less human-readable responses
|
||||
|
||||
---
|
||||
|
||||
## Learnings to Apply Elsewhere
|
||||
|
||||
1. **Quantization matters** - q5_1 dramatically outperforms q4 for tool calling (100% vs 70%)
|
||||
2. **Positional bias is real** - sort important items to top of lists
|
||||
3. **Smaller models need simpler workflows** - fewer tool calls, more context injection
|
||||
4. **Validation errors don't teach** - model often makes worse mistakes on retry
|
||||
5. **Entity IDs are hard** - domain.name format confuses the model
|
||||
6. **Consider pre-computation** - move matching logic to code, not LLM
|
||||
7. **Use explicit negative constraints** - "NEVER do X" works better than "always do Y"
|
||||
|
||||
---
|
||||
|
||||
## Notes
|
||||
|
||||
- Librarian may need higher temperature for creative synthesis
|
||||
- All "action" agents (Housekeeper, future agents) should use low temperature
|
||||
- Consider testing with Gemma 2 9B for better function calling (Google, open weights)
|
||||
@@ -1,317 +0,0 @@
|
||||
# Home Automation API Interface Specification
|
||||
|
||||
## Purpose
|
||||
|
||||
This document specifies the expected endpoints for a home automation abstraction layer in core-api. These endpoints will be consumed by the Tatlock Housekeeper agent and potentially other projects (scheduler, dashboards).
|
||||
|
||||
The goal is to provide a simplified, domain-specific interface for home automation that abstracts away the underlying platform (initially Home Assistant, but swappable).
|
||||
|
||||
---
|
||||
|
||||
## Endpoints
|
||||
|
||||
### Device Discovery
|
||||
|
||||
#### `GET /housekeeping/devices`
|
||||
|
||||
List available devices.
|
||||
|
||||
**Query Parameters:**
|
||||
- `domain` (optional): Filter by device type (e.g., `light`, `switch`, `climate`, `media_player`)
|
||||
- `area` (optional): Filter by area/room name
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"devices": [
|
||||
{
|
||||
"entity_id": "light.living_room",
|
||||
"name": "Living Room Light",
|
||||
"domain": "light",
|
||||
"area": "Living Room",
|
||||
"state": "on",
|
||||
"attributes": {
|
||||
"brightness": 255,
|
||||
"color_temp": 370
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `GET /housekeeping/devices/{entity_id}`
|
||||
|
||||
Get detailed state of a specific device.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"entity_id": "light.living_room",
|
||||
"name": "Living Room Light",
|
||||
"domain": "light",
|
||||
"area": "Living Room",
|
||||
"state": "on",
|
||||
"attributes": {
|
||||
"brightness": 255,
|
||||
"color_temp": 370,
|
||||
"supported_features": ["brightness", "color_temp"]
|
||||
},
|
||||
"last_changed": "2025-12-16T10:30:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `GET /housekeeping/areas`
|
||||
|
||||
List all areas/rooms.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"areas": [
|
||||
{"id": "living_room", "name": "Living Room"},
|
||||
{"id": "bedroom", "name": "Bedroom"},
|
||||
{"id": "kitchen", "name": "Kitchen"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Device Control
|
||||
|
||||
#### `POST /housekeeping/devices/{entity_id}/control`
|
||||
|
||||
Control a device (turn on, turn off, toggle, or set attributes).
|
||||
|
||||
**Request Body:**
|
||||
```json
|
||||
{
|
||||
"action": "turn_on",
|
||||
"brightness": 128,
|
||||
"color_temp": 400
|
||||
}
|
||||
```
|
||||
|
||||
- `action` (required): One of `turn_on`, `turn_off`, `toggle`
|
||||
- Additional attributes vary by device type (brightness, color_temp, rgb_color, etc.)
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"entity_id": "light.living_room",
|
||||
"new_state": "on",
|
||||
"message": "Light turned on"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Scenes
|
||||
|
||||
#### `GET /housekeeping/scenes`
|
||||
|
||||
List available scenes.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"scenes": [
|
||||
{"id": "scene.movie_night", "name": "Movie Night"},
|
||||
{"id": "scene.good_morning", "name": "Good Morning"},
|
||||
{"id": "scene.all_off", "name": "All Off"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `POST /housekeeping/scenes/{scene_id}/activate`
|
||||
|
||||
Activate a scene.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"scene_id": "scene.movie_night",
|
||||
"message": "Scene activated"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Scripts
|
||||
|
||||
#### `GET /housekeeping/scripts`
|
||||
|
||||
List available scripts/sequences.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"scripts": [
|
||||
{"id": "script.bedtime_routine", "name": "Bedtime Routine"},
|
||||
{"id": "script.welcome_home", "name": "Welcome Home"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `POST /housekeeping/scripts/{script_id}/run`
|
||||
|
||||
Execute a script with optional variables.
|
||||
|
||||
**Request Body (optional):**
|
||||
```json
|
||||
{
|
||||
"variables": {
|
||||
"brightness_level": 50,
|
||||
"target_room": "bedroom"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"script_id": "script.bedtime_routine",
|
||||
"message": "Script executed"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Automations
|
||||
|
||||
#### `GET /housekeeping/automations`
|
||||
|
||||
List automations and their enabled/disabled status.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"automations": [
|
||||
{
|
||||
"id": "automation.motion_lights",
|
||||
"name": "Motion Lights",
|
||||
"enabled": true
|
||||
},
|
||||
{
|
||||
"id": "automation.night_mode",
|
||||
"name": "Night Mode",
|
||||
"enabled": false
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `POST /housekeeping/automations/{automation_id}/toggle`
|
||||
|
||||
Enable or disable an automation.
|
||||
|
||||
**Request Body:**
|
||||
```json
|
||||
{
|
||||
"enabled": true
|
||||
}
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"automation_id": "automation.motion_lights",
|
||||
"enabled": true,
|
||||
"message": "Automation enabled"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Utility
|
||||
|
||||
#### `GET /housekeeping/history`
|
||||
|
||||
Get state history for a device.
|
||||
|
||||
**Query Parameters:**
|
||||
- `entity_id` (required): Device to get history for
|
||||
- `hours` (optional, default 24): Hours of history to retrieve
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"entity_id": "light.living_room",
|
||||
"history": [
|
||||
{
|
||||
"state": "on",
|
||||
"timestamp": "2025-12-16T10:30:00Z",
|
||||
"attributes": {"brightness": 255}
|
||||
},
|
||||
{
|
||||
"state": "off",
|
||||
"timestamp": "2025-12-16T08:00:00Z",
|
||||
"attributes": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `GET /housekeeping/health`
|
||||
|
||||
Health check for home automation connection.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"status": "healthy",
|
||||
"connected": true,
|
||||
"platform": "home_assistant",
|
||||
"version": "2024.12.0"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Error Responses
|
||||
|
||||
All endpoints should return consistent error responses:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": true,
|
||||
"code": "DEVICE_NOT_FOUND",
|
||||
"message": "Device light.nonexistent not found"
|
||||
}
|
||||
```
|
||||
|
||||
Common error codes:
|
||||
- `DEVICE_NOT_FOUND` - Entity ID doesn't exist
|
||||
- `INVALID_ACTION` - Unsupported action for device type
|
||||
- `CONNECTION_ERROR` - Cannot reach home automation platform
|
||||
- `UNAUTHORIZED` - Invalid or missing credentials
|
||||
|
||||
---
|
||||
|
||||
## Authentication
|
||||
|
||||
All endpoints require authentication via Bearer token in the `Authorization` header.
|
||||
|
||||
---
|
||||
|
||||
## Consuming Client
|
||||
|
||||
The Tatlock project has an existing client (`src/agents/housekeeper/client.py`) that expects these endpoints. No changes to Tatlock are needed once these endpoints are available.
|
||||
|
||||
Reference: `CoreAPIClient` class in Tatlock expects these exact endpoint patterns.
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
# Tatlock Implementation Roadmap
|
||||
|
||||
> **Reference**: See [philosophy.md](philosophy.md) for the target architecture and vision
|
||||
|
||||
This document tracks open/planned work. Completed phases have been removed.
|
||||
|
||||
## Current State (v2.0.5)
|
||||
|
||||
**What we have**:
|
||||
- OpenAI-compatible API (Responses API + Chat Completions)
|
||||
- Two-tier architecture (Steward → Tatlock)
|
||||
- Household staff: Tatlock (Butler), Steward, Librarian, Biographer
|
||||
- Core tools: Calculator, Date/Time, Web search (SearXNG)
|
||||
- Memory system: Qdrant (vector), Redis (session cache), multi-tenancy via ContextVar
|
||||
- Dual backend: Claude (preferred) + Ollama (fallback)
|
||||
- 439 tests with good coverage
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Expert Household Staff — Remaining Agents
|
||||
|
||||
**Goal**: Implement remaining domain-specific expert agents
|
||||
|
||||
### Planned Agents
|
||||
|
||||
1. **The Developer** (Software Development)
|
||||
- Code generation assistance
|
||||
- Debugging support
|
||||
- Documentation generation
|
||||
- Architecture guidance
|
||||
|
||||
2. **The Handyman** (System Maintenance)
|
||||
- System status queries
|
||||
- Log analysis
|
||||
- Basic troubleshooting
|
||||
- Infrastructure monitoring
|
||||
|
||||
3. **The Secretary** (Scheduling & Organization)
|
||||
- Calendar integration
|
||||
- Task management
|
||||
- Reminder system
|
||||
- Schedule conflict detection
|
||||
|
||||
4. **The Housekeeper** (Home Automation)
|
||||
- Home Assistant integration
|
||||
- Device control interface
|
||||
- Status queries
|
||||
- Automation triggers
|
||||
|
||||
### Each Agent Includes
|
||||
- Specialized prompt and personality
|
||||
- Domain-specific tools
|
||||
- MCP integration points (where applicable)
|
||||
- Integration with Butler orchestration
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Each agent implemented as separate module
|
||||
- [ ] Agents callable via tool framework
|
||||
- [ ] Can invoke specialized models (e.g., Codestral for Developer)
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: Persistence Layer — Database & Multi-Tenancy
|
||||
|
||||
**Goal**: Add persistent storage and multi-user support
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **PostgreSQL Integration**
|
||||
- Docker compose configuration
|
||||
- Database schema with tenant isolation
|
||||
- Alembic migrations
|
||||
- SQLAlchemy models
|
||||
|
||||
2. **Multi-Tenant Architecture**
|
||||
- Tenant identification middleware
|
||||
- Tenant-scoped database sessions
|
||||
- User authentication system
|
||||
- Per-tenant data isolation
|
||||
|
||||
3. **Core Data Models**
|
||||
- Users and tenants
|
||||
- Conversations and messages (migrate from in-memory)
|
||||
- Agent interactions log
|
||||
- System configuration and preferences
|
||||
|
||||
### Success Criteria
|
||||
- [ ] PostgreSQL container running
|
||||
- [ ] Multiple users authenticate separately
|
||||
- [ ] Each user sees only their own data
|
||||
- [ ] Conversations persist across restarts
|
||||
- [ ] Database migrations work correctly
|
||||
|
||||
---
|
||||
|
||||
## Phase 7: MCP (Model Context Protocol) Integration
|
||||
|
||||
**Goal**: Enable rich tool integrations via MCP
|
||||
|
||||
See also [claude-integration.md](claude-integration.md) for MCP server implementation details.
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **MCP Server Framework**
|
||||
- MCP server implementation
|
||||
- Tool registration via MCP
|
||||
- Schema validation
|
||||
- Error handling
|
||||
|
||||
2. **MCP Client in Agents**
|
||||
- PydanticAI MCP integration
|
||||
- Tool discovery from MCP servers
|
||||
- Dynamic tool loading
|
||||
|
||||
3. **Initial MCP Tools**
|
||||
- File system operations
|
||||
- Database queries
|
||||
- API integrations
|
||||
- System commands
|
||||
|
||||
### Success Criteria
|
||||
- [ ] MCP server running
|
||||
- [ ] Tools exposed via MCP protocol
|
||||
- [ ] Agents can discover and use MCP tools
|
||||
- [ ] New tools addable without code changes
|
||||
- [ ] MCP tools visible in Steward recommendations
|
||||
|
||||
---
|
||||
|
||||
## Phase 8: Advanced Memory & Context — Remaining Work
|
||||
|
||||
**Goal**: Implement sophisticated context management and personalization
|
||||
|
||||
### Open Deliverables
|
||||
|
||||
1. **Context Management**
|
||||
- Smart context window trimming
|
||||
- Conversation branching
|
||||
- Topic tracking
|
||||
|
||||
2. **Personalization**
|
||||
- User preference learning
|
||||
- Interaction pattern analysis
|
||||
- Adaptive responses
|
||||
- Custom agent personalities per user
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Conversations automatically embedded to Qdrant
|
||||
- [ ] Memory improves over time (learning from interactions)
|
||||
|
||||
---
|
||||
|
||||
## Phase 9: Extended Household Staff
|
||||
|
||||
**Goal**: Add specialized agents for additional domains
|
||||
|
||||
### Future Agents
|
||||
- **The Accountant** — Expense tracking, budgets, financial reports
|
||||
- **The Chef** — Meal planning, recipes, nutrition tracking
|
||||
- Others as needs emerge
|
||||
|
||||
---
|
||||
|
||||
## Phase 10: User Experience Refinement
|
||||
|
||||
**Goal**: Polish the interaction experience
|
||||
|
||||
- Personality tuning and consistency
|
||||
- Better progress indicators
|
||||
- Response time improvements
|
||||
- Streaming smoothness
|
||||
|
||||
---
|
||||
|
||||
## Phase 11: Production Hardening
|
||||
|
||||
**Goal**: Make the system production-ready for homelab deployment
|
||||
|
||||
- Complete docker-compose stack
|
||||
- Health checks and monitoring
|
||||
- Authentication hardening and rate limiting
|
||||
- Installation and troubleshooting documentation
|
||||
|
||||
---
|
||||
|
||||
## Dependencies
|
||||
|
||||
```
|
||||
Phase 4 (Remaining Agents)
|
||||
↓
|
||||
Phase 5 (Database/Multi-Tenancy) ← Can be deferred
|
||||
↓
|
||||
Phase 7 (MCP) → Phase 8 (Advanced Memory)
|
||||
↓
|
||||
Phase 9 (Extended Staff) → Phase 10 (UX) → Phase 11 (Production)
|
||||
```
|
||||
|
||||
**Can Be Deferred**: Phase 5 until you need persistence
|
||||
**Parallel Opportunities**: Phases 7 and 8 can overlap; 9 and 10 ongoing
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Implement The Developer agent for code assistance
|
||||
2. Add Home Assistant integration for The Housekeeper
|
||||
3. Integrate scheduling service for The Secretary
|
||||
4. MCP server for external Claude access
|
||||
File diff suppressed because it is too large
Load Diff
+58
-3
@@ -4,17 +4,67 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "tatlock"
|
||||
version = "1.8.6"
|
||||
version = "2.2.0"
|
||||
description = "OpenAI-compatible API with Ollama backend"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = []
|
||||
dependencies = [
|
||||
"fastapi>=0.123,<0.124",
|
||||
"uvicorn[standard]>=0.38,<0.39",
|
||||
"pydantic>=2.11,<2.13",
|
||||
"pydantic-settings>=2.12,<2.13",
|
||||
"pydantic-ai-slim[openai,anthropic]>=1.27,<1.28",
|
||||
"httpx>=0.28,<0.29",
|
||||
"sse-starlette>=3.0,<3.1",
|
||||
"python-dotenv>=1.2,<1.3",
|
||||
"starlette>=0.45,<0.46",
|
||||
"redis[hiredis]>=5.2,<6.0",
|
||||
"qdrant-client>=1.12,<2.0",
|
||||
"structlog>=24.1,<25.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.3,<8.4",
|
||||
"pytest-asyncio>=0.25,<0.26",
|
||||
"pytest-cov>=6.0,<6.1",
|
||||
"pytest-mock>=3.14,<3.15",
|
||||
"ruff>=0.8,<0.9",
|
||||
"mypy>=1.14,<1.15",
|
||||
"faker>=34.0,<35.0",
|
||||
"coverage[toml]>=7.7,<7.8",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
python_classes = ["Test*"]
|
||||
python_functions = ["test_*"]
|
||||
asyncio_mode = "auto"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
cache_dir = ".cache/pytest"
|
||||
markers = [
|
||||
"unit: Unit tests",
|
||||
"integration: Integration tests",
|
||||
"slow: Slow running tests",
|
||||
]
|
||||
addopts = [
|
||||
"--verbose",
|
||||
"--strict-markers",
|
||||
"--tb=short",
|
||||
"--cov=src",
|
||||
"--cov-report=term-missing",
|
||||
"--cov-report=html:build/coverage/html",
|
||||
"--cov-report=xml:build/coverage/coverage.xml",
|
||||
"--cov-branch",
|
||||
]
|
||||
filterwarnings = [
|
||||
"ignore::DeprecationWarning",
|
||||
]
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["src"]
|
||||
branch = true
|
||||
data_file = "build/coverage/.coverage"
|
||||
omit = [
|
||||
"*/tests/*",
|
||||
"*/__pycache__/*",
|
||||
@@ -37,11 +87,15 @@ exclude_lines = [
|
||||
]
|
||||
|
||||
[tool.coverage.html]
|
||||
directory = "htmlcov"
|
||||
directory = "build/coverage/html"
|
||||
|
||||
[tool.coverage.xml]
|
||||
output = "build/coverage/coverage.xml"
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py312"
|
||||
cache-dir = ".cache/ruff"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
@@ -64,6 +118,7 @@ ignore = [
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.12"
|
||||
cache_dir = ".cache/mypy"
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
disallow_untyped_defs = true
|
||||
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
[pytest]
|
||||
testpaths = tests
|
||||
python_files = test_*.py
|
||||
python_classes = Test*
|
||||
python_functions = test_*
|
||||
asyncio_mode = auto
|
||||
asyncio_default_fixture_loop_scope = function
|
||||
|
||||
# Markers
|
||||
markers =
|
||||
unit: Unit tests
|
||||
integration: Integration tests
|
||||
slow: Slow running tests
|
||||
|
||||
# Coverage options (overridden by pyproject.toml)
|
||||
addopts =
|
||||
--verbose
|
||||
--strict-markers
|
||||
--tb=short
|
||||
--cov=src
|
||||
--cov-report=term-missing
|
||||
--cov-report=html
|
||||
--cov-report=xml
|
||||
--cov-branch
|
||||
|
||||
# Ignore warnings from dependencies
|
||||
filterwarnings =
|
||||
ignore::DeprecationWarning
|
||||
@@ -1,25 +0,0 @@
|
||||
# Development and Testing Dependencies
|
||||
# Install with: pip install -r requirements.txt -r requirements-dev.txt
|
||||
|
||||
# Testing Framework
|
||||
# Latest pytest with async support
|
||||
pytest>=8.3,<8.4
|
||||
pytest-asyncio>=0.25,<0.26
|
||||
pytest-cov>=6.0,<6.1
|
||||
|
||||
# Test client for FastAPI
|
||||
httpx>=0.28,<0.29 # Already in requirements.txt but needed for test client
|
||||
|
||||
# Code Quality
|
||||
# Linting and formatting
|
||||
ruff>=0.8,<0.9
|
||||
|
||||
# Type checking
|
||||
mypy>=1.14,<1.15
|
||||
|
||||
# Testing utilities
|
||||
pytest-mock>=3.14,<3.15
|
||||
faker>=34.0,<35.0
|
||||
|
||||
# Coverage reporting
|
||||
coverage[toml]>=7.7,<7.8
|
||||
@@ -1,61 +0,0 @@
|
||||
# Core FastAPI framework and server
|
||||
# FastAPI: Modern, fast web framework for building APIs
|
||||
# Latest: 0.123.9 (Dec 4, 2025) - No known CVEs
|
||||
fastapi>=0.123,<0.124
|
||||
|
||||
# ASGI server for running FastAPI
|
||||
# Latest: 0.38.0 (Oct 18, 2025) - No known CVEs
|
||||
# Note: Old versions had CVE-2020-7694/7695, but 0.38.0 is secure
|
||||
uvicorn[standard]>=0.38,<0.39
|
||||
|
||||
# Additional dependencies
|
||||
# Pydantic for data validation (comes with pydantic-ai but pinning explicitly)
|
||||
# Updated to >=2.11 due to ag-ui-protocol dependency requirement
|
||||
# Latest: 2.12.4 (Nov 5, 2025) - No known CVEs
|
||||
pydantic>=2.11,<2.13
|
||||
|
||||
# Pydantic settings for configuration management
|
||||
# Required explicitly since pydantic-ai-slim doesn't include it
|
||||
# Latest: 2.12.0 (Dec 2025) - No known CVEs
|
||||
pydantic-settings>=2.12,<2.13
|
||||
|
||||
# AI/LLM integration
|
||||
# PydanticAI: Agent framework for using Pydantic with LLMs
|
||||
# Using slim version with only openai extra (Ollama uses OpenAI-compatible API)
|
||||
# This avoids installing SDKs for anthropic, cohere, google, groq, huggingface, etc.
|
||||
# See DEPENDENCY_SLIM.md for rollback instructions if this breaks
|
||||
pydantic-ai-slim[openai]>=1.27,<1.28
|
||||
|
||||
# HTTP client for Ollama communication
|
||||
# Latest: 0.28.1 - No known CVEs
|
||||
httpx>=0.28,<0.29
|
||||
|
||||
# Server-Sent Events for streaming responses
|
||||
# Required for OpenAI-compatible streaming endpoints
|
||||
# Latest: 3.0.2 (Oct 30, 2025) - No known CVEs
|
||||
sse-starlette>=3.0,<3.1
|
||||
|
||||
# Configuration management
|
||||
# Latest: 1.2.1 (Oct 26, 2025) - No known CVEs
|
||||
python-dotenv>=1.2,<1.3
|
||||
|
||||
# ASGI toolkit (dependency of FastAPI, pinning for security)
|
||||
starlette>=0.45,<0.46
|
||||
|
||||
# Redis for performance benchmarking and caching
|
||||
# Latest: 5.2.1 (Dec 5, 2025) - No known CVEs
|
||||
# hiredis: C parser for better performance
|
||||
redis[hiredis]>=5.2,<6.0
|
||||
|
||||
# Qdrant vector database client for memory storage
|
||||
# Latest: 1.12.1 (Dec 2025) - No known CVEs
|
||||
qdrant-client>=1.12,<2.0
|
||||
|
||||
# Structured logging for observability
|
||||
# Latest: 24.4.0 (Aug 22, 2024) - No known CVEs
|
||||
structlog>=24.1,<25.0
|
||||
|
||||
# Note on version locking strategy:
|
||||
# Using >=X.Y,<X.(Y+1) format to lock to minor versions
|
||||
# This protects against supply chain attacks while allowing patch updates
|
||||
# Update regularly and review changelogs before upgrading minor versions
|
||||
@@ -0,0 +1,542 @@
|
||||
"""
|
||||
Benchmark tool calling across different Ollama models via Tatlock API.
|
||||
|
||||
Sends test prompts through the full Tatlock pipeline (Steward -> Orchestration
|
||||
-> Synthesis) and records tool selection accuracy, latency, and response quality.
|
||||
|
||||
Between models, swaps OLLAMA_DEFAULT_MODEL in .env and waits for uvicorn
|
||||
auto-reload. Requires the server to be running via ./wakeup.sh.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python scripts/benchmark_tool_calling.py
|
||||
.venv/bin/python scripts/benchmark_tool_calling.py --models "gemma4:e4b,gemma4:e2b"
|
||||
.venv/bin/python scripts/benchmark_tool_calling.py --iterations 3
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
API_BASE = "http://localhost:8777"
|
||||
CHAT_URL = f"{API_BASE}/v1/chat/completions"
|
||||
HEALTH_URL = f"{API_BASE}/health"
|
||||
OLLAMA_URL = "http://localhost:11434"
|
||||
ENV_PATH = Path(__file__).parent.parent / ".env"
|
||||
|
||||
DEFAULT_MODELS = ["mistral-nemo-large:latest", "gemma4:e4b", "gemma4:e2b"]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test scenarios
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Scenario:
|
||||
name: str
|
||||
prompt: str
|
||||
expected_tool: str | None # None = no tool expected
|
||||
# Patterns to check in the response text for indirect tool-use evidence
|
||||
success_patterns: list[str] = field(default_factory=list)
|
||||
category: str = "basic"
|
||||
|
||||
|
||||
SCENARIOS = [
|
||||
# --- Should call calculate_math ---
|
||||
Scenario(
|
||||
name="Simple arithmetic",
|
||||
prompt="What is 144 divided by 12?",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["12"],
|
||||
category="calculator",
|
||||
),
|
||||
Scenario(
|
||||
name="Square root",
|
||||
prompt="What's the square root of 256?",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["16"],
|
||||
category="calculator",
|
||||
),
|
||||
Scenario(
|
||||
name="Complex math",
|
||||
prompt="Calculate pi times the square of 5",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["78.5"], # pi * 25 ≈ 78.54
|
||||
category="calculator",
|
||||
),
|
||||
Scenario(
|
||||
name="Word problem",
|
||||
prompt="If I have 3 bags with 17 apples each and I eat 4, how many apples do I have?",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["47"],
|
||||
category="calculator",
|
||||
),
|
||||
|
||||
# --- Should call get_current_time ---
|
||||
Scenario(
|
||||
name="Current date",
|
||||
prompt="What's today's date?",
|
||||
expected_tool="get_current_time",
|
||||
success_patterns=["2026"], # Should contain current year
|
||||
category="datetime",
|
||||
),
|
||||
Scenario(
|
||||
name="Current time",
|
||||
prompt="What time is it right now?",
|
||||
expected_tool="get_current_time",
|
||||
success_patterns=[":"], # Time format contains colons
|
||||
category="datetime",
|
||||
),
|
||||
|
||||
# --- Should call calculate_date_offset ---
|
||||
Scenario(
|
||||
name="Relative date past",
|
||||
prompt="What was the date 2 weeks ago?",
|
||||
expected_tool="calculate_date_offset",
|
||||
success_patterns=["2026"],
|
||||
category="datetime",
|
||||
),
|
||||
|
||||
# --- Should call calculate_time_difference ---
|
||||
Scenario(
|
||||
name="Date difference",
|
||||
prompt="How many days between January 1st 2025 and March 15th 2025?",
|
||||
expected_tool="calculate_time_difference",
|
||||
success_patterns=["73", "74"], # 73 or 74 days
|
||||
category="datetime",
|
||||
),
|
||||
|
||||
# --- Should NOT call any tool ---
|
||||
Scenario(
|
||||
name="Greeting",
|
||||
prompt="Hello! How are you?",
|
||||
expected_tool=None,
|
||||
success_patterns=["sir"], # Butler personality
|
||||
category="no_tool",
|
||||
),
|
||||
Scenario(
|
||||
name="Knowledge question",
|
||||
prompt="What is the capital of France?",
|
||||
expected_tool=None,
|
||||
success_patterns=["Paris"],
|
||||
category="no_tool",
|
||||
),
|
||||
Scenario(
|
||||
name="Opinion request",
|
||||
prompt="What do you think about rainy days?",
|
||||
expected_tool=None,
|
||||
category="no_tool",
|
||||
),
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunResult:
|
||||
scenario: str
|
||||
model: str
|
||||
iteration: int
|
||||
latency: float
|
||||
response_text: str
|
||||
has_correct_answer: bool
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelStats:
|
||||
model: str
|
||||
results: list[RunResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return len(self.results)
|
||||
|
||||
@property
|
||||
def errors(self) -> int:
|
||||
return sum(1 for r in self.results if r.error)
|
||||
|
||||
@property
|
||||
def accuracy(self) -> float:
|
||||
valid = [r for r in self.results if not r.error]
|
||||
if not valid:
|
||||
return 0
|
||||
return sum(1 for r in valid if r.has_correct_answer) / len(valid) * 100
|
||||
|
||||
@property
|
||||
def avg_latency(self) -> float:
|
||||
lats = [r.latency for r in self.results if not r.error]
|
||||
return statistics.mean(lats) if lats else 0
|
||||
|
||||
@property
|
||||
def p95_latency(self) -> float:
|
||||
lats = sorted(r.latency for r in self.results if not r.error)
|
||||
if not lats:
|
||||
return 0
|
||||
return lats[min(int(len(lats) * 0.95), len(lats) - 1)]
|
||||
|
||||
@property
|
||||
def max_latency(self) -> float:
|
||||
lats = [r.latency for r in self.results if not r.error]
|
||||
return max(lats) if lats else 0
|
||||
|
||||
def category_accuracy(self, category: str) -> float:
|
||||
cat_scenarios = {s.name for s in SCENARIOS if s.category == category}
|
||||
valid = [r for r in self.results if not r.error and r.scenario in cat_scenarios]
|
||||
if not valid:
|
||||
return 0
|
||||
return sum(1 for r in valid if r.has_correct_answer) / len(valid) * 100
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# .env manipulation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def swap_model_in_env(model_name: str):
|
||||
"""Swap OLLAMA_DEFAULT_MODEL in .env file."""
|
||||
content = ENV_PATH.read_text()
|
||||
content = re.sub(
|
||||
r'^OLLAMA_DEFAULT_MODEL=.*$',
|
||||
f'OLLAMA_DEFAULT_MODEL={model_name}',
|
||||
content,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
ENV_PATH.write_text(content)
|
||||
print(f" .env updated: OLLAMA_DEFAULT_MODEL={model_name}")
|
||||
|
||||
|
||||
async def wait_for_server_reload(client: httpx.AsyncClient, timeout: float = 30):
|
||||
"""Wait for uvicorn to auto-reload after .env change."""
|
||||
# Give uvicorn a moment to detect the file change
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Poll health endpoint
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
r = await client.get(HEALTH_URL, timeout=5)
|
||||
if r.status_code == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(1)
|
||||
|
||||
raise TimeoutError("Server did not come back after reload")
|
||||
|
||||
|
||||
async def warm_up_ollama_model(client: httpx.AsyncClient, model_name: str):
|
||||
"""Send a throwaway request to load the model into VRAM."""
|
||||
print(f" Warming up {model_name} in Ollama...", end=" ", flush=True)
|
||||
try:
|
||||
r = await client.post(
|
||||
f"{OLLAMA_URL}/api/generate",
|
||||
json={"model": model_name, "prompt": "hi", "stream": False},
|
||||
timeout=120,
|
||||
)
|
||||
r.raise_for_status()
|
||||
duration = r.json().get("total_duration", 0) / 1e9
|
||||
print(f"OK ({duration:.1f}s)")
|
||||
except Exception as e:
|
||||
print(f"WARN: {e}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core benchmark logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def run_scenario(
|
||||
client: httpx.AsyncClient,
|
||||
scenario: Scenario,
|
||||
model: str,
|
||||
iteration: int,
|
||||
) -> RunResult:
|
||||
"""Run a single scenario through the Tatlock API."""
|
||||
payload = {
|
||||
"model": "Tatlock",
|
||||
"messages": [{"role": "user", "content": scenario.prompt}],
|
||||
}
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r = await client.post(CHAT_URL, json=payload, timeout=120)
|
||||
latency = time.monotonic() - start
|
||||
|
||||
if r.status_code != 200:
|
||||
return RunResult(
|
||||
scenario=scenario.name,
|
||||
model=model,
|
||||
iteration=iteration,
|
||||
latency=latency,
|
||||
response_text="",
|
||||
has_correct_answer=False,
|
||||
error=f"HTTP {r.status_code}: {r.text[:100]}",
|
||||
)
|
||||
|
||||
data = r.json()
|
||||
response_text = data["choices"][0]["message"]["content"]
|
||||
|
||||
# Check if the response contains expected patterns
|
||||
has_correct = True
|
||||
if scenario.success_patterns:
|
||||
has_correct = any(
|
||||
p.lower() in response_text.lower()
|
||||
for p in scenario.success_patterns
|
||||
)
|
||||
|
||||
return RunResult(
|
||||
scenario=scenario.name,
|
||||
model=model,
|
||||
iteration=iteration,
|
||||
latency=latency,
|
||||
response_text=response_text,
|
||||
has_correct_answer=has_correct,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
latency = time.monotonic() - start
|
||||
return RunResult(
|
||||
scenario=scenario.name,
|
||||
model=model,
|
||||
iteration=iteration,
|
||||
latency=latency,
|
||||
response_text="",
|
||||
has_correct_answer=False,
|
||||
error=str(e)[:200],
|
||||
)
|
||||
|
||||
|
||||
async def benchmark_model(
|
||||
client: httpx.AsyncClient,
|
||||
model_name: str,
|
||||
iterations: int,
|
||||
) -> ModelStats:
|
||||
"""Run all scenarios for a single model."""
|
||||
stats = ModelStats(model=model_name)
|
||||
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f" Model: {model_name}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
# Swap model in .env
|
||||
swap_model_in_env(model_name)
|
||||
|
||||
# Warm up model in Ollama BEFORE server reload picks it up
|
||||
await warm_up_ollama_model(client, model_name)
|
||||
|
||||
# Wait for server to reload with new model
|
||||
print(" Waiting for server reload...", end=" ", flush=True)
|
||||
await wait_for_server_reload(client)
|
||||
print("OK")
|
||||
|
||||
# Run a throwaway request through the full pipeline to warm up
|
||||
print(" Warming up pipeline...", end=" ", flush=True)
|
||||
try:
|
||||
await client.post(
|
||||
CHAT_URL,
|
||||
json={"model": "Tatlock", "messages": [{"role": "user", "content": "hi"}]},
|
||||
timeout=120,
|
||||
)
|
||||
print("OK")
|
||||
except Exception as e:
|
||||
print(f"WARN: {e}")
|
||||
|
||||
for iteration in range(iterations):
|
||||
if iterations > 1:
|
||||
print(f"\n --- Iteration {iteration + 1}/{iterations} ---")
|
||||
|
||||
for scenario in SCENARIOS:
|
||||
result = await run_scenario(client, scenario, model_name, iteration)
|
||||
stats.results.append(result)
|
||||
|
||||
# Display
|
||||
if result.error:
|
||||
print(
|
||||
f" [ERR ] {scenario.name:30s} {result.latency:5.1f}s "
|
||||
f"{result.error[:60]}"
|
||||
)
|
||||
elif result.has_correct_answer:
|
||||
preview = result.response_text[:60].replace("\n", " ")
|
||||
print(f" [OK ] {scenario.name:30s} {result.latency:5.1f}s {preview}")
|
||||
else:
|
||||
preview = result.response_text[:60].replace("\n", " ")
|
||||
print(f" [MISS] {scenario.name:30s} {result.latency:5.1f}s {preview}")
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def print_comparison(all_stats: list[ModelStats]):
|
||||
"""Print side-by-side comparison table."""
|
||||
print("\n" + "=" * 80)
|
||||
print(" COMPARISON SUMMARY")
|
||||
print("=" * 80)
|
||||
|
||||
col_width = max(len(s.model) for s in all_stats) + 2
|
||||
label_width = 32
|
||||
|
||||
header = f"{'Metric':<{label_width}}"
|
||||
for s in all_stats:
|
||||
header += f" {s.model:>{col_width}}"
|
||||
print(f"\n{header}")
|
||||
print("-" * (label_width + (col_width + 2) * len(all_stats)))
|
||||
|
||||
# Answer accuracy
|
||||
row = f"{'Correct answer rate':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.accuracy:>{col_width - 1}.1f}%"
|
||||
print(row)
|
||||
|
||||
# Latency
|
||||
row = f"{'Avg latency':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.avg_latency:>{col_width - 1}.1f}s"
|
||||
print(row)
|
||||
|
||||
row = f"{'P95 latency':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.p95_latency:>{col_width - 1}.1f}s"
|
||||
print(row)
|
||||
|
||||
row = f"{'Max latency':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.max_latency:>{col_width - 1}.1f}s"
|
||||
print(row)
|
||||
|
||||
# Errors
|
||||
row = f"{'Errors':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.errors:>{col_width}}"
|
||||
print(row)
|
||||
|
||||
# Per-category
|
||||
categories = sorted(set(sc.category for sc in SCENARIOS))
|
||||
print(f"\n{'Per-category accuracy':<{label_width}}")
|
||||
print("-" * (label_width + (col_width + 2) * len(all_stats)))
|
||||
for cat in categories:
|
||||
row = f" {cat:<{label_width - 2}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.category_accuracy(cat):>{col_width - 1}.1f}%"
|
||||
print(row)
|
||||
|
||||
# Mismatches
|
||||
print(f"\n{'Missed answers':<50}")
|
||||
print("-" * 80)
|
||||
any_miss = False
|
||||
for scenario in SCENARIOS:
|
||||
misses = []
|
||||
for s in all_stats:
|
||||
sc_results = [r for r in s.results if r.scenario == scenario.name]
|
||||
fails = [r for r in sc_results if not r.has_correct_answer and not r.error]
|
||||
if fails:
|
||||
preview = fails[0].response_text[:50].replace("\n", " ")
|
||||
misses.append(f"{s.model}: \"{preview}\"")
|
||||
if misses:
|
||||
any_miss = True
|
||||
print(f" {scenario.name}")
|
||||
for m in misses:
|
||||
print(f" {m}")
|
||||
|
||||
if not any_miss:
|
||||
print(" (none)")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
|
||||
|
||||
def save_results(all_stats: list[ModelStats], output_path: Path):
|
||||
"""Save detailed results to JSON."""
|
||||
data = {}
|
||||
for stats in all_stats:
|
||||
data[stats.model] = {
|
||||
"summary": {
|
||||
"accuracy": stats.accuracy,
|
||||
"avg_latency": round(stats.avg_latency, 2),
|
||||
"p95_latency": round(stats.p95_latency, 2),
|
||||
"max_latency": round(stats.max_latency, 2),
|
||||
"errors": stats.errors,
|
||||
"total_runs": stats.total,
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"scenario": r.scenario,
|
||||
"iteration": r.iteration,
|
||||
"latency": round(r.latency, 3),
|
||||
"has_correct_answer": r.has_correct_answer,
|
||||
"response_text": r.response_text,
|
||||
"error": r.error,
|
||||
}
|
||||
for r in stats.results
|
||||
],
|
||||
}
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(data, indent=2))
|
||||
print(f"\nDetailed results saved to: {output_path}")
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description="Benchmark tool calling across Ollama models via Tatlock API")
|
||||
parser.add_argument(
|
||||
"--iterations", type=int, default=1,
|
||||
help="Iterations per model (default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--models", type=str, default=",".join(DEFAULT_MODELS),
|
||||
help=f"Comma-separated models (default: {','.join(DEFAULT_MODELS)})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", type=str, default="logs/benchmark_results.json",
|
||||
help="JSON output path (default: logs/benchmark_results.json)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
models = [m.strip() for m in args.models.split(",")]
|
||||
|
||||
# Verify server is running
|
||||
async with httpx.AsyncClient() as client:
|
||||
try:
|
||||
r = await client.get(HEALTH_URL, timeout=5)
|
||||
r.raise_for_status()
|
||||
print("Server is running.")
|
||||
except Exception:
|
||||
print("ERROR: Server not running. Start it with ./wakeup.sh first.")
|
||||
return
|
||||
|
||||
print("=" * 70)
|
||||
print(" Tool Calling Benchmark (via Tatlock API)")
|
||||
print("=" * 70)
|
||||
print(f" Models: {', '.join(models)}")
|
||||
print(f" Scenarios: {len(SCENARIOS)}")
|
||||
print(f" Iterations: {args.iterations}")
|
||||
print(f" Total runs: {len(SCENARIOS) * args.iterations * len(models)}")
|
||||
|
||||
# Remember original model to restore after benchmark
|
||||
original_env = ENV_PATH.read_text()
|
||||
|
||||
all_stats = []
|
||||
async with httpx.AsyncClient() as client:
|
||||
for model in models:
|
||||
stats = await benchmark_model(client, model, args.iterations)
|
||||
all_stats.append(stats)
|
||||
|
||||
# Restore original .env
|
||||
ENV_PATH.write_text(original_env)
|
||||
print(f"\n .env restored to original")
|
||||
|
||||
print_comparison(all_stats)
|
||||
save_results(all_stats, Path(args.output))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/bin/bash
|
||||
# Housekeeper Room Group Detection Test Suite
|
||||
# Verifies room groups are controlled by checking actual state changes
|
||||
|
||||
API_URL="http://localhost:8777/v1/chat/completions"
|
||||
CORE_API="http://192.168.86.149:8083"
|
||||
RESULTS_FILE="/tmp/housekeeper_test_results.txt"
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
get_state() {
|
||||
curl -s "$CORE_API/housekeeping/devices/$1" 2>/dev/null | jq -r '.state' 2>/dev/null
|
||||
}
|
||||
|
||||
echo "=========================================="
|
||||
echo "Housekeeper Room Group Test Suite"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
> "$RESULTS_FILE"
|
||||
|
||||
run_toggle_test() {
|
||||
local test_num=$1
|
||||
local room=$2
|
||||
local entity="light.$room"
|
||||
local prompt_room="${room//_/ }"
|
||||
|
||||
printf "Test %2d: Toggle %-12s lights ... " "$test_num" "$prompt_room"
|
||||
|
||||
local before=$(get_state "$entity")
|
||||
if [ -z "$before" ] || [ "$before" = "null" ]; then
|
||||
echo -e "${YELLOW}SKIP${NC} (cannot get state)"
|
||||
echo "SKIP|$test_num|Toggle $room|error" >> "$RESULTS_FILE"
|
||||
return
|
||||
fi
|
||||
|
||||
curl -s -X POST "$API_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\": \"tatlock\", \"messages\": [{\"role\": \"user\", \"content\": \"Toggle the $prompt_room lights\"}]}" > /dev/null
|
||||
|
||||
sleep 4
|
||||
|
||||
local after=$(get_state "$entity")
|
||||
|
||||
if [ "$before" != "$after" ]; then
|
||||
echo -e "${GREEN}PASS${NC} ($before -> $after)"
|
||||
echo "PASS|$test_num|Toggle $room|$before->$after" >> "$RESULTS_FILE"
|
||||
else
|
||||
echo -e "${RED}FAIL${NC} (state unchanged: $before)"
|
||||
echo "FAIL|$test_num|Toggle $room|unchanged:$before" >> "$RESULTS_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
run_onoff_test() {
|
||||
local test_num=$1
|
||||
local room=$2
|
||||
local action=$3
|
||||
local expected_state=$4
|
||||
# Entity uses underscore, prompt uses space
|
||||
local entity="light.${room//_/ }"
|
||||
entity="light.$room"
|
||||
local prompt_room="${room//_/ }"
|
||||
|
||||
printf "Test %2d: %-8s %-12s lights ... " "$test_num" "$action" "$prompt_room"
|
||||
|
||||
curl -s -X POST "$API_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\": \"tatlock\", \"messages\": [{\"role\": \"user\", \"content\": \"$action the $prompt_room lights\"}]}" > /dev/null
|
||||
|
||||
sleep 4
|
||||
|
||||
local after=$(get_state "$entity")
|
||||
|
||||
if [ "$after" = "$expected_state" ]; then
|
||||
echo -e "${GREEN}PASS${NC} ($after)"
|
||||
echo "PASS|$test_num|$action $room|$after" >> "$RESULTS_FILE"
|
||||
else
|
||||
echo -e "${RED}FAIL${NC} (got $after, expected $expected_state)"
|
||||
echo "FAIL|$test_num|$action $room|got:$after,expected:$expected_state" >> "$RESULTS_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Running tests (~4s each)..."
|
||||
echo ""
|
||||
|
||||
# Study tests
|
||||
run_onoff_test 1 "study" "Turn off" "off"
|
||||
run_onoff_test 2 "study" "Turn on" "on"
|
||||
run_toggle_test 3 "study"
|
||||
|
||||
# Kitchen tests
|
||||
run_onoff_test 4 "kitchen" "Turn off" "off"
|
||||
run_onoff_test 5 "kitchen" "Turn on" "on"
|
||||
run_toggle_test 6 "kitchen"
|
||||
|
||||
# Bedroom tests
|
||||
run_onoff_test 7 "bedroom" "Turn off" "off"
|
||||
run_onoff_test 8 "bedroom" "Turn on" "on"
|
||||
|
||||
# Living room tests (entity is light.living_room)
|
||||
run_onoff_test 9 "living_room" "Turn off" "off"
|
||||
run_onoff_test 10 "living_room" "Turn on" "on"
|
||||
|
||||
# Ensure all lights end up ON
|
||||
echo ""
|
||||
echo "Restoring all lights to ON..."
|
||||
for room in "study" "kitchen" "bedroom" "living room"; do
|
||||
curl -s -X POST "$API_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\": \"tatlock\", \"messages\": [{\"role\": \"user\", \"content\": \"Turn on the $room lights\"}]}" > /dev/null
|
||||
sleep 3
|
||||
done
|
||||
echo "Done."
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Results"
|
||||
echo "=========================================="
|
||||
|
||||
PASS=$(grep -c "^PASS" "$RESULTS_FILE" 2>/dev/null || echo 0)
|
||||
FAIL=$(grep -c "^FAIL" "$RESULTS_FILE" 2>/dev/null || echo 0)
|
||||
SKIP=$(grep -c "^SKIP" "$RESULTS_FILE" 2>/dev/null || echo 0)
|
||||
TOTAL=$((PASS + FAIL))
|
||||
|
||||
echo "Passed: $PASS"
|
||||
echo "Failed: $FAIL"
|
||||
echo "Skipped: $SKIP"
|
||||
|
||||
if [ "$TOTAL" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "Success Rate: $((PASS * 100 / TOTAL))% ($PASS/$TOTAL)"
|
||||
fi
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "Failures:"
|
||||
grep "^FAIL" "$RESULTS_FILE"
|
||||
fi
|
||||
@@ -102,16 +102,10 @@ _biographer_agent: Optional[Agent[None, str]] = None
|
||||
|
||||
def _create_biographer_agent() -> Agent[None, str]:
|
||||
"""Create The Biographer PydanticAI agent."""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
# Create Ollama model with sanitized provider
|
||||
# (fixes 'content: null' issue with tool calls)
|
||||
model = OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
agent: Agent[None, str] = Agent(
|
||||
model=model,
|
||||
@@ -131,9 +125,12 @@ def _create_biographer_agent() -> Agent[None, str]:
|
||||
# Register management tools
|
||||
agent.tool_plain(forget_memory)
|
||||
|
||||
from src.anthropic.model_selector import get_model_info
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"biographer_agent_created",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
tool_count=6,
|
||||
)
|
||||
|
||||
|
||||
+145
-84
@@ -13,6 +13,7 @@ from enum import Enum
|
||||
from typing import AsyncGenerator, Callable, Optional, Any
|
||||
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import trace_span, SpanType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -239,38 +240,58 @@ async def delegate_to_librarian(
|
||||
has_context=bool(context),
|
||||
)
|
||||
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_librarian(task=task, context=context)
|
||||
async with trace_span(
|
||||
"delegate_to_librarian",
|
||||
SpanType.EXPERT,
|
||||
metadata={
|
||||
"expert": "librarian",
|
||||
"task_preview": task[:100],
|
||||
"has_context": bool(context),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_librarian(task=task, context=context)
|
||||
|
||||
logger.info(
|
||||
"delegation_to_librarian_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
logger.info(
|
||||
"delegation_to_librarian_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
if span:
|
||||
span.metadata["success"] = True
|
||||
span.metadata["output_length"] = len(output)
|
||||
span.details["task"] = task
|
||||
span.details["context"] = context[:500] if context else None
|
||||
span.details["result_preview"] = output[:1000]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_librarian_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_librarian_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if span:
|
||||
span.metadata["success"] = False
|
||||
span.details["error"] = str(e)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
async def delegate_to_biographer(
|
||||
@@ -317,38 +338,58 @@ async def delegate_to_biographer(
|
||||
has_context=bool(context),
|
||||
)
|
||||
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_biographer(task=task, context=context)
|
||||
async with trace_span(
|
||||
"delegate_to_biographer",
|
||||
SpanType.EXPERT,
|
||||
metadata={
|
||||
"expert": "biographer",
|
||||
"task_preview": task[:100],
|
||||
"has_context": bool(context),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_biographer(task=task, context=context)
|
||||
|
||||
logger.info(
|
||||
"delegation_to_biographer_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
logger.info(
|
||||
"delegation_to_biographer_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
if span:
|
||||
span.metadata["success"] = True
|
||||
span.metadata["output_length"] = len(output)
|
||||
span.details["task"] = task
|
||||
span.details["context"] = context[:500] if context else None
|
||||
span.details["result_preview"] = output[:1000]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_biographer_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_biographer_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if span:
|
||||
span.metadata["success"] = False
|
||||
span.details["error"] = str(e)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
async def delegate_to_housekeeper(
|
||||
@@ -394,38 +435,58 @@ async def delegate_to_housekeeper(
|
||||
has_context=bool(context),
|
||||
)
|
||||
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_housekeeper(task=task, context=context)
|
||||
async with trace_span(
|
||||
"delegate_to_housekeeper",
|
||||
SpanType.EXPERT,
|
||||
metadata={
|
||||
"expert": "housekeeper",
|
||||
"task_preview": task[:100],
|
||||
"has_context": bool(context),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_housekeeper(task=task, context=context)
|
||||
|
||||
logger.info(
|
||||
"delegation_to_housekeeper_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
logger.info(
|
||||
"delegation_to_housekeeper_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
if span:
|
||||
span.metadata["success"] = True
|
||||
span.metadata["output_length"] = len(output)
|
||||
span.details["task"] = task
|
||||
span.details["context"] = context[:500] if context else None
|
||||
span.details["result_preview"] = output[:1000]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_housekeeper_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_housekeeper_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if span:
|
||||
span.metadata["success"] = False
|
||||
span.details["error"] = str(e)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@@ -32,93 +32,69 @@ from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Housekeeper system prompt
|
||||
HOUSEKEEPER_SYSTEM_PROMPT = """You are The Housekeeper, an expert home automation assistant in the Tatlock household.
|
||||
# Housekeeper system prompt - Optimized for Mistral-Nemo function calling
|
||||
HOUSEKEEPER_SYSTEM_PROMPT = """You are a strictly tool-based home automation assistant.
|
||||
|
||||
Your role is to help users control and monitor their smart home through Home Assistant:
|
||||
- Lights, switches, and other devices
|
||||
- Scenes (pre-configured device states)
|
||||
- Scripts (automation sequences)
|
||||
- Automations (event-triggered rules)
|
||||
## CRITICAL: You Have NO Internal Knowledge
|
||||
|
||||
## Your Personality
|
||||
- Efficient and practical
|
||||
- Safety-conscious (confirm destructive actions)
|
||||
- Proactive in suggesting optimizations
|
||||
- Clear about what actions you're taking
|
||||
You do NOT know what devices exist. You do NOT know any entity IDs.
|
||||
Entity IDs are different in every installation. You MUST discover them using tools.
|
||||
|
||||
## Your Tools
|
||||
## Entity ID Format
|
||||
|
||||
### Discovery Tools
|
||||
- **list_areas**: See all rooms/areas configured in Home Assistant
|
||||
- **list_devices**: Find devices by type (domain) or location (area)
|
||||
- **get_device_state**: Check a device's current state and attributes
|
||||
Entity IDs follow the format: `domain.name`
|
||||
Examples: `light.kitchen`, `light.study_main`, `switch.coffee_maker`
|
||||
|
||||
### Control Tools
|
||||
- **turn_on**: Turn on lights, switches, etc. (supports brightness/color for lights)
|
||||
- **turn_off**: Turn off devices
|
||||
- **toggle**: Flip a device's state
|
||||
The `entity_id` parameter MUST be the COMPLETE value including the domain prefix.
|
||||
WRONG: `entity_id="kitchen"`
|
||||
RIGHT: `entity_id="light.kitchen"`
|
||||
|
||||
### Scene Tools
|
||||
- **list_scenes**: See available scene presets
|
||||
- **activate_scene**: Activate a scene (e.g., "movie night", "good morning")
|
||||
## Step-by-Step Process (ALWAYS FOLLOW)
|
||||
|
||||
### Script Tools
|
||||
- **list_scripts**: See available automation scripts
|
||||
- **run_script**: Execute a script
|
||||
When asked to control devices in a room:
|
||||
|
||||
### Automation Tools
|
||||
- **list_automations**: See all automations and their status
|
||||
- **toggle_automation**: Enable or disable an automation
|
||||
1. THINK: What domain? (light, switch, climate, etc.)
|
||||
2. CALL: list_devices(domain="light") to discover available devices
|
||||
3. CHECK: Look for EXACT match `light.<room_name>` first!
|
||||
- For "study lights" → look for `light.study` (not light.study_main, not light.studeerlamp)
|
||||
- For "kitchen lights" → look for `light.kitchen` (not light.kitchen_spot_1)
|
||||
- These room groups control ALL lights in that room at once
|
||||
- If found, use ONLY the group (stop looking for individual lights)
|
||||
4. FALLBACK: Only if no exact room group exists, find entity_ids containing the room name
|
||||
5. CALL: turn_on/turn_off using the EXACT entity_id from step 3 or 4
|
||||
|
||||
### History Tools
|
||||
- **get_history**: Check a device's state history
|
||||
Example for "Turn off study lights":
|
||||
1. Domain is "light"
|
||||
2. Call list_devices(domain="light")
|
||||
3. Look for room group: `light.study` - FOUND!
|
||||
4. Call turn_off(entity_id="light.study") # This controls all study lights
|
||||
|
||||
## Critical Rules
|
||||
Example for "Turn off hallway lights" (no room group):
|
||||
1. Domain is "light"
|
||||
2. Call list_devices(domain="light")
|
||||
3. Look for room group: `light.hallway` - NOT FOUND
|
||||
4. Find all with "hallway": light.hallway_spot_1, light.hallway_spot_2
|
||||
5. Call turn_off for each
|
||||
|
||||
**NEVER GUESS ENTITY IDs.** You do not know what devices exist. Entity IDs vary between
|
||||
installations. You MUST call list_devices() FIRST to discover actual entity_ids before
|
||||
ANY control action (turn_on, turn_off, toggle).
|
||||
## Tool Parameter Names
|
||||
|
||||
Wrong approach:
|
||||
User: "Turn off the study lights"
|
||||
You: turn_off("light.study") ← WRONG! You guessed the entity_id
|
||||
- turn_on, turn_off, toggle: Use `entity_id` (NOT device_id, NOT id)
|
||||
- activate_scene: Use `scene_id`
|
||||
- run_script: Use `script_id`
|
||||
|
||||
Correct approach:
|
||||
User: "Turn off the study lights"
|
||||
You: list_devices(domain="light") ← First discover what exists
|
||||
You: [See results like light.study_main, light.study]
|
||||
You: turn_off("light.study_main"), turn_off("light.study") ← Use actual IDs
|
||||
## What NOT To Do
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **ALWAYS list_devices first** before any control action. No exceptions.
|
||||
Filter by domain and/or area to narrow results.
|
||||
|
||||
2. **Use exact entity_ids** from list_devices results. Never construct or guess them.
|
||||
|
||||
3. **Area-aware filtering**: Use area parameter when users mention a room.
|
||||
Note: Some devices may have area=None but contain the room name in entity_id.
|
||||
|
||||
4. **Verify after actions**: Use get_device_state to confirm state changes if needed.
|
||||
|
||||
5. **Safety for bulk actions**: When affecting multiple devices, summarize first.
|
||||
|
||||
## Common Patterns
|
||||
|
||||
- "Turn on the lights" → list_devices(domain="light"), then turn_on each
|
||||
- "What's on?" → list_devices() and filter for state="on"
|
||||
- "Movie time" → Either activate_scene("scene.movie_night") or run_script if available
|
||||
- "Dim the bedroom" → turn_on("light.bedroom", brightness=64)
|
||||
- NEVER guess an entity_id
|
||||
- NEVER construct an entity_id from the room name
|
||||
- NEVER drop the domain prefix (light., switch., etc.)
|
||||
- NEVER use "device_id" - the parameter is called "entity_id"
|
||||
- NEVER provide an answer without calling list_devices first
|
||||
|
||||
## Response Format
|
||||
Your responses are returned to Tatlock (the butler) who will synthesize them into
|
||||
a final answer for the user. Keep this in mind:
|
||||
- Lead with confirmation of what you did or found
|
||||
- Be specific about which devices were affected
|
||||
- Include relevant state information
|
||||
- Note any issues or failures
|
||||
- Be concise - Tatlock will format the final response
|
||||
|
||||
After completing actions, briefly confirm:
|
||||
- Which devices were affected (list the entity_ids)
|
||||
- Whether each action succeeded or failed
|
||||
"""
|
||||
|
||||
# Lazy initialization to avoid connection issues during imports
|
||||
@@ -127,16 +103,10 @@ _housekeeper_agent: Optional[Agent[None, str]] = None
|
||||
|
||||
def _create_housekeeper_agent() -> Agent[None, str]:
|
||||
"""Create the Housekeeper PydanticAI agent."""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
# Create Ollama model with sanitized provider
|
||||
# (fixes 'content: null' issue with tool calls)
|
||||
model = OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
agent: Agent[None, str] = Agent(
|
||||
model=model,
|
||||
@@ -169,9 +139,12 @@ def _create_housekeeper_agent() -> Agent[None, str]:
|
||||
# Register history tools
|
||||
agent.tool_plain(get_history)
|
||||
|
||||
from src.anthropic.model_selector import get_model_info
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"housekeeper_agent_created",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
tool_count=13,
|
||||
)
|
||||
|
||||
@@ -231,9 +204,13 @@ async def run_housekeeper(
|
||||
)
|
||||
|
||||
try:
|
||||
# Use temperature 0.1 for slight exploration
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
result = await agent.run(
|
||||
prompt,
|
||||
message_history=message_history,
|
||||
model_settings=ModelSettings(temperature=0.1),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -289,9 +266,13 @@ async def run_housekeeper_stream(
|
||||
)
|
||||
|
||||
try:
|
||||
# Use temperature 0.1 for slight exploration
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
async with agent.run_stream(
|
||||
prompt,
|
||||
message_history=message_history,
|
||||
model_settings=ModelSettings(temperature=0.1),
|
||||
) as response:
|
||||
async for delta in response.stream_text(delta=True):
|
||||
yield delta
|
||||
|
||||
@@ -59,10 +59,27 @@ async def list_devices(
|
||||
|
||||
for dom, dom_devices in sorted(by_domain.items()):
|
||||
output_parts.append(f"### {dom.title()}s")
|
||||
for device in dom_devices:
|
||||
|
||||
# Sort devices: room groups first (using Home Assistant's is_hue_group attribute)
|
||||
def is_room_group(d: object) -> bool:
|
||||
"""Check if device is a room group based on HA attributes."""
|
||||
attrs = getattr(d, "attributes", {})
|
||||
# Check for Hue room groups
|
||||
if attrs.get("is_hue_group") and attrs.get("hue_type") == "room":
|
||||
return True
|
||||
# Check for other group indicators (icon or entity_id list)
|
||||
if "entity_id" in attrs and isinstance(attrs["entity_id"], list):
|
||||
return True
|
||||
return False
|
||||
|
||||
sorted_devices = sorted(dom_devices, key=lambda d: (not is_room_group(d), d.entity_id))
|
||||
|
||||
for device in sorted_devices:
|
||||
state_icon = "on" if device.state == "on" else "off" if device.state == "off" else device.state
|
||||
area_str = f" ({device.area})" if device.area else ""
|
||||
output_parts.append(f"- **{device.name}**{area_str}: {state_icon}")
|
||||
# Mark room groups clearly using actual HA data
|
||||
group_marker = " [ROOM GROUP]" if is_room_group(device) else ""
|
||||
output_parts.append(f"- **{device.name}**{area_str}{group_marker}: {state_icon}")
|
||||
output_parts.append(f" ID: `{device.entity_id}`")
|
||||
output_parts.append("")
|
||||
|
||||
@@ -164,12 +181,12 @@ async def turn_on(
|
||||
color_temp: int | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
Turn on a device.
|
||||
Turn on a device. Use the entity_id parameter with the EXACT value from list_devices.
|
||||
|
||||
For lights, can optionally set brightness and color temperature.
|
||||
|
||||
Args:
|
||||
entity_id: Device to turn on (e.g., light.living_room, switch.coffee_maker)
|
||||
entity_id: The EXACT entity ID from list_devices including domain prefix.
|
||||
brightness: Optional brightness for lights (0-255, where 255 is full brightness)
|
||||
color_temp: Optional color temperature in Kelvin (2700=warm, 6500=cool)
|
||||
|
||||
@@ -177,10 +194,9 @@ async def turn_on(
|
||||
Confirmation of the action
|
||||
|
||||
Examples:
|
||||
turn_on("light.living_room") # Turn on at current brightness
|
||||
turn_on("light.bedroom", brightness=128) # Turn on at 50% brightness
|
||||
turn_on("light.office", brightness=255, color_temp=4000) # Full, neutral white
|
||||
turn_on("switch.coffee_maker") # Turn on a switch
|
||||
turn_on(entity_id="light.living_room")
|
||||
turn_on(entity_id="light.bedroom", brightness=128)
|
||||
turn_on(entity_id="switch.coffee_maker")
|
||||
"""
|
||||
try:
|
||||
async with CoreAPIClient() as client:
|
||||
@@ -209,17 +225,18 @@ async def turn_on(
|
||||
|
||||
async def turn_off(entity_id: str) -> str:
|
||||
"""
|
||||
Turn off a device.
|
||||
Turn off a device. Use the entity_id parameter with the EXACT value from list_devices.
|
||||
|
||||
Args:
|
||||
entity_id: Device to turn off (e.g., light.living_room, switch.coffee_maker)
|
||||
entity_id: The EXACT entity ID from list_devices including domain prefix.
|
||||
|
||||
Returns:
|
||||
Confirmation of the action
|
||||
|
||||
Examples:
|
||||
turn_off("light.living_room")
|
||||
turn_off("switch.coffee_maker")
|
||||
turn_off(entity_id="light.living_room")
|
||||
turn_off(entity_id="switch.coffee_maker")
|
||||
turn_off(entity_id="light.kitchen")
|
||||
"""
|
||||
try:
|
||||
async with CoreAPIClient() as client:
|
||||
@@ -239,15 +256,17 @@ async def toggle(entity_id: str) -> str:
|
||||
"""
|
||||
Toggle a device's state (on becomes off, off becomes on).
|
||||
|
||||
Use the entity_id parameter with the EXACT value from list_devices.
|
||||
|
||||
Args:
|
||||
entity_id: Device to toggle
|
||||
entity_id: The EXACT entity ID from list_devices including domain prefix.
|
||||
|
||||
Returns:
|
||||
Confirmation with the new state
|
||||
|
||||
Examples:
|
||||
toggle("light.living_room")
|
||||
toggle("switch.fan")
|
||||
toggle(entity_id="light.living_room")
|
||||
toggle(entity_id="switch.fan")
|
||||
"""
|
||||
try:
|
||||
async with CoreAPIClient() as client:
|
||||
|
||||
@@ -39,7 +39,14 @@ Your role is to help users find, understand, synthesize, and manage information
|
||||
- The personal wiki (Wiki.js) containing documentation and notes
|
||||
- The knowledge graph (Neo4j) with entities and relationships
|
||||
- Vector embeddings (Qdrant) for semantic search
|
||||
- Web search (SearXNG) for current information
|
||||
- Paperless documents (📑) - indexed PDFs, scanned documents, invoices, receipts from the user's document archive
|
||||
- Volatile cache (⚡) - pre-fetched real-time data for user-relevant locations and items:
|
||||
- weather/forecast: conditions and forecasts for user's configured cities
|
||||
- news: headlines from user's preferred sources
|
||||
- stock/crypto: quotes for user's watched symbols
|
||||
- sun/air_quality: data for user's locations
|
||||
- Note: volatile data may not exist for arbitrary queries - falls back to web search
|
||||
- Web search (SearXNG) for current information not available in cache
|
||||
|
||||
## Your Personality
|
||||
- Scholarly and thorough in your research
|
||||
@@ -60,7 +67,13 @@ Your role is to help users find, understand, synthesize, and manage information
|
||||
- Use for: comparing multiple sources, gathering info from several pages
|
||||
|
||||
### Internal Research Tools
|
||||
- **hybrid_search**: Your primary research tool - searches wiki, graph, and web at once
|
||||
- **hybrid_search**: Your primary research tool - searches ALL sources at once:
|
||||
- Wiki pages (vector similarity)
|
||||
- Knowledge graph (entity relationships)
|
||||
- Paperless documents (📑 indexed PDFs, scans)
|
||||
- Volatile cache (⚡ weather, news, stocks - when available)
|
||||
- Web search (current information)
|
||||
Results are fused and re-ranked by relevance. Volatile data gets priority when fresh.
|
||||
- **search_wiki**: Find specific wiki pages by keyword
|
||||
- **semantic_search**: Find conceptually similar content
|
||||
- **explore_knowledge_graph** / **find_related_entities**: Discover connections
|
||||
@@ -114,6 +127,14 @@ Your responses are returned to Tatlock (the butler) who will synthesize them int
|
||||
- Note any gaps in available information
|
||||
- Be concise but thorough - Tatlock will format the final response
|
||||
- Structure your findings clearly so they can be easily integrated with other responses
|
||||
|
||||
## CRITICAL: Never Fabricate Information
|
||||
If a tool fails or you cannot access a data source:
|
||||
- Say "I was unable to retrieve [information type]" - be specific about what failed
|
||||
- Do NOT provide placeholder, template, or made-up data
|
||||
- Do NOT say "Here's what I would have said" or "Here's a sample response"
|
||||
- Do NOT invent specific numbers, dates, or facts when the actual data is unavailable
|
||||
- It is better to return no information than to return fabricated information
|
||||
"""
|
||||
|
||||
# Lazy initialization to avoid connection issues during imports
|
||||
@@ -122,16 +143,10 @@ _librarian_agent: Optional[Agent[None, str]] = None
|
||||
|
||||
def _create_librarian_agent() -> Agent[None, str]:
|
||||
"""Create the Librarian PydanticAI agent."""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
# Create Ollama model with sanitized provider
|
||||
# (fixes 'content: null' issue with tool calls)
|
||||
model = OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
agent: Agent[None, str] = Agent(
|
||||
model=model,
|
||||
@@ -161,9 +176,12 @@ def _create_librarian_agent() -> Agent[None, str]:
|
||||
agent.tool_plain(update_wiki_page)
|
||||
agent.tool_plain(smart_create_wiki_page)
|
||||
|
||||
from src.anthropic.model_selector import get_model_info
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"librarian_agent_created",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
tool_count=14, # 7 research + 3 web + 1 wiki read + 3 wiki write
|
||||
)
|
||||
|
||||
|
||||
@@ -225,18 +225,22 @@ class LibraryDeskClient:
|
||||
vector_limit: int = 10,
|
||||
graph_limit: int = 10,
|
||||
web_limit: int = 5,
|
||||
document_limit: int = 5,
|
||||
volatile_limit: int = 3,
|
||||
enable_reranking: bool = True,
|
||||
final_result_count: int = 10,
|
||||
) -> HybridRAGResponse:
|
||||
"""
|
||||
Execute HybridRAG search combining vector, graph, and web results.
|
||||
Execute HybridRAG search combining vector, graph, documents, volatile, and web.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
user: User identifier for multi-tenancy (defaults to request context)
|
||||
vector_limit: Max results from vector search
|
||||
vector_limit: Max results from vector search (wiki pages)
|
||||
graph_limit: Max results from graph search
|
||||
web_limit: Max results from web search
|
||||
web_limit: Max results from web search (0 to disable)
|
||||
document_limit: Max results from Paperless documents (0 to disable)
|
||||
volatile_limit: Max results from volatile cache (0 to disable)
|
||||
enable_reranking: Whether to rerank with LLM
|
||||
final_result_count: Number of final results after fusion
|
||||
|
||||
@@ -252,6 +256,11 @@ class LibraryDeskClient:
|
||||
"vector_limit": vector_limit,
|
||||
"graph_limit": graph_limit,
|
||||
"web_limit": web_limit,
|
||||
"document_limit": document_limit,
|
||||
"volatile_limit": volatile_limit,
|
||||
"enable_documents": document_limit > 0,
|
||||
"enable_volatile": volatile_limit > 0,
|
||||
"enable_web": web_limit > 0,
|
||||
"enable_reranking": enable_reranking,
|
||||
"final_result_count": final_result_count,
|
||||
},
|
||||
|
||||
@@ -17,20 +17,26 @@ logger = get_logger(__name__)
|
||||
async def hybrid_search(
|
||||
query: str,
|
||||
include_web: bool = True,
|
||||
include_documents: bool = True,
|
||||
include_volatile: bool = True,
|
||||
) -> str:
|
||||
"""
|
||||
Search across all knowledge sources using HybridRAG.
|
||||
|
||||
This is the primary research tool, combining:
|
||||
- Vector search (semantic similarity over documents)
|
||||
- Vector search (semantic similarity over wiki pages)
|
||||
- Knowledge graph (entities and relationships)
|
||||
- Paperless documents (📑 indexed PDFs, scans, invoices)
|
||||
- Volatile cache (⚡ weather, news, stocks - for user's configured items)
|
||||
- Web search (current information from SearXNG)
|
||||
|
||||
Results are fused and re-ranked by relevance.
|
||||
Results are fused and re-ranked by relevance. Volatile data gets priority when fresh.
|
||||
|
||||
Args:
|
||||
query: Natural language research query
|
||||
include_web: Whether to include web results (default: True)
|
||||
include_documents: Whether to include Paperless documents (default: True)
|
||||
include_volatile: Whether to include volatile cache data (default: True)
|
||||
|
||||
Returns:
|
||||
Formatted search results with sources and context
|
||||
@@ -38,12 +44,16 @@ async def hybrid_search(
|
||||
Examples:
|
||||
hybrid_search("How does Docker orchestration work with Kubernetes?")
|
||||
hybrid_search("What projects use Neo4j?", include_web=False)
|
||||
hybrid_search("Find my electricity invoices", include_web=False, include_volatile=False)
|
||||
hybrid_search("What's the weather in Rotterdam?") # May hit volatile cache
|
||||
"""
|
||||
try:
|
||||
async with LibraryDeskClient() as client:
|
||||
response = await client.hybrid_search(
|
||||
query=query,
|
||||
web_limit=5 if include_web else 0,
|
||||
document_limit=5 if include_documents else 0,
|
||||
volatile_limit=3 if include_volatile else 0,
|
||||
)
|
||||
|
||||
if not response.results:
|
||||
@@ -70,6 +80,8 @@ async def hybrid_search(
|
||||
"vector": "📄",
|
||||
"graph": "🔗",
|
||||
"web": "🌐",
|
||||
"document": "📑",
|
||||
"volatile": "⚡",
|
||||
}.get(result.source, "•")
|
||||
|
||||
output_parts.append(
|
||||
|
||||
+110
-29
@@ -5,11 +5,13 @@ The Steward analyzes incoming requests, identifies relevant household
|
||||
capabilities, and provides focused recommendations to Tatlock (the Butler).
|
||||
This creates a two-tier architecture that prevents cognitive overload.
|
||||
|
||||
Uses plain text output (not JSON) for reliability with Ollama models.
|
||||
Uses plain text output (not JSON) for reliability. Supports both Claude
|
||||
(preferred) and Ollama (fallback) backends via direct API calls.
|
||||
"""
|
||||
import httpx
|
||||
from typing import Optional
|
||||
|
||||
from src.anthropic.model_selector import is_claude_available, get_model_info
|
||||
from src.core.config import config
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
@@ -56,14 +58,22 @@ USER QUERY: {query}
|
||||
GUIDELINES:
|
||||
- Be conservative - only recommend truly necessary capabilities
|
||||
- Simple greetings/chat → no capabilities needed (conversational response only)
|
||||
- Questions about prior conversation ("what did I say", "my name", "what we discussed") → no capabilities (Tatlock has full history)
|
||||
- Questions about prior conversation ("what did I say", "what we discussed") → no capabilities (Tatlock has full history)
|
||||
- Math/calculations → tatlock_core
|
||||
- Time/date queries → tatlock_core
|
||||
- PERSONAL MEMORY queries → biographer to recall (ALWAYS use for questions about the user themselves):
|
||||
- "where do I live", "what's my location", "my address" → biographer to recall location
|
||||
- "what's my name", "who am I" → biographer to recall name
|
||||
- "what car do I drive", "my vehicle" → biographer to recall car
|
||||
- "what do you know about me", "what have I told you" → biographer to recall or list_memories
|
||||
- "remember that I...", "store that..." → biographer to store_insight
|
||||
- "forget my...", "delete..." → biographer to forget_memory
|
||||
- "my timezone", "my preferences" → biographer to recall preferences
|
||||
- Web searches, weather, news, current information → librarian with search_web
|
||||
- Read a URL or article → librarian with read_url
|
||||
- Wiki creation ("create a page about X", "add X to wiki") → librarian with smart_create
|
||||
- Wiki updates ("update the page", "add to dossier") → librarian with update
|
||||
- Research queries ("find info", "what do we know about", "search for") → librarian with hybrid_search
|
||||
- Research queries about TOPICS (not about the user) → librarian with hybrid_search
|
||||
- In-depth research, knowledge synthesis, document lookup → librarian with hybrid_search
|
||||
- If conversation history is relevant, note which previous turns matter
|
||||
- Assess complexity: simple (1 tool), moderate (2-3 tools), complex (multiple steps)
|
||||
@@ -75,6 +85,10 @@ COMPLEXITY: [simple/moderate/complex]
|
||||
CONTEXT: [any relevant conversation context, or "none"]
|
||||
|
||||
EXAMPLES:
|
||||
- "DELEGATE: biographer to recall the user's location" (for "where do I live?")
|
||||
- "DELEGATE: biographer to recall the user's car" (for "what car do I drive?")
|
||||
- "DELEGATE: biographer to list_memories about the user" (for "what do you know about me?")
|
||||
- "DELEGATE: biographer to store_insight about user's pet" (for "remember that I have a dog named Max")
|
||||
- "DELEGATE: librarian to search_web for tomorrow's weather forecast"
|
||||
- "DELEGATE: librarian to create a wiki page about CI/CD pipelines"
|
||||
- "DELEGATE: librarian to hybrid_search for information about Docker networking"
|
||||
@@ -93,22 +107,74 @@ class StewardAgent:
|
||||
Analyzes requests with full conversation context and recommends
|
||||
which household capabilities the Butler should use.
|
||||
|
||||
Uses plain text output for reliability with Ollama models.
|
||||
Uses plain text output for reliability. Supports both Claude
|
||||
(preferred) and Ollama (fallback) backends via direct API calls.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Steward with Ollama model (same as Tatlock for VRAM efficiency)."""
|
||||
"""Initialize Steward with backend selection based on availability."""
|
||||
# Ollama config (fallback)
|
||||
self.ollama_host = str(config.OLLAMA_HOST).rstrip('/')
|
||||
self.model_name = config.OLLAMA_DEFAULT_MODEL
|
||||
self.ollama_model = config.OLLAMA_DEFAULT_MODEL
|
||||
|
||||
# Claude config (preferred)
|
||||
self.claude_model = config.ANTHROPIC_MODEL
|
||||
self._anthropic_client = None
|
||||
|
||||
# Determine which backend to use
|
||||
self._use_claude = config.PREFER_CLOUD_BACKEND and is_claude_available()
|
||||
|
||||
self.timeout = 30.0 # 30 second timeout for analysis
|
||||
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"steward_agent_created",
|
||||
ollama_host=self.ollama_host,
|
||||
model=self.model_name,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
def _get_anthropic_client(self):
|
||||
"""Get or create Anthropic client (lazy initialization)."""
|
||||
if self._anthropic_client is None:
|
||||
from anthropic import AsyncAnthropic
|
||||
self._anthropic_client = AsyncAnthropic(api_key=config.ANTHROPIC_API_KEY)
|
||||
return self._anthropic_client
|
||||
|
||||
async def _call_claude(self, system_prompt: str, user_message: str) -> str:
|
||||
"""Call Claude API directly for plain text generation."""
|
||||
client = self._get_anthropic_client()
|
||||
|
||||
response = await client.messages.create(
|
||||
model=self.claude_model,
|
||||
max_tokens=1024,
|
||||
system=system_prompt,
|
||||
messages=[{"role": "user", "content": user_message}],
|
||||
temperature=0.3, # Lower = more consistent
|
||||
)
|
||||
|
||||
return response.content[0].text.strip()
|
||||
|
||||
async def _call_ollama(self, prompt: str) -> str:
|
||||
"""Call Ollama API directly for plain text generation."""
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.ollama_host}/api/generate",
|
||||
json={
|
||||
"model": self.ollama_model,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.3, # Lower = more consistent
|
||||
"top_p": 0.9
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
return result["response"].strip()
|
||||
|
||||
async def analyze(
|
||||
self,
|
||||
query: str,
|
||||
@@ -117,6 +183,8 @@ class StewardAgent:
|
||||
"""
|
||||
Analyze query and return plain text recommendation.
|
||||
|
||||
Uses Claude if available, falls back to Ollama.
|
||||
|
||||
Args:
|
||||
query: User's query to analyze
|
||||
conversation_history: Previous conversation turns
|
||||
@@ -132,35 +200,48 @@ class StewardAgent:
|
||||
history = conversation_history or []
|
||||
prompt = build_steward_prompt(query, history)
|
||||
|
||||
logger.debug("steward_calling_ollama", query_preview=query[:100])
|
||||
backend = "claude" if self._use_claude else "ollama"
|
||||
logger.debug(
|
||||
"steward_calling_llm",
|
||||
backend=backend,
|
||||
query_preview=query[:100],
|
||||
)
|
||||
|
||||
# Call Ollama API directly (more reliable than PydanticAI for plain text)
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.ollama_host}/api/generate",
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.3, # Lower = more consistent
|
||||
"top_p": 0.9
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
|
||||
analysis_text = result["response"].strip()
|
||||
try:
|
||||
if self._use_claude:
|
||||
# For Claude, split into system + user message
|
||||
# The prompt contains both, but Claude prefers explicit system
|
||||
analysis_text = await self._call_claude(
|
||||
system_prompt="You are the Steward of the household, advising the Butler (Tatlock) on which capabilities to use. Be concise and specific.",
|
||||
user_message=prompt,
|
||||
)
|
||||
else:
|
||||
analysis_text = await self._call_ollama(prompt)
|
||||
|
||||
logger.debug(
|
||||
"steward_analysis_received",
|
||||
text_preview=analysis_text[:150]
|
||||
backend=backend,
|
||||
text_preview=analysis_text[:150],
|
||||
)
|
||||
|
||||
return analysis_text
|
||||
|
||||
except Exception as e:
|
||||
# If Claude fails, try Ollama as fallback
|
||||
if self._use_claude:
|
||||
logger.warning(
|
||||
"steward_claude_fallback",
|
||||
error=str(e),
|
||||
)
|
||||
analysis_text = await self._call_ollama(prompt)
|
||||
logger.debug(
|
||||
"steward_analysis_received",
|
||||
backend="ollama_fallback",
|
||||
text_preview=analysis_text[:150],
|
||||
)
|
||||
return analysis_text
|
||||
raise
|
||||
|
||||
|
||||
# Global Steward instance
|
||||
_steward_agent = None
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""
|
||||
Steward service layer.
|
||||
|
||||
Provides high-level interface for request analysis with logging,
|
||||
benchmarking, and error handling.
|
||||
Provides high-level interface for request analysis with logging
|
||||
and error handling.
|
||||
|
||||
Parses plain text recommendations into structured data.
|
||||
Includes memory pre-fetch for user context injection.
|
||||
@@ -10,7 +10,6 @@ Includes memory pre-fetch for user context injection.
|
||||
import re
|
||||
from typing import Any, Optional
|
||||
|
||||
from src.core.benchmarks import PerformanceBenchmark, get_benchmark_store
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger, log_operation
|
||||
from src.core.memory_service import memory_service
|
||||
@@ -237,7 +236,9 @@ async def _prefetch_memory_context(user_request: str) -> dict[str, Any]:
|
||||
# Location-related queries
|
||||
if any(word in request_lower for word in [
|
||||
"weather", "temperature", "forecast", "nearby", "local",
|
||||
"directions", "distance", "map", "here"
|
||||
"directions", "distance", "map", "here",
|
||||
# Direct location questions
|
||||
"live", "where", "home", "reside", "location", "address",
|
||||
]):
|
||||
profile_keys.append("location")
|
||||
|
||||
@@ -285,8 +286,7 @@ async def analyze_request(
|
||||
This is the main entry point for Steward analysis. It:
|
||||
1. Calls the Steward agent with full conversation history
|
||||
2. Logs the operation with timing
|
||||
3. Records performance benchmarks to Redis
|
||||
4. Returns structured recommendations
|
||||
3. Returns structured recommendations
|
||||
|
||||
Args:
|
||||
user_request: The current user message to analyze
|
||||
@@ -365,23 +365,6 @@ async def analyze_request(
|
||||
reasoning=analysis_text[:200], # First 200 chars
|
||||
)
|
||||
|
||||
# Record performance benchmark
|
||||
if log_ctx.get("duration_seconds"):
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="steward_analysis",
|
||||
duration_seconds=log_ctx["duration_seconds"],
|
||||
success=True,
|
||||
recommendation_count=len(recommendation.recommended_capabilities),
|
||||
confidence=None, # Could add confidence scoring in future
|
||||
conversation_id=conversation_id,
|
||||
metadata={
|
||||
"complexity": recommendation.estimated_complexity,
|
||||
"has_context": recommendation.conversation_context.has_previous_context,
|
||||
"missing_capabilities": recommendation.missing_capabilities is not None,
|
||||
},
|
||||
)
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
return recommendation
|
||||
|
||||
except Exception as e:
|
||||
|
||||
+96
-73
@@ -20,6 +20,11 @@ from src.agents.tatlock_core.tools import (
|
||||
)
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import (
|
||||
start_span, end_span, get_current_span,
|
||||
add_tool_spans_from_messages,
|
||||
SpanType, SpanStatus,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -42,7 +47,16 @@ def generate_id() -> str:
|
||||
# System prompt defining Tatlock's personality
|
||||
TATLOCK_SYSTEM_PROMPT = """You are Tatlock, a helpful personal assistant with the demeanor of a British butler.
|
||||
|
||||
Address users as "sir" and maintain a formal yet personable tone. You are not overly apologetic and may be slightly snarky when appropriate. If an opportunity for a pun presents itself, you cannot resist.
|
||||
## Personality
|
||||
|
||||
Address users as "sir". Be confident, direct, and efficient - you are an unflappable English butler who gets things done. Dry wit and puns are encouraged.
|
||||
|
||||
**CRITICAL - Do NOT:**
|
||||
- Apologize unless you genuinely made an error
|
||||
- Say "Apologies for any confusion" or "Allow me to rectify" when nothing went wrong
|
||||
- Preface successful results with caveats or apologies
|
||||
|
||||
When presenting findings: lead with the answer, be concise, skip the preamble.
|
||||
|
||||
You coordinate with various household staff (expert agents) to provide comprehensive assistance across:
|
||||
- Research and knowledge work
|
||||
@@ -124,10 +138,7 @@ class TatlockAgent(AgentInterface):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Tatlock configuration (lazy agent creation)."""
|
||||
# Store Ollama configuration
|
||||
self.ollama_host = str(config.OLLAMA_HOST)
|
||||
self.model_name = config.OLLAMA_DEFAULT_MODEL
|
||||
"""Initialize Tatlock (lazy agent creation)."""
|
||||
self._agent = None # Lazy initialization
|
||||
|
||||
def _ensure_agent(self):
|
||||
@@ -135,30 +146,21 @@ class TatlockAgent(AgentInterface):
|
||||
if self._agent is not None:
|
||||
return
|
||||
|
||||
from src.anthropic.model_selector import get_model, get_model_info
|
||||
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"tatlock_agent_initializing",
|
||||
ollama_host=self.ollama_host,
|
||||
model=self.model_name,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
)
|
||||
|
||||
# Import required classes for Ollama configuration
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
# PydanticAI expects Ollama base URL to end with /v1
|
||||
# Remove trailing slash from ollama_host if present
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
# Create Ollama model with provider
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
|
||||
# Create PydanticAI agent with Ollama model
|
||||
# Create PydanticAI agent
|
||||
self._agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
)
|
||||
|
||||
@@ -433,7 +435,7 @@ class TatlockAgent(AgentInterface):
|
||||
steward_note: Note from Steward (prepended to request, invisible to user)
|
||||
scoped_tools: List of tool definitions from household registry
|
||||
message_history: Conversation history in PydanticAI format
|
||||
tool_tracker: Optional tool call tracker for benchmarking
|
||||
tool_tracker: Optional tool call tracker for analysis
|
||||
|
||||
Returns:
|
||||
str: Tatlock's response text
|
||||
@@ -447,8 +449,7 @@ class TatlockAgent(AgentInterface):
|
||||
... tool_tracker=tracker,
|
||||
... )
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_run_with_scoped_tools",
|
||||
@@ -459,18 +460,12 @@ class TatlockAgent(AgentInterface):
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
# This ensures Tatlock can ONLY use tools recommended by the Steward
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools
|
||||
# Tools from household registry are already PydanticAI Tool objects
|
||||
scoped_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
tools=scoped_tools, # Pass tools directly to Agent constructor
|
||||
)
|
||||
@@ -499,13 +494,13 @@ class TatlockAgent(AgentInterface):
|
||||
)
|
||||
|
||||
# Run with scoped tools and tracker
|
||||
# Force tool_choice: required to make LLM actually call tools
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
# Force tool_choice to make LLM actually call tools
|
||||
from src.anthropic.model_selector import get_tool_choice_settings
|
||||
result = await scoped_agent.run(
|
||||
enriched_message,
|
||||
message_history=pydantic_history if pydantic_history else None,
|
||||
deps=tool_tracker,
|
||||
model_settings=ModelSettings(extra_body={"tool_choice": "required"})
|
||||
model_settings=get_tool_choice_settings(),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -541,8 +536,7 @@ class TatlockAgent(AgentInterface):
|
||||
Yields:
|
||||
Text chunks from the streaming response
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_run_with_scoped_tools_stream",
|
||||
@@ -552,17 +546,11 @@ class TatlockAgent(AgentInterface):
|
||||
)
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools
|
||||
scoped_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
tools=scoped_tools,
|
||||
)
|
||||
@@ -627,7 +615,7 @@ class TatlockAgent(AgentInterface):
|
||||
steward_note: Note from Steward (invisible to user)
|
||||
scoped_tools: List of tool definitions from household registry
|
||||
message_history: Conversation history
|
||||
tool_tracker: Optional tool call tracker for benchmarking
|
||||
tool_tracker: Optional tool call tracker for analysis
|
||||
|
||||
Returns:
|
||||
dict with:
|
||||
@@ -636,9 +624,6 @@ class TatlockAgent(AgentInterface):
|
||||
- tool_outputs: Dict mapping tool names to their outputs
|
||||
- raw_output: The agent's raw text output
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
from pydantic_ai.messages import (
|
||||
ModelRequest,
|
||||
ModelResponse,
|
||||
@@ -647,6 +632,7 @@ class TatlockAgent(AgentInterface):
|
||||
ToolCallPart,
|
||||
ToolReturnPart,
|
||||
)
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_orchestrate_tool_calls",
|
||||
@@ -655,18 +641,22 @@ class TatlockAgent(AgentInterface):
|
||||
history_length=len(message_history),
|
||||
)
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
# Start tracing span for orchestration phase
|
||||
orchestrate_span = start_span(
|
||||
"tatlock_orchestrate",
|
||||
SpanType.TATLOCK,
|
||||
metadata={
|
||||
"scoped_tool_count": len(scoped_tools),
|
||||
"tool_names": [getattr(t, '__name__', str(t)) for t in scoped_tools[:5]],
|
||||
},
|
||||
)
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools
|
||||
scoped_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
tools=scoped_tools,
|
||||
)
|
||||
@@ -693,11 +683,12 @@ class TatlockAgent(AgentInterface):
|
||||
)
|
||||
|
||||
# Run with scoped tools and tracker
|
||||
from src.anthropic.model_selector import get_tool_choice_settings
|
||||
result = await scoped_agent.run(
|
||||
enriched_message,
|
||||
message_history=pydantic_history if pydantic_history else None,
|
||||
deps=tool_tracker,
|
||||
model_settings=ModelSettings(extra_body={"tool_choice": "required"})
|
||||
model_settings=get_tool_choice_settings(),
|
||||
)
|
||||
|
||||
# Extract tool calls and results from the agent's messages
|
||||
@@ -731,6 +722,23 @@ class TatlockAgent(AgentInterface):
|
||||
tool_output_count=len(tool_outputs),
|
||||
)
|
||||
|
||||
# Add tool-level spans from result messages
|
||||
if orchestrate_span:
|
||||
add_tool_spans_from_messages(result.new_messages(), orchestrate_span)
|
||||
|
||||
# End orchestration span with results
|
||||
end_span(
|
||||
orchestrate_span,
|
||||
metadata_update={
|
||||
"tools_called": tools_called,
|
||||
"expert_count": len(expert_results),
|
||||
"tool_output_count": len(tool_outputs),
|
||||
},
|
||||
details_update={
|
||||
"steward_note_preview": steward_note[:500] if steward_note else None,
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"tools_called": tools_called,
|
||||
"expert_results": expert_results,
|
||||
@@ -758,9 +766,8 @@ class TatlockAgent(AgentInterface):
|
||||
Returns:
|
||||
str: Butler-toned response synthesized from all results
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from pydantic_ai.messages import ModelRequest, ModelResponse, UserPromptPart, TextPart
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_synthesize_from_results",
|
||||
@@ -769,6 +776,16 @@ class TatlockAgent(AgentInterface):
|
||||
tool_count=len(orchestration_results.get("tool_outputs", {})),
|
||||
)
|
||||
|
||||
# Start tracing span for synthesis phase
|
||||
synthesize_span = start_span(
|
||||
"tatlock_synthesize",
|
||||
SpanType.TATLOCK,
|
||||
metadata={
|
||||
"expert_count": len(orchestration_results.get("expert_results", {})),
|
||||
"tool_output_count": len(orchestration_results.get("tool_outputs", {})),
|
||||
},
|
||||
)
|
||||
|
||||
# Build synthesis prompt with all available information
|
||||
synthesis_parts = []
|
||||
synthesis_parts.append(f"The user asked: {user_message}")
|
||||
@@ -789,25 +806,19 @@ class TatlockAgent(AgentInterface):
|
||||
synthesis_parts.append("")
|
||||
|
||||
synthesis_parts.append(
|
||||
"Based on this information, provide a response to the user. "
|
||||
"Maintain your butler personality - address them as 'sir', "
|
||||
"use formal but personable language, and be helpful."
|
||||
"Synthesize a response for the user. Be direct and confident. "
|
||||
"Lead with the answer - no apologies, no caveats, no 'mix-ups'. "
|
||||
"Address them as 'sir', be concise, add dry wit if appropriate."
|
||||
)
|
||||
|
||||
synthesis_prompt = "\n".join(synthesis_parts)
|
||||
|
||||
# Create synthesis agent (no tools needed)
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
model = get_model()
|
||||
|
||||
# Synthesis agent uses butler prompt but no tools
|
||||
synthesis_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
# No tools for synthesis phase
|
||||
)
|
||||
@@ -841,6 +852,18 @@ class TatlockAgent(AgentInterface):
|
||||
response_preview=result.output[:100],
|
||||
)
|
||||
|
||||
# End synthesis span with result
|
||||
end_span(
|
||||
synthesize_span,
|
||||
metadata_update={
|
||||
"response_length": len(result.output),
|
||||
},
|
||||
details_update={
|
||||
"synthesis_prompt": synthesis_prompt[:1000],
|
||||
"response_preview": result.output[:500],
|
||||
},
|
||||
)
|
||||
|
||||
return result.output
|
||||
|
||||
async def get_capabilities(self) -> dict:
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""
|
||||
Anthropic/Claude integration module.
|
||||
|
||||
Provides model selection with automatic fallback between Claude and Ollama.
|
||||
"""
|
||||
|
||||
from src.anthropic.model_selector import (
|
||||
check_claude_health,
|
||||
get_model,
|
||||
get_tool_choice_settings,
|
||||
is_claude_available,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"check_claude_health",
|
||||
"get_model",
|
||||
"get_tool_choice_settings",
|
||||
"is_claude_available",
|
||||
]
|
||||
@@ -0,0 +1,169 @@
|
||||
"""
|
||||
Model selector for Claude/Ollama backend switching.
|
||||
|
||||
Provides automatic model selection with Claude as preferred backend
|
||||
and Ollama as offline fallback.
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from pydantic_ai.providers.anthropic import AnthropicProvider
|
||||
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Cached health check result (set once at startup)
|
||||
_claude_available: bool | None = None
|
||||
|
||||
|
||||
async def check_claude_health() -> bool:
|
||||
"""
|
||||
Check if Claude API is reachable and working.
|
||||
|
||||
This should be called once at application startup.
|
||||
The result is cached in `_claude_available`.
|
||||
|
||||
Returns:
|
||||
True if Claude API is accessible, False otherwise.
|
||||
"""
|
||||
global _claude_available
|
||||
|
||||
# No API key configured - Claude not available
|
||||
if not config.ANTHROPIC_API_KEY:
|
||||
logger.info(
|
||||
"claude_health_check_skipped",
|
||||
reason="no_api_key",
|
||||
)
|
||||
_claude_available = False
|
||||
return False
|
||||
|
||||
try:
|
||||
from anthropic import AsyncAnthropic
|
||||
|
||||
client = AsyncAnthropic(api_key=config.ANTHROPIC_API_KEY)
|
||||
|
||||
# Minimal API call to verify connectivity
|
||||
# Using a tiny max_tokens to minimize cost
|
||||
await client.messages.create(
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
max_tokens=1,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
_claude_available = True
|
||||
logger.info(
|
||||
"claude_health_check_passed",
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
_claude_available = False
|
||||
logger.warning(
|
||||
"claude_health_check_failed",
|
||||
error=str(e),
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def is_claude_available() -> bool:
|
||||
"""
|
||||
Check if Claude is available (from cached health check result).
|
||||
|
||||
Returns:
|
||||
True if Claude API was reachable at startup, False otherwise.
|
||||
|
||||
Note:
|
||||
Returns False if health check hasn't been run yet.
|
||||
Call `check_claude_health()` at startup first.
|
||||
"""
|
||||
return _claude_available is True
|
||||
|
||||
|
||||
def get_model(prefer_cloud: bool | None = None) -> Union[AnthropicModel, OpenAIChatModel]:
|
||||
"""
|
||||
Get the best available model.
|
||||
|
||||
Returns Claude if available and preferred, otherwise Ollama.
|
||||
|
||||
Args:
|
||||
prefer_cloud: Override config.PREFER_CLOUD_BACKEND for this call.
|
||||
If None, uses the config value.
|
||||
|
||||
Returns:
|
||||
PydanticAI model instance (AnthropicModel or OpenAIChatModel).
|
||||
|
||||
Example:
|
||||
>>> model = get_model()
|
||||
>>> agent = Agent(model, system_prompt="...")
|
||||
"""
|
||||
# Determine preference
|
||||
use_cloud = prefer_cloud if prefer_cloud is not None else config.PREFER_CLOUD_BACKEND
|
||||
|
||||
# Use Claude if available and preferred
|
||||
if use_cloud and is_claude_available():
|
||||
logger.debug(
|
||||
"model_selected",
|
||||
backend="claude",
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return AnthropicModel(
|
||||
model_name=config.ANTHROPIC_MODEL,
|
||||
provider=AnthropicProvider(api_key=config.ANTHROPIC_API_KEY),
|
||||
)
|
||||
|
||||
# Fall back to Ollama
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
logger.debug(
|
||||
"model_selected",
|
||||
backend="ollama",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
reason="fallback" if use_cloud else "preferred_local",
|
||||
)
|
||||
return OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
|
||||
|
||||
def get_tool_choice_settings() -> 'ModelSettings':
|
||||
"""
|
||||
Get model_settings for forcing tool calls on the first request.
|
||||
|
||||
For Claude: PydanticAI handles tool_choice natively, so no extra_body needed.
|
||||
For Ollama: Pass tool_choice="required" via extra_body to force tool calling.
|
||||
"""
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
if is_claude_available() and config.PREFER_CLOUD_BACKEND:
|
||||
# PydanticAI's Anthropic model handles tool_choice internally
|
||||
return ModelSettings()
|
||||
else:
|
||||
# Ollama needs explicit tool_choice via extra_body
|
||||
return ModelSettings(extra_body={"tool_choice": "required"})
|
||||
|
||||
|
||||
def get_model_info() -> dict:
|
||||
"""
|
||||
Get information about the current model configuration.
|
||||
|
||||
Useful for health checks and debugging.
|
||||
|
||||
Returns:
|
||||
Dict with backend, model name, and availability info.
|
||||
"""
|
||||
use_cloud = config.PREFER_CLOUD_BACKEND and is_claude_available()
|
||||
|
||||
return {
|
||||
"backend": "claude" if use_cloud else "ollama",
|
||||
"model": config.ANTHROPIC_MODEL if use_cloud else config.OLLAMA_DEFAULT_MODEL,
|
||||
"claude_available": is_claude_available(),
|
||||
"claude_configured": bool(config.ANTHROPIC_API_KEY),
|
||||
"prefer_cloud": config.PREFER_CLOUD_BACKEND,
|
||||
}
|
||||
+22
-18
@@ -7,7 +7,7 @@ import logging
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from fastapi import APIRouter
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from src.chat import service
|
||||
from src.chat.schemas import (
|
||||
@@ -22,47 +22,51 @@ router = APIRouter(prefix="/chat", tags=["chat"])
|
||||
|
||||
async def _stream_response(
|
||||
request: ChatCompletionRequest,
|
||||
) -> AsyncGenerator[dict, None]:
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Generate SSE stream for chat completion.
|
||||
|
||||
EventSourceResponse adds "data: " prefix automatically.
|
||||
We just yield the dict/string content.
|
||||
Yields raw SSE-formatted strings matching OpenAI's format exactly:
|
||||
data: {json}\n\n
|
||||
"""
|
||||
try:
|
||||
async for chunk in service.create_chat_completion_stream(request):
|
||||
# Yield dict - EventSourceResponse will format as SSE
|
||||
yield {"data": chunk.model_dump_json()}
|
||||
yield f"data: {chunk.model_dump_json(exclude_unset=True)}\n\n"
|
||||
|
||||
# Send [DONE] message
|
||||
yield {"data": "[DONE]"}
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in streaming response: {e}")
|
||||
error_data = {"error": {"message": str(e), "type": "internal_error"}}
|
||||
yield {"data": json.dumps(error_data)}
|
||||
error_data = json.dumps({"error": {"message": str(e), "type": "internal_error"}})
|
||||
yield f"data: {error_data}\n\n"
|
||||
|
||||
|
||||
@router.post("/completions", response_model=ChatCompletionResponse)
|
||||
async def create_chat_completion(
|
||||
request: ChatCompletionRequest,
|
||||
) -> ChatCompletionResponse | EventSourceResponse:
|
||||
) -> ChatCompletionResponse | StreamingResponse:
|
||||
"""
|
||||
Create chat completion (OpenAI-compatible).
|
||||
|
||||
|
||||
Supports both regular and streaming responses.
|
||||
Currently returns mock lorem ipsum responses.
|
||||
|
||||
|
||||
Args:
|
||||
request: Chat completion request
|
||||
|
||||
|
||||
Returns:
|
||||
Chat completion response or SSE stream
|
||||
"""
|
||||
logger.info(f"Chat completion request for model: {request.model}")
|
||||
|
||||
|
||||
if request.stream:
|
||||
logger.info("Streaming response requested")
|
||||
return EventSourceResponse(_stream_response(request))
|
||||
|
||||
return StreamingResponse(
|
||||
_stream_response(request),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-store",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
return await service.create_chat_completion(request)
|
||||
|
||||
@@ -1,345 +0,0 @@
|
||||
"""
|
||||
Performance benchmark storage using Redis.
|
||||
|
||||
Tracks operation timing, tool usage, and recommendation accuracy across sessions.
|
||||
Provides time-series data for performance analysis and optimization.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
import redis.asyncio as redis
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .config import config
|
||||
from .logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class PerformanceBenchmark(BaseModel):
|
||||
"""
|
||||
Performance benchmark record.
|
||||
|
||||
Stores timing and metadata for operations like Steward analysis,
|
||||
tool calls, and agent execution.
|
||||
"""
|
||||
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
operation: str # "steward_analysis", "tool_call", "tatlock_execution"
|
||||
duration_seconds: float
|
||||
success: bool
|
||||
|
||||
# Steward-specific fields
|
||||
recommendation_count: Optional[int] = None
|
||||
confidence: Optional[float] = None
|
||||
|
||||
# Tool-specific fields
|
||||
tool_name: Optional[str] = None
|
||||
was_recommended: Optional[bool] = None
|
||||
was_actually_used: Optional[bool] = None
|
||||
|
||||
# Context
|
||||
conversation_id: Optional[str] = None
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
def to_redis_dict(self) -> dict[str, Any]:
|
||||
"""Convert to dict suitable for Redis storage."""
|
||||
data = self.model_dump()
|
||||
data["timestamp"] = self.timestamp.isoformat()
|
||||
data["metadata"] = json.dumps(self.metadata)
|
||||
# Convert booleans to strings (Redis doesn't accept bool type)
|
||||
for key, value in data.items():
|
||||
if isinstance(value, bool):
|
||||
data[key] = str(value)
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def from_redis_dict(cls, data: dict[str, Any]) -> "PerformanceBenchmark":
|
||||
"""Reconstruct from Redis dict."""
|
||||
data["timestamp"] = datetime.fromisoformat(data["timestamp"])
|
||||
data["metadata"] = json.loads(data.get("metadata", "{}"))
|
||||
# Convert string booleans back to bool
|
||||
for key in ["success", "was_recommended", "was_actually_used"]:
|
||||
if key in data and isinstance(data[key], str):
|
||||
data[key] = data[key] == "True"
|
||||
return cls(**data)
|
||||
|
||||
|
||||
class BenchmarkStore:
|
||||
"""
|
||||
Redis-backed benchmark storage with automatic expiry.
|
||||
|
||||
Stores performance metrics in time-series format with 30-day retention.
|
||||
Provides querying capabilities for analysis and reporting.
|
||||
"""
|
||||
|
||||
def __init__(self, redis_client: Optional[redis.Redis] = None):
|
||||
"""
|
||||
Initialize benchmark store.
|
||||
|
||||
Args:
|
||||
redis_client: Optional Redis client. If None, creates from config.
|
||||
"""
|
||||
self._client = redis_client
|
||||
self._ttl_days = 30 # 30-day retention
|
||||
|
||||
async def _get_client(self) -> redis.Redis:
|
||||
"""Get or create Redis client."""
|
||||
if self._client is None:
|
||||
self._client = redis.from_url(
|
||||
config.redis_url,
|
||||
encoding="utf-8",
|
||||
decode_responses=True,
|
||||
socket_timeout=config.REDIS_TIMEOUT,
|
||||
socket_connect_timeout=config.REDIS_TIMEOUT,
|
||||
)
|
||||
return self._client
|
||||
|
||||
async def record(self, benchmark: PerformanceBenchmark) -> None:
|
||||
"""
|
||||
Record a performance benchmark.
|
||||
|
||||
Args:
|
||||
benchmark: Performance benchmark to record
|
||||
|
||||
Example:
|
||||
>>> await store.record(PerformanceBenchmark(
|
||||
... operation="steward_analysis",
|
||||
... duration_seconds=1.23,
|
||||
... success=True,
|
||||
... recommendation_count=3,
|
||||
... ))
|
||||
"""
|
||||
if not config.ENABLE_BENCHMARKS:
|
||||
return
|
||||
|
||||
try:
|
||||
client = await self._get_client()
|
||||
|
||||
# Generate key: benchmark:{operation}:{timestamp_ms}
|
||||
timestamp_ms = int(benchmark.timestamp.timestamp() * 1000)
|
||||
key = f"benchmark:{benchmark.operation}:{timestamp_ms}"
|
||||
|
||||
# Store as hash
|
||||
await client.hset(key, mapping=benchmark.to_redis_dict())
|
||||
|
||||
# Set expiry
|
||||
await client.expire(key, self._ttl_days * 24 * 60 * 60)
|
||||
|
||||
# Add to sorted set for time-based queries
|
||||
index_key = f"benchmark_index:{benchmark.operation}"
|
||||
await client.zadd(index_key, {key: timestamp_ms})
|
||||
await client.expire(index_key, self._ttl_days * 24 * 60 * 60)
|
||||
|
||||
logger.debug(
|
||||
"benchmark_recorded",
|
||||
operation=benchmark.operation,
|
||||
duration=benchmark.duration_seconds,
|
||||
success=benchmark.success,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"benchmark_recording_failed",
|
||||
error=str(e),
|
||||
operation=benchmark.operation,
|
||||
)
|
||||
# Don't fail the request if benchmarking fails
|
||||
|
||||
async def query(
|
||||
self,
|
||||
operation: str,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
limit: int = 100,
|
||||
) -> list[PerformanceBenchmark]:
|
||||
"""
|
||||
Query benchmarks by operation and time range.
|
||||
|
||||
Args:
|
||||
operation: Operation name to filter by
|
||||
start_time: Start of time range (inclusive)
|
||||
end_time: End of time range (inclusive)
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of benchmarks matching the query
|
||||
|
||||
Example:
|
||||
>>> from datetime import timedelta
|
||||
>>> now = datetime.now(timezone.utc)
|
||||
>>> yesterday = now - timedelta(days=1)
|
||||
>>> benchmarks = await store.query(
|
||||
... "steward_analysis",
|
||||
... start_time=yesterday,
|
||||
... limit=50
|
||||
... )
|
||||
"""
|
||||
if not config.ENABLE_BENCHMARKS:
|
||||
return []
|
||||
|
||||
try:
|
||||
client = await self._get_client()
|
||||
index_key = f"benchmark_index:{operation}"
|
||||
|
||||
# Convert time range to timestamps
|
||||
min_score = (
|
||||
int(start_time.timestamp() * 1000)
|
||||
if start_time
|
||||
else "-inf"
|
||||
)
|
||||
max_score = (
|
||||
int(end_time.timestamp() * 1000)
|
||||
if end_time
|
||||
else "+inf"
|
||||
)
|
||||
|
||||
# Query sorted set
|
||||
keys = await client.zrevrangebyscore(
|
||||
index_key,
|
||||
max_score,
|
||||
min_score,
|
||||
start=0,
|
||||
num=limit,
|
||||
)
|
||||
|
||||
# Fetch benchmark data
|
||||
benchmarks = []
|
||||
for key in keys:
|
||||
data = await client.hgetall(key)
|
||||
if data:
|
||||
benchmarks.append(PerformanceBenchmark.from_redis_dict(data))
|
||||
|
||||
return benchmarks
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"benchmark_query_failed",
|
||||
error=str(e),
|
||||
operation=operation,
|
||||
)
|
||||
return []
|
||||
|
||||
async def get_statistics(
|
||||
self,
|
||||
operation: str,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Get aggregate statistics for an operation.
|
||||
|
||||
Args:
|
||||
operation: Operation name
|
||||
start_time: Start of time range
|
||||
end_time: End of time range
|
||||
|
||||
Returns:
|
||||
Dictionary with statistics (count, avg_duration, success_rate, etc.)
|
||||
|
||||
Example:
|
||||
>>> stats = await store.get_statistics("steward_analysis")
|
||||
>>> print(f"Average duration: {stats['avg_duration']}s")
|
||||
>>> print(f"Success rate: {stats['success_rate']}%")
|
||||
"""
|
||||
benchmarks = await self.query(operation, start_time, end_time, limit=1000)
|
||||
|
||||
if not benchmarks:
|
||||
return {
|
||||
"count": 0,
|
||||
"avg_duration": 0.0,
|
||||
"min_duration": 0.0,
|
||||
"max_duration": 0.0,
|
||||
"success_rate": 0.0,
|
||||
}
|
||||
|
||||
durations = [b.duration_seconds for b in benchmarks]
|
||||
successes = sum(1 for b in benchmarks if b.success)
|
||||
|
||||
return {
|
||||
"count": len(benchmarks),
|
||||
"avg_duration": sum(durations) / len(durations),
|
||||
"min_duration": min(durations),
|
||||
"max_duration": max(durations),
|
||||
"success_rate": (successes / len(benchmarks)) * 100,
|
||||
"total_successes": successes,
|
||||
"total_failures": len(benchmarks) - successes,
|
||||
}
|
||||
|
||||
async def get_tool_accuracy(
|
||||
self,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Analyze tool recommendation accuracy.
|
||||
|
||||
Compares recommended tools vs actually used tools to measure
|
||||
Steward's recommendation precision.
|
||||
|
||||
Args:
|
||||
start_time: Start of time range
|
||||
end_time: End of time range
|
||||
|
||||
Returns:
|
||||
Dictionary with accuracy metrics
|
||||
|
||||
Example:
|
||||
>>> accuracy = await store.get_tool_accuracy()
|
||||
>>> print(f"Precision: {accuracy['precision']}%")
|
||||
"""
|
||||
tool_calls = await self.query("tool_call", start_time, end_time, limit=1000)
|
||||
|
||||
if not tool_calls:
|
||||
return {
|
||||
"total_calls": 0,
|
||||
"recommended_and_used": 0,
|
||||
"recommended_not_used": 0,
|
||||
"not_recommended_but_used": 0,
|
||||
"precision": 0.0,
|
||||
}
|
||||
|
||||
recommended_and_used = sum(
|
||||
1 for b in tool_calls
|
||||
if b.was_recommended and b.was_actually_used
|
||||
)
|
||||
not_recommended_but_used = sum(
|
||||
1 for b in tool_calls
|
||||
if not b.was_recommended and b.was_actually_used
|
||||
)
|
||||
|
||||
total_used = sum(1 for b in tool_calls if b.was_actually_used)
|
||||
precision = (
|
||||
(recommended_and_used / total_used * 100) if total_used > 0 else 0.0
|
||||
)
|
||||
|
||||
return {
|
||||
"total_calls": len(tool_calls),
|
||||
"total_used": total_used,
|
||||
"recommended_and_used": recommended_and_used,
|
||||
"not_recommended_but_used": not_recommended_but_used,
|
||||
"precision": precision,
|
||||
}
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close Redis connection."""
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
|
||||
# Global benchmark store instance
|
||||
_benchmark_store: Optional[BenchmarkStore] = None
|
||||
|
||||
|
||||
def get_benchmark_store() -> BenchmarkStore:
|
||||
"""
|
||||
Get global benchmark store instance.
|
||||
|
||||
Returns:
|
||||
BenchmarkStore instance
|
||||
"""
|
||||
global _benchmark_store
|
||||
if _benchmark_store is None:
|
||||
_benchmark_store = BenchmarkStore()
|
||||
return _benchmark_store
|
||||
+17
-13
@@ -64,13 +64,27 @@ class Config(BaseSettings):
|
||||
API_PORT: int = Field(default=8000, description="API port")
|
||||
API_PREFIX: str = Field(default="/v1", description="API route prefix")
|
||||
|
||||
# Ollama Configuration
|
||||
# Anthropic Configuration (Claude - preferred backend)
|
||||
ANTHROPIC_API_KEY: str | None = Field(
|
||||
default=None,
|
||||
description="Anthropic API key for Claude access"
|
||||
)
|
||||
ANTHROPIC_MODEL: str = Field(
|
||||
default="claude-sonnet-4-20250514",
|
||||
description="Claude model to use"
|
||||
)
|
||||
PREFER_CLOUD_BACKEND: bool = Field(
|
||||
default=True,
|
||||
description="Prefer Claude over Ollama when available"
|
||||
)
|
||||
|
||||
# Ollama Configuration (local fallback)
|
||||
OLLAMA_HOST: HttpUrl = Field(
|
||||
default="http://localhost:11434",
|
||||
description="Ollama server URL"
|
||||
)
|
||||
OLLAMA_DEFAULT_MODEL: str = Field(
|
||||
default="mistral-nemo:latest",
|
||||
default="gemma4:e2b",
|
||||
description="Default Ollama model"
|
||||
)
|
||||
OLLAMA_TIMEOUT: int = Field(
|
||||
@@ -101,10 +115,6 @@ class Config(BaseSettings):
|
||||
default=6379,
|
||||
description="Redis server port"
|
||||
)
|
||||
REDIS_BENCHMARK_DB: int = Field(
|
||||
default=6,
|
||||
description="Redis database number for benchmarks"
|
||||
)
|
||||
REDIS_TIMEOUT: int = Field(
|
||||
default=5,
|
||||
description="Redis connection timeout in seconds"
|
||||
@@ -158,7 +168,7 @@ class Config(BaseSettings):
|
||||
description="Ollama model for embeddings"
|
||||
)
|
||||
|
||||
# Redis Memory Database (separate from benchmarks)
|
||||
# Redis Memory Database
|
||||
REDIS_MEMORY_DB: int = Field(
|
||||
default=1,
|
||||
description="Redis database number for memory cache"
|
||||
@@ -173,7 +183,6 @@ class Config(BaseSettings):
|
||||
default=None,
|
||||
description="Logging level (auto-set based on environment if not specified)"
|
||||
)
|
||||
ENABLE_BENCHMARKS: bool = Field(default=True, description="Enable performance benchmarking")
|
||||
|
||||
# User Configuration
|
||||
DEFAULT_USER: str | None = Field(
|
||||
@@ -190,11 +199,6 @@ class Config(BaseSettings):
|
||||
CORS_ALLOW_METHODS: list[str] = ["*"]
|
||||
CORS_ALLOW_HEADERS: list[str] = ["*"]
|
||||
|
||||
@property
|
||||
def redis_url(self) -> str:
|
||||
"""Construct Redis connection URL for benchmarks."""
|
||||
return f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/{self.REDIS_BENCHMARK_DB}"
|
||||
|
||||
@property
|
||||
def redis_memory_url(self) -> str:
|
||||
"""Construct Redis connection URL for memory cache."""
|
||||
|
||||
@@ -6,7 +6,7 @@ Provides short-term memory storage with TTL:
|
||||
- Recent entities mentioned in conversation
|
||||
- User-scoped with conversation isolation
|
||||
|
||||
Uses Redis DB 2 (separate from benchmarks in DB 1).
|
||||
Uses Redis DB 1.
|
||||
"""
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
@@ -11,6 +11,7 @@ from src.agents.steward import analyze_request, format_steward_note
|
||||
from src.agents.steward.schemas import StewardRecommendation
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import trace_span, SpanType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -93,12 +94,32 @@ async def preprocess_request(
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Call Steward with full conversation history
|
||||
recommendation = await analyze_request(
|
||||
enriched_request,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
# Call Steward with full conversation history (traced)
|
||||
async with trace_span(
|
||||
"steward_analysis",
|
||||
SpanType.STEWARD,
|
||||
metadata={
|
||||
"request_preview": user_request[:100],
|
||||
"history_length": len(conversation_history),
|
||||
},
|
||||
) as span:
|
||||
recommendation = await analyze_request(
|
||||
enriched_request,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Update span with results
|
||||
if span:
|
||||
span.metadata.update({
|
||||
"recommended_capabilities": recommendation.recommended_capabilities,
|
||||
"complexity": recommendation.estimated_complexity,
|
||||
"has_memory_context": bool(recommendation.memory_context),
|
||||
"has_conversation_context": recommendation.conversation_context.has_previous_context,
|
||||
})
|
||||
span.details["reasoning"] = recommendation.reasoning
|
||||
if recommendation.enriched_query:
|
||||
span.details["enriched_query"] = recommendation.enriched_query
|
||||
|
||||
# Format note for Tatlock (includes conversation context)
|
||||
steward_note = await format_steward_note(recommendation)
|
||||
|
||||
+15
-4
@@ -9,6 +9,7 @@ from src.agents.biographer import register_biographer
|
||||
from src.agents.housekeeper import register_housekeeper
|
||||
from src.agents.librarian import register_librarian
|
||||
from src.agents.tatlock_core import TATLOCK_CORE_CAPABILITY, tatlock_core_tools
|
||||
from src.anthropic.model_selector import check_claude_health, get_model_info
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
@@ -81,19 +82,29 @@ def register_household_members():
|
||||
)
|
||||
|
||||
|
||||
def initialize_application():
|
||||
async def initialize_application():
|
||||
"""
|
||||
Initialize the application.
|
||||
|
||||
Performs all startup tasks:
|
||||
1. Register household members
|
||||
2. (Future) Initialize connections
|
||||
3. (Future) Load configuration
|
||||
1. Check Claude API health (for backend selection)
|
||||
2. Register household members
|
||||
3. (Future) Initialize connections
|
||||
|
||||
This should be called once during application startup.
|
||||
"""
|
||||
logger.info("application_initialization_starting")
|
||||
|
||||
# Check Claude API health for backend selection
|
||||
await check_claude_health()
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"model_backend_configured",
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
claude_available=model_info["claude_available"],
|
||||
)
|
||||
|
||||
# Register household members
|
||||
register_household_members()
|
||||
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
"""
|
||||
Tool call tracking and benchmarking.
|
||||
Tool call tracking.
|
||||
|
||||
Tracks which tools are recommended by the Steward versus which tools
|
||||
are actually used by Tatlock, recording benchmarks for analysis.
|
||||
are actually used by Tatlock for debugging and analysis.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from src.core.benchmarks import PerformanceBenchmark, get_benchmark_store
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -15,7 +13,7 @@ logger = get_logger(__name__)
|
||||
|
||||
class ToolCallTracker:
|
||||
"""
|
||||
Tracks tool calls for benchmarking and accuracy analysis.
|
||||
Tracks tool calls for accuracy analysis.
|
||||
|
||||
Compares Steward's recommendations with Tatlock's actual tool usage
|
||||
to measure recommendation accuracy.
|
||||
@@ -53,6 +51,10 @@ class ToolCallTracker:
|
||||
return tool_name.replace("delegate_to_", "")
|
||||
return tool_name
|
||||
|
||||
def log_call(self, message: str):
|
||||
"""Log a tool call message (for UI display)."""
|
||||
logger.debug("tool_call_message", message=message)
|
||||
|
||||
async def track_call(self, tool_name: str, duration: float):
|
||||
"""
|
||||
Record a tool call with timing.
|
||||
@@ -78,23 +80,6 @@ class ToolCallTracker:
|
||||
recommended=list(self.recommended_capabilities),
|
||||
)
|
||||
|
||||
# Record benchmark to Redis
|
||||
benchmark = PerformanceBenchmark(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
operation="tool_call",
|
||||
duration_seconds=duration,
|
||||
success=True, # If we got here, the call succeeded
|
||||
tool_name=tool_name,
|
||||
was_recommended=was_recommended,
|
||||
was_actually_used=True,
|
||||
conversation_id=self.conversation_id,
|
||||
metadata={
|
||||
"recommended_capabilities": list(self.recommended_capabilities),
|
||||
},
|
||||
)
|
||||
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
logger.debug(
|
||||
"tool_call_tracked",
|
||||
tool_name=tool_name,
|
||||
@@ -124,24 +109,6 @@ class ToolCallTracker:
|
||||
conversation_id=self.conversation_id,
|
||||
)
|
||||
|
||||
# Record benchmarks for unused recommendations
|
||||
for tool_name in unused_tools:
|
||||
benchmark = PerformanceBenchmark(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
operation="tool_call",
|
||||
duration_seconds=0.0, # Not used
|
||||
success=True,
|
||||
tool_name=tool_name,
|
||||
was_recommended=True,
|
||||
was_actually_used=False,
|
||||
conversation_id=self.conversation_id,
|
||||
metadata={
|
||||
"recommended_capabilities": list(self.recommended_capabilities),
|
||||
"reason": "recommended_but_unused",
|
||||
},
|
||||
)
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
# Log summary
|
||||
total_calls = sum(len(durations) for durations in self.actual_calls.values())
|
||||
logger.info(
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
"""
|
||||
Lightweight request tracing for local development.
|
||||
|
||||
Captures the full request flow through Tatlock's multi-agent architecture
|
||||
as structured JSON traces for debugging and optimization.
|
||||
|
||||
Enable via DEBUG=true environment variable.
|
||||
|
||||
Traces are written to logs/traces/{trace_id}.json
|
||||
View with logs/traces/viewer.html
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import json
|
||||
import secrets
|
||||
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class SpanType(str, Enum):
|
||||
"""Types of traced operations."""
|
||||
ROUTER = "router"
|
||||
STEWARD = "steward"
|
||||
TATLOCK = "tatlock"
|
||||
EXPERT = "expert"
|
||||
TOOL = "tool"
|
||||
|
||||
|
||||
class SpanStatus(str, Enum):
|
||||
"""Span completion status."""
|
||||
OK = "ok"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Span:
|
||||
"""A single traced operation."""
|
||||
span_id: str
|
||||
name: str
|
||||
type: SpanType
|
||||
start_time: datetime
|
||||
parent_id: str | None = None
|
||||
end_time: datetime | None = None
|
||||
status: SpanStatus = SpanStatus.OK
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
children: list[str] = field(default_factory=list)
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def duration_ms(self) -> float | None:
|
||||
"""Calculate duration in milliseconds."""
|
||||
if self.end_time and self.start_time:
|
||||
return (self.end_time - self.start_time).total_seconds() * 1000
|
||||
return None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert span to dictionary for JSON serialization."""
|
||||
result = {
|
||||
"span_id": self.span_id,
|
||||
"parent_id": self.parent_id,
|
||||
"name": self.name,
|
||||
"type": self.type.value,
|
||||
"start_time": self.start_time.isoformat(),
|
||||
"end_time": self.end_time.isoformat() if self.end_time else None,
|
||||
"duration_ms": round(self.duration_ms, 2) if self.duration_ms else None,
|
||||
"status": self.status.value,
|
||||
"metadata": self.metadata if self.metadata else None,
|
||||
}
|
||||
# Only include non-empty optional fields
|
||||
if self.details:
|
||||
result["details"] = self.details
|
||||
if self.children:
|
||||
result["children"] = self.children
|
||||
if self.error:
|
||||
result["error"] = self.error
|
||||
return {k: v for k, v in result.items() if v is not None}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Trace:
|
||||
"""Complete trace of a request."""
|
||||
trace_id: str
|
||||
conversation_id: str | None
|
||||
user: str
|
||||
timestamp: datetime
|
||||
request: dict[str, Any]
|
||||
spans: list[Span] = field(default_factory=list)
|
||||
response: dict[str, Any] | None = None
|
||||
status: str = "in_progress"
|
||||
|
||||
@property
|
||||
def total_duration_ms(self) -> float | None:
|
||||
"""Calculate total trace duration from span timings."""
|
||||
if not self.spans:
|
||||
return None
|
||||
start = min(s.start_time for s in self.spans)
|
||||
ends = [s.end_time for s in self.spans if s.end_time]
|
||||
if not ends:
|
||||
return None
|
||||
end = max(ends)
|
||||
return (end - start).total_seconds() * 1000
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert trace to dictionary for JSON serialization."""
|
||||
return {
|
||||
"trace_id": self.trace_id,
|
||||
"conversation_id": self.conversation_id,
|
||||
"user": self.user,
|
||||
"timestamp": self.timestamp.isoformat(),
|
||||
"total_duration_ms": round(self.total_duration_ms, 2) if self.total_duration_ms else None,
|
||||
"status": self.status,
|
||||
"request": self.request,
|
||||
"response": self.response,
|
||||
"spans": [s.to_dict() for s in self.spans],
|
||||
}
|
||||
|
||||
|
||||
# ContextVar for async-safe trace propagation
|
||||
_current_trace: ContextVar[Trace | None] = ContextVar("current_trace", default=None)
|
||||
_current_span: ContextVar[Span | None] = ContextVar("current_span", default=None)
|
||||
|
||||
|
||||
def tracing_enabled() -> bool:
|
||||
"""Check if tracing is enabled (requires DEBUG=true)."""
|
||||
from src.core.config import config
|
||||
return config.DEBUG
|
||||
|
||||
|
||||
def _generate_id(prefix: str = "") -> str:
|
||||
"""Generate unique ID with optional prefix."""
|
||||
return f"{prefix}{secrets.token_hex(8)}"
|
||||
|
||||
|
||||
def start_trace(
|
||||
conversation_id: str | None,
|
||||
user: str,
|
||||
request: dict[str, Any],
|
||||
) -> Trace | None:
|
||||
"""
|
||||
Start a new trace for a request.
|
||||
|
||||
Args:
|
||||
conversation_id: Conversation identifier
|
||||
user: User identifier
|
||||
request: Request data (should include preview and full)
|
||||
|
||||
Returns:
|
||||
Trace object if tracing enabled, None otherwise
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
return None
|
||||
|
||||
trace = Trace(
|
||||
trace_id=_generate_id("trace_"),
|
||||
conversation_id=conversation_id,
|
||||
user=user,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
request=request,
|
||||
)
|
||||
_current_trace.set(trace)
|
||||
|
||||
logger.debug("trace_started", trace_id=trace.trace_id, user=user)
|
||||
return trace
|
||||
|
||||
|
||||
def get_current_trace() -> Trace | None:
|
||||
"""Get the current trace from context."""
|
||||
return _current_trace.get()
|
||||
|
||||
|
||||
def get_current_span() -> Span | None:
|
||||
"""Get the current span from context."""
|
||||
return _current_span.get()
|
||||
|
||||
|
||||
def start_span(
|
||||
name: str,
|
||||
span_type: SpanType,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> Span | None:
|
||||
"""
|
||||
Start a new span within the current trace.
|
||||
|
||||
Args:
|
||||
name: Span name (e.g., "steward_analysis")
|
||||
span_type: Type of operation
|
||||
metadata: Quick-access metadata (shown in timeline)
|
||||
details: Expandable details (prompts, full responses)
|
||||
|
||||
Returns:
|
||||
Span object if tracing enabled, None otherwise
|
||||
"""
|
||||
trace = get_current_trace()
|
||||
if not trace:
|
||||
return None
|
||||
|
||||
parent = get_current_span()
|
||||
span = Span(
|
||||
span_id=_generate_id("span_"),
|
||||
name=name,
|
||||
type=span_type,
|
||||
start_time=datetime.now(timezone.utc),
|
||||
parent_id=parent.span_id if parent else None,
|
||||
metadata=metadata or {},
|
||||
details=details or {},
|
||||
)
|
||||
|
||||
# Add to parent's children list
|
||||
if parent:
|
||||
parent.children.append(span.span_id)
|
||||
|
||||
trace.spans.append(span)
|
||||
_current_span.set(span)
|
||||
|
||||
logger.debug(
|
||||
"span_started",
|
||||
span_id=span.span_id,
|
||||
name=name,
|
||||
type=span_type.value,
|
||||
parent_id=span.parent_id,
|
||||
)
|
||||
return span
|
||||
|
||||
|
||||
def end_span(
|
||||
span: Span | None = None,
|
||||
status: SpanStatus = SpanStatus.OK,
|
||||
metadata_update: dict[str, Any] | None = None,
|
||||
details_update: dict[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
End a span and restore parent as current.
|
||||
|
||||
Args:
|
||||
span: Span to end (defaults to current span)
|
||||
status: Completion status
|
||||
metadata_update: Additional metadata to merge
|
||||
details_update: Additional details to merge
|
||||
error: Error message if failed
|
||||
"""
|
||||
if span is None:
|
||||
span = get_current_span()
|
||||
if not span:
|
||||
return
|
||||
|
||||
span.end_time = datetime.now(timezone.utc)
|
||||
span.status = status
|
||||
if error:
|
||||
span.error = error
|
||||
span.status = SpanStatus.ERROR
|
||||
if metadata_update:
|
||||
span.metadata.update(metadata_update)
|
||||
if details_update:
|
||||
span.details.update(details_update)
|
||||
|
||||
# Restore parent span as current
|
||||
trace = get_current_trace()
|
||||
if trace and span.parent_id:
|
||||
parent = next((s for s in trace.spans if s.span_id == span.parent_id), None)
|
||||
_current_span.set(parent)
|
||||
else:
|
||||
_current_span.set(None)
|
||||
|
||||
logger.debug(
|
||||
"span_ended",
|
||||
span_id=span.span_id,
|
||||
duration_ms=span.duration_ms,
|
||||
status=status.value,
|
||||
)
|
||||
|
||||
|
||||
def end_trace(
|
||||
response: dict[str, Any] | None = None,
|
||||
status: str = "completed",
|
||||
) -> str | None:
|
||||
"""
|
||||
End the current trace and write to file.
|
||||
|
||||
Args:
|
||||
response: Response data to include
|
||||
status: Final trace status ("completed" or "error")
|
||||
|
||||
Returns:
|
||||
Path to trace file if written, None otherwise
|
||||
"""
|
||||
trace = get_current_trace()
|
||||
if not trace:
|
||||
return None
|
||||
|
||||
trace.response = response
|
||||
trace.status = status
|
||||
|
||||
# Write trace to file
|
||||
trace_path = _write_trace(trace)
|
||||
|
||||
# Clear context
|
||||
_current_trace.set(None)
|
||||
_current_span.set(None)
|
||||
|
||||
logger.info(
|
||||
"trace_completed",
|
||||
trace_id=trace.trace_id,
|
||||
total_duration_ms=round(trace.total_duration_ms, 2) if trace.total_duration_ms else None,
|
||||
span_count=len(trace.spans),
|
||||
path=str(trace_path) if trace_path else None,
|
||||
)
|
||||
|
||||
return str(trace_path) if trace_path else None
|
||||
|
||||
|
||||
def _write_trace(trace: Trace) -> Path | None:
|
||||
"""Write trace to JSON file."""
|
||||
try:
|
||||
# Ensure traces directory exists
|
||||
traces_dir = Path("logs/traces")
|
||||
traces_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write trace file
|
||||
trace_path = traces_dir / f"{trace.trace_id}.json"
|
||||
with open(trace_path, "w") as f:
|
||||
json.dump(trace.to_dict(), f, indent=2, default=str)
|
||||
|
||||
return trace_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error("trace_write_failed", error=str(e), trace_id=trace.trace_id)
|
||||
return None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def trace_span(
|
||||
name: str,
|
||||
span_type: SpanType,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
):
|
||||
"""
|
||||
Async context manager for tracing a span.
|
||||
|
||||
Automatically handles start/end timing and error capture.
|
||||
|
||||
Usage:
|
||||
async with trace_span("steward_analysis", SpanType.STEWARD) as span:
|
||||
result = await analyze_request(...)
|
||||
if span:
|
||||
span.metadata["result_count"] = len(result)
|
||||
|
||||
Args:
|
||||
name: Span name
|
||||
span_type: Type of operation
|
||||
metadata: Initial metadata
|
||||
details: Initial details (expandable in viewer)
|
||||
|
||||
Yields:
|
||||
Span object or None if tracing disabled
|
||||
"""
|
||||
span = start_span(name, span_type, metadata, details)
|
||||
try:
|
||||
yield span
|
||||
except Exception as e:
|
||||
end_span(span, SpanStatus.ERROR, error=str(e))
|
||||
raise
|
||||
else:
|
||||
end_span(span, SpanStatus.OK)
|
||||
|
||||
|
||||
def add_tool_spans_from_messages(messages: list[Any], parent_span: Span | None = None) -> None:
|
||||
"""
|
||||
Extract tool calls from PydanticAI result messages and add as child spans.
|
||||
|
||||
Call this after an agent.run() to capture tool-level timing retroactively.
|
||||
Note: Since we don't have actual timing, we estimate based on sequence.
|
||||
|
||||
Args:
|
||||
messages: List from result.new_messages()
|
||||
parent_span: Parent span to attach tool spans to
|
||||
"""
|
||||
trace = get_current_trace()
|
||||
if not trace or not parent_span:
|
||||
return
|
||||
|
||||
# Import PydanticAI message types
|
||||
try:
|
||||
from pydantic_ai.messages import ModelRequest, ModelResponse, ToolCallPart, ToolReturnPart
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
# Track tool calls and their returns
|
||||
tool_calls: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for msg in messages:
|
||||
if isinstance(msg, ModelResponse):
|
||||
for part in msg.parts:
|
||||
if isinstance(part, ToolCallPart):
|
||||
tool_calls[part.tool_call_id] = {
|
||||
"name": part.tool_name,
|
||||
"args": part.args if hasattr(part, 'args') else {},
|
||||
}
|
||||
elif isinstance(msg, ModelRequest):
|
||||
for part in msg.parts:
|
||||
if isinstance(part, ToolReturnPart):
|
||||
if part.tool_call_id in tool_calls:
|
||||
tool_info = tool_calls[part.tool_call_id]
|
||||
# Create a span for this tool call
|
||||
span = Span(
|
||||
span_id=_generate_id("span_"),
|
||||
name=tool_info["name"],
|
||||
type=SpanType.TOOL,
|
||||
start_time=parent_span.start_time, # Approximate
|
||||
end_time=parent_span.end_time or datetime.now(timezone.utc),
|
||||
parent_id=parent_span.span_id,
|
||||
status=SpanStatus.OK,
|
||||
metadata={
|
||||
"tool_name": tool_info["name"],
|
||||
"args_preview": str(tool_info.get("args", {}))[:100],
|
||||
},
|
||||
details={
|
||||
"args": tool_info.get("args", {}),
|
||||
"result": part.content[:2000] if isinstance(part.content, str) else str(part.content)[:2000],
|
||||
},
|
||||
)
|
||||
parent_span.children.append(span.span_id)
|
||||
trace.spans.append(span)
|
||||
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
Trace viewer router.
|
||||
|
||||
Serves the trace viewer UI and trace files when tracing is enabled.
|
||||
Only available when DEBUG=true.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/traces", tags=["traces"])
|
||||
|
||||
TRACES_DIR = Path("logs/traces")
|
||||
VIEWER_PATH = TRACES_DIR / "viewer.html"
|
||||
|
||||
|
||||
def tracing_enabled() -> bool:
|
||||
"""Check if tracing is enabled."""
|
||||
return config.DEBUG
|
||||
|
||||
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def get_trace_viewer():
|
||||
"""
|
||||
Serve the trace viewer UI.
|
||||
|
||||
Returns the standalone HTML viewer for browsing traces.
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
raise HTTPException(status_code=404, detail="Tracing not enabled")
|
||||
|
||||
if not VIEWER_PATH.exists():
|
||||
raise HTTPException(status_code=404, detail="Viewer not found")
|
||||
|
||||
return HTMLResponse(content=VIEWER_PATH.read_text())
|
||||
|
||||
|
||||
@router.get("/list")
|
||||
async def list_traces(
|
||||
limit: int = 50,
|
||||
since_minutes: int | None = None,
|
||||
status: str | None = None,
|
||||
search: str | None = None,
|
||||
):
|
||||
"""
|
||||
List available trace files.
|
||||
|
||||
Returns most recent traces first, with basic metadata.
|
||||
|
||||
Args:
|
||||
limit: Maximum number of traces to return (default 50)
|
||||
since_minutes: Only return traces from the last N minutes
|
||||
status: Filter by status (completed, error, streaming)
|
||||
search: Search in request preview text
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
raise HTTPException(status_code=404, detail="Tracing not enabled")
|
||||
|
||||
if not TRACES_DIR.exists():
|
||||
return {"traces": [], "total": 0}
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
# Calculate cutoff time if filtering by time
|
||||
cutoff_time = None
|
||||
if since_minutes:
|
||||
cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
|
||||
|
||||
# Get all trace files, sorted by modification time (newest first)
|
||||
trace_files = sorted(
|
||||
TRACES_DIR.glob("trace_*.json"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
traces = []
|
||||
for path in trace_files:
|
||||
if len(traces) >= limit:
|
||||
break
|
||||
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Parse timestamp for filtering
|
||||
trace_timestamp = data.get("timestamp")
|
||||
if cutoff_time and trace_timestamp:
|
||||
try:
|
||||
ts = datetime.fromisoformat(trace_timestamp.replace('Z', '+00:00'))
|
||||
if ts < cutoff_time:
|
||||
continue
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Filter by status
|
||||
trace_status = data.get("status", "")
|
||||
if status and trace_status != status:
|
||||
continue
|
||||
|
||||
# Filter by search text
|
||||
request_preview = data.get("request", {}).get("input_preview", "")
|
||||
if search and search.lower() not in request_preview.lower():
|
||||
continue
|
||||
|
||||
traces.append({
|
||||
"trace_id": data.get("trace_id"),
|
||||
"timestamp": trace_timestamp,
|
||||
"user": data.get("user"),
|
||||
"status": trace_status,
|
||||
"total_duration_ms": data.get("total_duration_ms"),
|
||||
"span_count": len(data.get("spans", [])),
|
||||
"request_preview": request_preview[:100],
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning("trace_list_parse_error", path=str(path), error=str(e))
|
||||
|
||||
return {"traces": traces, "total": len(traces)}
|
||||
|
||||
|
||||
@router.get("/{trace_id}")
|
||||
async def get_trace(trace_id: str):
|
||||
"""
|
||||
Get a specific trace by ID.
|
||||
|
||||
Returns the full trace JSON.
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
raise HTTPException(status_code=404, detail="Tracing not enabled")
|
||||
|
||||
# Sanitize trace_id to prevent path traversal
|
||||
if not trace_id.startswith("trace_") or "/" in trace_id or "\\" in trace_id:
|
||||
raise HTTPException(status_code=400, detail="Invalid trace ID")
|
||||
|
||||
trace_path = TRACES_DIR / f"{trace_id}.json"
|
||||
|
||||
if not trace_path.exists():
|
||||
raise HTTPException(status_code=404, detail="Trace not found")
|
||||
|
||||
try:
|
||||
import json
|
||||
with open(trace_path) as f:
|
||||
data = json.load(f)
|
||||
return JSONResponse(content=data)
|
||||
except Exception as e:
|
||||
logger.error("trace_read_error", trace_id=trace_id, error=str(e))
|
||||
raise HTTPException(status_code=500, detail="Failed to read trace")
|
||||
+12
-4
@@ -23,6 +23,7 @@ from src.core.exceptions import AppException
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.router import router as core_router
|
||||
from src.core.startup import initialize_application
|
||||
from src.core.tracing_router import router as tracing_router
|
||||
from src.models.router import router as models_router
|
||||
from src.responses.router import router as responses_router
|
||||
|
||||
@@ -43,14 +44,16 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app_name=config.APP_NAME,
|
||||
version=config.APP_VERSION,
|
||||
environment=config.ENVIRONMENT.value,
|
||||
prefer_cloud=config.PREFER_CLOUD_BACKEND,
|
||||
anthropic_model=config.ANTHROPIC_MODEL,
|
||||
ollama_host=str(config.OLLAMA_HOST),
|
||||
ollama_model=config.OLLAMA_DEFAULT_MODEL,
|
||||
redis_url=config.redis_url,
|
||||
redis_url=config.redis_memory_url,
|
||||
log_format=config.log_format,
|
||||
)
|
||||
|
||||
# Initialize application (register household members, etc.)
|
||||
initialize_application()
|
||||
# Initialize application (check Claude health, register household members, etc.)
|
||||
await initialize_application()
|
||||
|
||||
yield
|
||||
|
||||
@@ -90,7 +93,12 @@ def create_application() -> FastAPI:
|
||||
application.include_router(chat_router, prefix=config.API_PREFIX)
|
||||
application.include_router(models_router, prefix=config.API_PREFIX)
|
||||
application.include_router(responses_router, prefix=config.API_PREFIX) # Responses API
|
||||
|
||||
|
||||
# Conditionally include tracing router (only in debug mode)
|
||||
if config.DEBUG:
|
||||
application.include_router(tracing_router)
|
||||
logger.info("tracing_router_enabled")
|
||||
|
||||
return application
|
||||
|
||||
|
||||
|
||||
+5
-73
@@ -10,7 +10,6 @@ from sse_starlette.sse import EventSourceResponse
|
||||
from src.responses import service
|
||||
from src.responses.schemas import ResponseRequest, Response
|
||||
from src.core.exceptions import ModelNotFoundError, AppException
|
||||
from src.core.context import current_user, current_conversation, get_default_user
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -37,77 +36,16 @@ async def create_response(
|
||||
|
||||
Returns:
|
||||
Response object or SSE stream
|
||||
|
||||
Example non-streaming request:
|
||||
POST /v1/responses
|
||||
{
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Hello"}],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
"stream": false
|
||||
}
|
||||
|
||||
Example streaming request:
|
||||
POST /v1/responses
|
||||
{
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Hello"}],
|
||||
"stream": true
|
||||
}
|
||||
|
||||
Response format (non-streaming):
|
||||
{
|
||||
"id": "resp_...",
|
||||
"object": "response",
|
||||
"created_at": 1733529600,
|
||||
"model": "lorem-tester",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_...",
|
||||
"summary": ["Analyzing...", "Considering..."]
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_...",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Lorem ipsum..."}]
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 50,
|
||||
"reasoning_tokens": 20,
|
||||
"total_tokens": 80
|
||||
}
|
||||
}
|
||||
|
||||
Streaming format (SSE):
|
||||
event: response.reasoning_summary_text.delta
|
||||
data: {"delta": "Analyzing..."}
|
||||
|
||||
event: response.output_text.delta
|
||||
data: {"delta": "Lorem"}
|
||||
|
||||
event: response.done
|
||||
data: {"response": {...}}
|
||||
"""
|
||||
# Set request context (propagates through all async calls)
|
||||
effective_user = request.user or get_default_user()
|
||||
user_token = current_user.set(effective_user)
|
||||
conv_id = request.metadata.get("conversation_id") if request.metadata else None
|
||||
conv_token = current_conversation.set(conv_id)
|
||||
|
||||
logger.info(
|
||||
"response_request_received",
|
||||
model=request.model,
|
||||
user=effective_user,
|
||||
conversation_id=conv_id,
|
||||
user=request.user,
|
||||
streaming=request.stream,
|
||||
)
|
||||
|
||||
try:
|
||||
# Check if this is a Tatlock request - use Steward preprocessing (Phase 2)
|
||||
# Check if this is a Tatlock request - use Steward preprocessing
|
||||
model_id = request.model
|
||||
if "." in model_id:
|
||||
model_id = model_id.split(".", 1)[1]
|
||||
@@ -116,21 +54,20 @@ async def create_response(
|
||||
|
||||
if request.stream:
|
||||
logger.info("Streaming response requested")
|
||||
|
||||
if use_steward:
|
||||
logger.info("Streaming with Steward preprocessing for Tatlock request")
|
||||
# Use Steward + Tatlock streaming (Milestone 3.5)
|
||||
from src.responses.streaming import StreamingCoordinator
|
||||
coordinator = StreamingCoordinator()
|
||||
return EventSourceResponse(
|
||||
coordinator.stream_response_with_steward(request)
|
||||
)
|
||||
else:
|
||||
# Regular streaming for non-Tatlock models
|
||||
return EventSourceResponse(
|
||||
service.create_response_stream(request)
|
||||
)
|
||||
|
||||
# Use appropriate service method
|
||||
# Non-streaming response
|
||||
if use_steward:
|
||||
logger.info("Using Steward preprocessing for Tatlock request")
|
||||
return await service.create_response_with_steward(request)
|
||||
@@ -148,8 +85,3 @@ async def create_response(
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
finally:
|
||||
# Reset context (important for connection reuse)
|
||||
current_user.reset(user_token)
|
||||
current_conversation.reset(conv_token)
|
||||
|
||||
+249
-140
@@ -26,6 +26,8 @@ from src.responses.context import ContextWindow
|
||||
from src.core.preprocessing import preprocess_request
|
||||
from src.core.tool_tracking import ToolCallTracker
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import start_trace, end_trace, start_span, SpanType
|
||||
from src.core.context import current_user, current_conversation, get_default_user
|
||||
from src.agents.steward.schemas import StewardRecommendation
|
||||
|
||||
import re
|
||||
@@ -34,6 +36,29 @@ import asyncio
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def _extract_user_input(input_data) -> str:
|
||||
"""Extract user input text from request input for tracing."""
|
||||
if isinstance(input_data, str):
|
||||
return input_data
|
||||
elif isinstance(input_data, list) and input_data:
|
||||
last_msg = input_data[-1]
|
||||
if isinstance(last_msg, dict):
|
||||
return last_msg.get("content", str(last_msg))
|
||||
return str(last_msg)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_response_preview(response: Response) -> str:
|
||||
"""Extract response preview text for tracing."""
|
||||
if response.output:
|
||||
for item in response.output:
|
||||
if hasattr(item, 'content'):
|
||||
for content in item.content:
|
||||
if hasattr(content, 'text'):
|
||||
return content.text[:200]
|
||||
return ""
|
||||
|
||||
|
||||
async def _execute_single_delegation(
|
||||
agent_name: str,
|
||||
task: str,
|
||||
@@ -405,45 +430,88 @@ async def create_response(request: ResponseRequest) -> Response:
|
||||
# Get or generate conversation ID
|
||||
conversation_id = await _conversation_history.get_conversation_id(request)
|
||||
|
||||
# Strip pipeline prefix if present (e.g., "pipeline.model" -> "model")
|
||||
model_id = request.model
|
||||
if "." in model_id:
|
||||
model_id = model_id.split(".", 1)[1]
|
||||
# Set context for tracing
|
||||
effective_user = request.user or get_default_user()
|
||||
current_user.set(effective_user)
|
||||
current_conversation.set(conversation_id)
|
||||
|
||||
# Get agent for model
|
||||
agent = ModelRegistry.get_agent(model_id)
|
||||
# Extract user input for tracing
|
||||
user_input = _extract_user_input(request.input)
|
||||
|
||||
# Collect all output items from agent
|
||||
output_items = []
|
||||
async for item in agent.generate_response(
|
||||
messages=request.input,
|
||||
reasoning=request.reasoning,
|
||||
tools=request.tools,
|
||||
temperature=request.temperature,
|
||||
max_tokens=request.max_output_tokens,
|
||||
stop=request.stop,
|
||||
):
|
||||
output_items.append(item)
|
||||
|
||||
# Convert agent OutputItems to schema OutputItems
|
||||
converted_items = _convert_output_items(output_items)
|
||||
|
||||
# Calculate token usage
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=converted_items,
|
||||
usage=usage
|
||||
# Start trace
|
||||
trace = start_trace(
|
||||
conversation_id=conversation_id,
|
||||
user=effective_user,
|
||||
request={
|
||||
"model": request.model,
|
||||
"input_preview": user_input[:200] if user_input else "",
|
||||
"full_input": request.input,
|
||||
"streaming": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Track conversation history (for analytics and future vector memory)
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
# Start service span
|
||||
service_span = start_span(
|
||||
"create_response",
|
||||
SpanType.ROUTER,
|
||||
metadata={"model": request.model, "user": effective_user},
|
||||
)
|
||||
|
||||
return response
|
||||
try:
|
||||
# Strip pipeline prefix if present (e.g., "pipeline.model" -> "model")
|
||||
model_id = request.model
|
||||
if "." in model_id:
|
||||
model_id = model_id.split(".", 1)[1]
|
||||
|
||||
# Get agent for model
|
||||
agent = ModelRegistry.get_agent(model_id)
|
||||
|
||||
# Collect all output items from agent
|
||||
output_items = []
|
||||
async for item in agent.generate_response(
|
||||
messages=request.input,
|
||||
reasoning=request.reasoning,
|
||||
tools=request.tools,
|
||||
temperature=request.temperature,
|
||||
max_tokens=request.max_output_tokens,
|
||||
stop=request.stop,
|
||||
):
|
||||
output_items.append(item)
|
||||
|
||||
# Convert agent OutputItems to schema OutputItems
|
||||
converted_items = _convert_output_items(output_items)
|
||||
|
||||
# Calculate token usage
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=converted_items,
|
||||
usage=usage
|
||||
)
|
||||
|
||||
# Track conversation history (for analytics and future vector memory)
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
|
||||
# End trace with response info
|
||||
response_preview = _extract_response_preview(response)
|
||||
end_trace(
|
||||
response={
|
||||
"output_preview": response_preview,
|
||||
"output_count": len(response.output) if response.output else 0,
|
||||
"status": response.status,
|
||||
},
|
||||
status="completed",
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
end_trace(status="error")
|
||||
raise
|
||||
|
||||
|
||||
async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
@@ -454,7 +522,7 @@ async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
1. Steward analyzes the request and recommends capabilities
|
||||
2. Phase 1: Tatlock orchestrates tool calls and expert delegations
|
||||
3. Phase 2: Tatlock synthesizes butler-toned response from results
|
||||
4. Tool usage is tracked for benchmarking
|
||||
4. Tool usage is tracked for analysis
|
||||
|
||||
Args:
|
||||
request: Response request
|
||||
@@ -473,133 +541,174 @@ async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
# Get or generate conversation ID
|
||||
conversation_id = await _conversation_history.get_conversation_id(request)
|
||||
|
||||
# Extract user message and conversation history
|
||||
user_message = ""
|
||||
for msg in reversed(request.input):
|
||||
if msg.get("role") == "user":
|
||||
user_message = msg.get("content", "")
|
||||
break
|
||||
# Set context for tracing
|
||||
effective_user = request.user or get_default_user()
|
||||
current_user.set(effective_user)
|
||||
current_conversation.set(conversation_id)
|
||||
|
||||
# Conversation history is all messages except the current one
|
||||
conversation_history = request.input[:-1] if len(request.input) > 1 else []
|
||||
# Extract user input for tracing
|
||||
user_input = _extract_user_input(request.input)
|
||||
|
||||
logger.info(
|
||||
"creating_response_with_steward",
|
||||
user_message_preview=user_message[:100],
|
||||
history_length=len(conversation_history),
|
||||
# Start trace
|
||||
trace = start_trace(
|
||||
conversation_id=conversation_id,
|
||||
user=effective_user,
|
||||
request={
|
||||
"model": request.model,
|
||||
"input_preview": user_input[:200] if user_input else "",
|
||||
"full_input": request.input,
|
||||
"streaming": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Steward preprocessing
|
||||
enriched = await preprocess_request(
|
||||
user_message,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
# Start service span
|
||||
service_span = start_span(
|
||||
"create_response_with_steward",
|
||||
SpanType.ROUTER,
|
||||
metadata={"model": request.model, "user": effective_user},
|
||||
)
|
||||
|
||||
# Initialize tool tracker
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
try:
|
||||
# Extract user message and conversation history
|
||||
user_message = ""
|
||||
for msg in reversed(request.input):
|
||||
if msg.get("role") == "user":
|
||||
user_message = msg.get("content", "")
|
||||
break
|
||||
|
||||
# Check if direct delegation is recommended
|
||||
# If Steward recommends ONLY delegation agents (biographer/librarian/housekeeper),
|
||||
# we still use two-phase but delegate directly in Phase 1
|
||||
delegation_agents = {"biographer", "librarian", "housekeeper"}
|
||||
delegation_only = all(
|
||||
cap in delegation_agents
|
||||
for cap in enriched.recommendation.recommended_capabilities
|
||||
) and enriched.recommendation.recommended_capabilities
|
||||
# Conversation history is all messages except the current one
|
||||
conversation_history = request.input[:-1] if len(request.input) > 1 else []
|
||||
|
||||
from src.agents.tatlock import TatlockAgent
|
||||
tatlock = TatlockAgent()
|
||||
|
||||
# Use enriched query (with location/timezone context) if available
|
||||
effective_query = enriched.recommendation.enriched_query or user_message
|
||||
|
||||
if delegation_only:
|
||||
# Direct delegation path - collect results then synthesize
|
||||
orchestration_results = await _direct_delegation_with_results(
|
||||
effective_query, enriched.recommendation, tracker, conversation_id
|
||||
)
|
||||
else:
|
||||
# Phase 1: Orchestrate tool calls
|
||||
orchestration_results = await tatlock.orchestrate_tool_calls(
|
||||
user_message=effective_query,
|
||||
steward_note=enriched.steward_note,
|
||||
scoped_tools=enriched.scoped_tools,
|
||||
message_history=conversation_history,
|
||||
tool_tracker=tracker,
|
||||
logger.info(
|
||||
"creating_response_with_steward",
|
||||
user_message_preview=user_message[:100],
|
||||
history_length=len(conversation_history),
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Handle text-based delegation fallback if present
|
||||
if "[DELEGATE:" in orchestration_results.get("raw_output", ""):
|
||||
text_delegation_results = await _handle_text_delegation(
|
||||
orchestration_results["raw_output"], tracker, conversation_id
|
||||
# Steward preprocessing
|
||||
enriched = await preprocess_request(
|
||||
user_message,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Initialize tool tracker
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Check if direct delegation is recommended
|
||||
# If Steward recommends ONLY delegation agents (biographer/librarian/housekeeper),
|
||||
# we still use two-phase but delegate directly in Phase 1
|
||||
delegation_agents = {"biographer", "librarian", "housekeeper"}
|
||||
delegation_only = all(
|
||||
cap in delegation_agents
|
||||
for cap in enriched.recommendation.recommended_capabilities
|
||||
) and enriched.recommendation.recommended_capabilities
|
||||
|
||||
from src.agents.tatlock import TatlockAgent
|
||||
tatlock = TatlockAgent()
|
||||
|
||||
# Use enriched query (with location/timezone context) if available
|
||||
effective_query = enriched.recommendation.enriched_query or user_message
|
||||
|
||||
if delegation_only:
|
||||
# Direct delegation path - collect results then synthesize
|
||||
orchestration_results = await _direct_delegation_with_results(
|
||||
effective_query, enriched.recommendation, tracker, conversation_id
|
||||
)
|
||||
else:
|
||||
# Phase 1: Orchestrate tool calls
|
||||
orchestration_results = await tatlock.orchestrate_tool_calls(
|
||||
user_message=effective_query,
|
||||
steward_note=enriched.steward_note,
|
||||
scoped_tools=enriched.scoped_tools,
|
||||
message_history=conversation_history,
|
||||
tool_tracker=tracker,
|
||||
)
|
||||
# Add text delegation results to expert_results
|
||||
if text_delegation_results != orchestration_results["raw_output"]:
|
||||
orchestration_results["expert_results"]["text_delegation"] = text_delegation_results
|
||||
|
||||
# Phase 2: Synthesize butler-toned response from all results
|
||||
tatlock_response = await tatlock.synthesize_from_results(
|
||||
user_message=user_message,
|
||||
orchestration_results=orchestration_results,
|
||||
message_history=conversation_history,
|
||||
)
|
||||
# Handle text-based delegation fallback if present
|
||||
if "[DELEGATE:" in orchestration_results.get("raw_output", ""):
|
||||
text_delegation_results = await _handle_text_delegation(
|
||||
orchestration_results["raw_output"], tracker, conversation_id
|
||||
)
|
||||
# Add text delegation results to expert_results
|
||||
if text_delegation_results != orchestration_results["raw_output"]:
|
||||
orchestration_results["expert_results"]["text_delegation"] = text_delegation_results
|
||||
|
||||
# Finalize tool tracking
|
||||
await tracker.finalize()
|
||||
# Phase 2: Synthesize butler-toned response from all results
|
||||
tatlock_response = await tatlock.synthesize_from_results(
|
||||
user_message=user_message,
|
||||
orchestration_results=orchestration_results,
|
||||
message_history=conversation_history,
|
||||
)
|
||||
|
||||
# Build response output items
|
||||
output_items = []
|
||||
# Finalize tool tracking
|
||||
await tracker.finalize()
|
||||
|
||||
# Add Steward reasoning as a reasoning output item
|
||||
output_items.append(ReasoningOutputItem(
|
||||
id=f"reasoning_{generate_id()}",
|
||||
summary=[
|
||||
"🎩 Steward's Analysis:",
|
||||
enriched.steward_reasoning,
|
||||
],
|
||||
status="completed"
|
||||
))
|
||||
# Build response output items
|
||||
output_items = []
|
||||
|
||||
# Add Tatlock's message
|
||||
output_items.append(MessageOutputItem(
|
||||
id=f"msg_{generate_id()}",
|
||||
role="assistant",
|
||||
content=[OutputTextContent(
|
||||
type="output_text",
|
||||
text=tatlock_response,
|
||||
annotations=[]
|
||||
)],
|
||||
status="completed"
|
||||
))
|
||||
# Add Steward reasoning as reasoning output
|
||||
if enriched.steward_reasoning:
|
||||
output_items.append(ReasoningOutputItem(
|
||||
id=f"rs_{generate_id()}",
|
||||
summary=[enriched.steward_reasoning],
|
||||
status="completed"
|
||||
))
|
||||
|
||||
# Calculate usage (approximate)
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
# Add Tatlock's message
|
||||
output_items.append(MessageOutputItem(
|
||||
id=f"msg_{generate_id()}",
|
||||
role="assistant",
|
||||
content=[OutputTextContent(
|
||||
type="output_text",
|
||||
text=tatlock_response,
|
||||
annotations=[]
|
||||
)],
|
||||
status="completed"
|
||||
))
|
||||
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=output_items,
|
||||
usage=usage
|
||||
)
|
||||
# Calculate usage (approximate)
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
|
||||
# Track conversation history
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=output_items,
|
||||
usage=usage
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"response_with_steward_complete",
|
||||
response_id=response.id,
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
tool_summary=tracker.get_summary(),
|
||||
)
|
||||
# Track conversation history
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
|
||||
return response
|
||||
logger.info(
|
||||
"response_with_steward_complete",
|
||||
response_id=response.id,
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
tool_summary=tracker.get_summary(),
|
||||
)
|
||||
|
||||
# End trace with response info
|
||||
response_preview = _extract_response_preview(response)
|
||||
end_trace(
|
||||
response={
|
||||
"output_preview": response_preview,
|
||||
"output_count": len(response.output) if response.output else 0,
|
||||
"status": response.status,
|
||||
},
|
||||
status="completed",
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
end_trace(status="error")
|
||||
raise
|
||||
|
||||
|
||||
async def create_response_stream(
|
||||
|
||||
@@ -166,26 +166,6 @@ class StreamingCoordinator:
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Stream Steward's analysis as reasoning summary
|
||||
steward_lines = enriched.steward_reasoning.split('\n')
|
||||
for line in steward_lines:
|
||||
if line.strip():
|
||||
yield ReasoningSummaryDelta(delta=line + "\n")
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
yield ReasoningSummaryDone()
|
||||
|
||||
# Add Steward reasoning to output items
|
||||
reasoning_item = ReasoningOutputItem(
|
||||
id=f"reasoning_{generate_id()}",
|
||||
summary=[
|
||||
"🎩 Steward's Analysis:",
|
||||
enriched.steward_reasoning,
|
||||
],
|
||||
status="completed"
|
||||
)
|
||||
output_items.append(reasoning_item)
|
||||
|
||||
# Initialize tool tracker
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
|
||||
@@ -9,13 +9,13 @@ import pytest
|
||||
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
from src.agents.steward.service import analyze_request, format_steward_note, _build_enriched_query
|
||||
from src.core.startup import initialize_application
|
||||
from src.core.startup import register_household_members
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def setup_household_registry():
|
||||
"""Initialize household registry before running tests."""
|
||||
initialize_application()
|
||||
register_household_members()
|
||||
|
||||
|
||||
class TestAnalyzeRequest:
|
||||
@@ -29,17 +29,14 @@ class TestAnalyzeRequest:
|
||||
mock_agent.analyze = AsyncMock(return_value="Simple greeting requires no tools. This is a simple request.")
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"Hello!",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"Hello!",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert result.recommended_capabilities == []
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert mock_agent.analyze.called
|
||||
assert result.recommended_capabilities == []
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert mock_agent.analyze.called
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_math_request(self):
|
||||
@@ -50,16 +47,13 @@ class TestAnalyzeRequest:
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"What's sqrt(144)?",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"What's sqrt(144)?",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert "tatlock_core" in result.recommended_capabilities
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert "tatlock_core" in result.recommended_capabilities
|
||||
assert result.estimated_complexity == "simple"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_conversation_history(self):
|
||||
@@ -75,21 +69,18 @@ class TestAnalyzeRequest:
|
||||
]
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"And what's that times 5?",
|
||||
conversation_history=conversation_history,
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"And what's that times 5?",
|
||||
conversation_history=conversation_history,
|
||||
)
|
||||
assert result.conversation_context.has_previous_context is True
|
||||
assert 0 in result.conversation_context.relevant_turns
|
||||
|
||||
assert result.conversation_context.has_previous_context is True
|
||||
assert 0 in result.conversation_context.relevant_turns
|
||||
|
||||
# Verify conversation history was passed
|
||||
call_kwargs = mock_agent.analyze.call_args.kwargs
|
||||
assert "conversation_history" in call_kwargs
|
||||
assert len(call_kwargs["conversation_history"]) == 2
|
||||
# Verify conversation history was passed
|
||||
call_kwargs = mock_agent.analyze.call_args.kwargs
|
||||
assert "conversation_history" in call_kwargs
|
||||
assert len(call_kwargs["conversation_history"]) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_missing_capabilities(self):
|
||||
@@ -100,16 +91,13 @@ class TestAnalyzeRequest:
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"Generate an image of a sunset",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"Generate an image of a sunset",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert result.missing_capabilities is not None
|
||||
assert "not available" in result.missing_capabilities
|
||||
assert result.missing_capabilities is not None
|
||||
assert "not available" in result.missing_capabilities
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_conversation_id(self):
|
||||
@@ -120,18 +108,15 @@ class TestAnalyzeRequest:
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"Test request",
|
||||
conversation_history=[],
|
||||
conversation_id="test_conv_123",
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"Test request",
|
||||
conversation_history=[],
|
||||
conversation_id="test_conv_123",
|
||||
)
|
||||
|
||||
# Verify analysis completed successfully
|
||||
assert result.recommended_capabilities == ["tatlock_core"]
|
||||
assert result.estimated_complexity == "simple"
|
||||
# Verify analysis completed successfully
|
||||
assert result.recommended_capabilities == ["tatlock_core"]
|
||||
assert result.estimated_complexity == "simple"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_handles_errors(self):
|
||||
|
||||
@@ -34,7 +34,7 @@ async def test_tatlock_conversation_history_memory(async_client: AsyncClient):
|
||||
response_1 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data_1,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_1.status_code == 200
|
||||
@@ -56,7 +56,7 @@ async def test_tatlock_conversation_history_memory(async_client: AsyncClient):
|
||||
response_2 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data_2,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_2.status_code == 200
|
||||
@@ -95,7 +95,7 @@ async def test_tatlock_multi_turn_context(async_client: AsyncClient):
|
||||
response_1 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_1,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_1.status_code == 200
|
||||
@@ -117,7 +117,7 @@ async def test_tatlock_multi_turn_context(async_client: AsyncClient):
|
||||
response_2 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_2,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_2.status_code == 200
|
||||
@@ -150,7 +150,7 @@ async def test_tatlock_tool_call_logging_search(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=60.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -192,7 +192,7 @@ async def test_tatlock_tool_call_logging_calculator(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -243,7 +243,7 @@ async def test_tatlock_tool_call_logging_datetime(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -293,7 +293,7 @@ async def test_tatlock_no_tool_calls_no_logging(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -336,7 +336,7 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
|
||||
response_1 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_1,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_1.status_code == 200
|
||||
@@ -362,7 +362,7 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
|
||||
response_2 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_2,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_2.status_code == 200
|
||||
@@ -378,3 +378,55 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
|
||||
)
|
||||
if not has_calculation:
|
||||
pytest.xfail(f"LLM did not remember calculation (non-deterministic): {second_response[:200]}")
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_tatlock_ollama_fallback(async_client: AsyncClient):
|
||||
"""
|
||||
Test that Tatlock falls back to Ollama when Claude is unavailable.
|
||||
|
||||
Patches _claude_available to False to force the Ollama path,
|
||||
then verifies the system still produces a valid response.
|
||||
"""
|
||||
import src.anthropic.model_selector as model_selector
|
||||
|
||||
# Save original value
|
||||
original = model_selector._claude_available
|
||||
|
||||
try:
|
||||
# Force Ollama fallback
|
||||
model_selector._claude_available = False
|
||||
|
||||
# Verify we're actually using Ollama
|
||||
info = model_selector.get_model_info()
|
||||
assert info["backend"] == "ollama", f"Expected ollama backend, got {info['backend']}"
|
||||
|
||||
request_data = {
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Say hello to me."}
|
||||
],
|
||||
"stream": False
|
||||
}
|
||||
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify response structure is valid
|
||||
assert "choices" in data
|
||||
assert len(data["choices"]) == 1
|
||||
full_response = data["choices"][0]["message"]["content"]
|
||||
assert len(full_response) > 0, "Ollama should produce a non-empty response"
|
||||
|
||||
print(f"\nOllama fallback response: {full_response[:200]}")
|
||||
|
||||
finally:
|
||||
# Restore original value
|
||||
model_selector._claude_available = original
|
||||
|
||||
+16
-3
@@ -2,6 +2,8 @@
|
||||
Shared test fixtures for all tests.
|
||||
Following FastAPI testing best practices.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
@@ -9,11 +11,22 @@ from httpx import AsyncClient, ASGITransport
|
||||
from src.main import app
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _initialize_app():
|
||||
"""
|
||||
Run application lifespan (Claude health check, household registration, etc.)
|
||||
once per test session. ASGITransport doesn't trigger lifespan events,
|
||||
so we call it explicitly.
|
||||
"""
|
||||
from src.core.startup import initialize_application
|
||||
asyncio.run(initialize_application())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
"""
|
||||
Synchronous test client for FastAPI.
|
||||
|
||||
|
||||
Use for simple tests that don't require async.
|
||||
"""
|
||||
return TestClient(app)
|
||||
@@ -23,7 +36,7 @@ def client() -> TestClient:
|
||||
async def async_client() -> AsyncClient:
|
||||
"""
|
||||
Async test client for FastAPI.
|
||||
|
||||
|
||||
Use for testing async endpoints and streaming.
|
||||
"""
|
||||
async with AsyncClient(
|
||||
@@ -37,7 +50,7 @@ async def async_client() -> AsyncClient:
|
||||
def mock_chat_request() -> dict:
|
||||
"""Standard chat completion request fixture."""
|
||||
return {
|
||||
"model": "Tatlock",
|
||||
"model": "lorem-tester",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, world!"}
|
||||
],
|
||||
|
||||
@@ -1,352 +0,0 @@
|
||||
"""
|
||||
Tests for benchmark storage.
|
||||
|
||||
Tests performance tracking, Redis storage, and analytics features.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.benchmarks import (
|
||||
BenchmarkStore,
|
||||
PerformanceBenchmark,
|
||||
get_benchmark_store,
|
||||
)
|
||||
|
||||
|
||||
class TestPerformanceBenchmark:
|
||||
"""Test PerformanceBenchmark model."""
|
||||
|
||||
def test_benchmark_creation(self):
|
||||
"""Test creating a performance benchmark."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="steward_analysis",
|
||||
duration_seconds=1.23,
|
||||
success=True,
|
||||
recommendation_count=3,
|
||||
)
|
||||
|
||||
assert benchmark.operation == "steward_analysis"
|
||||
assert benchmark.duration_seconds == 1.23
|
||||
assert benchmark.success is True
|
||||
assert benchmark.recommendation_count == 3
|
||||
assert isinstance(benchmark.timestamp, datetime)
|
||||
|
||||
def test_benchmark_with_tool_fields(self):
|
||||
"""Test benchmark with tool-specific fields."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="tool_call",
|
||||
duration_seconds=0.5,
|
||||
success=True,
|
||||
tool_name="calculate",
|
||||
was_recommended=True,
|
||||
was_actually_used=True,
|
||||
)
|
||||
|
||||
assert benchmark.tool_name == "calculate"
|
||||
assert benchmark.was_recommended is True
|
||||
assert benchmark.was_actually_used is True
|
||||
|
||||
def test_benchmark_to_redis_dict(self):
|
||||
"""Test conversion to Redis dict."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
metadata={"key": "value"},
|
||||
)
|
||||
|
||||
redis_dict = benchmark.to_redis_dict()
|
||||
assert redis_dict["operation"] == "test_op"
|
||||
assert redis_dict["duration_seconds"] == 1.0
|
||||
assert redis_dict["success"] == "True" # Booleans stored as strings in Redis
|
||||
assert isinstance(redis_dict["timestamp"], str)
|
||||
assert isinstance(redis_dict["metadata"], str)
|
||||
|
||||
def test_benchmark_from_redis_dict(self):
|
||||
"""Test reconstruction from Redis dict."""
|
||||
now = datetime.now(timezone.utc)
|
||||
redis_dict = {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": 1.5,
|
||||
"success": "True", # Booleans stored as strings in Redis
|
||||
"metadata": json.dumps({"test": "data"}),
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
benchmark = PerformanceBenchmark.from_redis_dict(redis_dict)
|
||||
assert benchmark.operation == "test_op"
|
||||
assert benchmark.duration_seconds == 1.5
|
||||
assert benchmark.success is True # Converted back to bool
|
||||
assert benchmark.metadata == {"test": "data"}
|
||||
|
||||
|
||||
class TestBenchmarkStore:
|
||||
"""Test BenchmarkStore functionality."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_redis(self):
|
||||
"""Create mock Redis client."""
|
||||
mock = AsyncMock()
|
||||
mock.hset = AsyncMock()
|
||||
mock.expire = AsyncMock()
|
||||
mock.zadd = AsyncMock()
|
||||
mock.zrevrangebyscore = AsyncMock(return_value=[])
|
||||
mock.hgetall = AsyncMock(return_value={})
|
||||
mock.aclose = AsyncMock()
|
||||
return mock
|
||||
|
||||
@pytest.fixture
|
||||
def store(self, mock_redis):
|
||||
"""Create benchmark store with mock Redis."""
|
||||
return BenchmarkStore(redis_client=mock_redis)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark(self, store, mock_redis):
|
||||
"""Test recording a benchmark."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
await store.record(benchmark)
|
||||
|
||||
# Verify Redis calls
|
||||
mock_redis.hset.assert_called_once()
|
||||
mock_redis.expire.assert_called()
|
||||
mock_redis.zadd.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark_disabled(self, mock_redis):
|
||||
"""Test recording when benchmarks are disabled."""
|
||||
with patch("src.core.benchmarks.config.ENABLE_BENCHMARKS", False):
|
||||
store = BenchmarkStore(redis_client=mock_redis)
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
await store.record(benchmark)
|
||||
|
||||
# Should not call Redis
|
||||
mock_redis.hset.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark_handles_errors(self, store, mock_redis):
|
||||
"""Test recording handles Redis errors gracefully."""
|
||||
mock_redis.hset.side_effect = Exception("Redis error")
|
||||
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
# Should not raise exception
|
||||
await store.record(benchmark)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_benchmarks(self, store, mock_redis):
|
||||
"""Test querying benchmarks."""
|
||||
# Setup mock data
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_key = f"benchmark:test_op:{int(now.timestamp() * 1000)}"
|
||||
mock_redis.zrevrangebyscore.return_value = [mock_key]
|
||||
|
||||
# Mock hgetall to return proper data (booleans as strings, like Redis)
|
||||
mock_redis.hgetall.return_value = {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": 1.5, # Numeric, not string
|
||||
"success": "True", # Booleans stored as strings in Redis
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
results = await store.query("test_op", limit=10)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].operation == "test_op"
|
||||
mock_redis.zrevrangebyscore.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_with_time_range(self, store, mock_redis):
|
||||
"""Test querying with time range."""
|
||||
now = datetime.now(timezone.utc)
|
||||
start_time = now - timedelta(hours=1)
|
||||
end_time = now
|
||||
|
||||
await store.query("test_op", start_time=start_time, end_time=end_time)
|
||||
|
||||
# Verify time range was converted to timestamps
|
||||
call_args = mock_redis.zrevrangebyscore.call_args
|
||||
assert call_args is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_disabled_benchmarks(self, mock_redis):
|
||||
"""Test querying when benchmarks are disabled."""
|
||||
with patch("src.core.benchmarks.config.ENABLE_BENCHMARKS", False):
|
||||
store = BenchmarkStore(redis_client=mock_redis)
|
||||
results = await store.query("test_op")
|
||||
assert results == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_handles_errors(self, store, mock_redis):
|
||||
"""Test query handles errors gracefully."""
|
||||
mock_redis.zrevrangebyscore.side_effect = Exception("Redis error")
|
||||
|
||||
results = await store.query("test_op")
|
||||
assert results == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_statistics(self, store, mock_redis):
|
||||
"""Test getting statistics."""
|
||||
# Setup mock data with multiple benchmarks
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_keys = [
|
||||
f"benchmark:test_op:{int((now - timedelta(seconds=i)).timestamp() * 1000)}"
|
||||
for i in range(3)
|
||||
]
|
||||
mock_redis.zrevrangebyscore.return_value = mock_keys
|
||||
|
||||
# Return different durations and success values
|
||||
benchmarks_data = [
|
||||
{"duration_seconds": "1.0", "success": "True"},
|
||||
{"duration_seconds": "2.0", "success": "True"},
|
||||
{"duration_seconds": "3.0", "success": "False"},
|
||||
]
|
||||
|
||||
async def mock_hgetall(key):
|
||||
idx = mock_keys.index(key)
|
||||
data = benchmarks_data[idx]
|
||||
return {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": float(data["duration_seconds"]),
|
||||
"success": data["success"], # Pass string through, from_redis_dict converts
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
mock_redis.hgetall.side_effect = mock_hgetall
|
||||
|
||||
stats = await store.get_statistics("test_op")
|
||||
|
||||
assert stats["count"] == 3
|
||||
assert stats["avg_duration"] == 2.0 # (1 + 2 + 3) / 3
|
||||
assert stats["min_duration"] == 1.0
|
||||
assert stats["max_duration"] == 3.0
|
||||
assert stats["success_rate"] == pytest.approx(66.67, rel=0.01)
|
||||
assert stats["total_successes"] == 2
|
||||
assert stats["total_failures"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_statistics_empty(self, store, mock_redis):
|
||||
"""Test statistics with no data."""
|
||||
mock_redis.zrevrangebyscore.return_value = []
|
||||
|
||||
stats = await store.get_statistics("test_op")
|
||||
|
||||
assert stats["count"] == 0
|
||||
assert stats["avg_duration"] == 0.0
|
||||
assert stats["success_rate"] == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_tool_accuracy(self, store, mock_redis):
|
||||
"""Test tool accuracy calculation."""
|
||||
# Setup mock data
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_keys = [
|
||||
f"benchmark:tool_call:{int((now - timedelta(seconds=i)).timestamp() * 1000)}"
|
||||
for i in range(4)
|
||||
]
|
||||
mock_redis.zrevrangebyscore.return_value = mock_keys
|
||||
|
||||
# Different combinations of recommended/used
|
||||
tool_data = [
|
||||
{"was_recommended": "True", "was_actually_used": "True"}, # Good
|
||||
{"was_recommended": "True", "was_actually_used": "True"}, # Good
|
||||
{"was_recommended": "False", "was_actually_used": "True"}, # Missed
|
||||
{"was_recommended": "True", "was_actually_used": "False"}, # Not used
|
||||
]
|
||||
|
||||
async def mock_hgetall(key):
|
||||
idx = mock_keys.index(key)
|
||||
data = tool_data[idx]
|
||||
return {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "tool_call",
|
||||
"duration_seconds": 1.0,
|
||||
"success": "True", # Booleans stored as strings in Redis
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": "test_tool",
|
||||
"conversation_id": None,
|
||||
"was_recommended": data["was_recommended"], # Already strings
|
||||
"was_actually_used": data["was_actually_used"], # Already strings
|
||||
}
|
||||
|
||||
mock_redis.hgetall.side_effect = mock_hgetall
|
||||
|
||||
accuracy = await store.get_tool_accuracy()
|
||||
|
||||
assert accuracy["total_calls"] == 4
|
||||
assert accuracy["total_used"] == 3
|
||||
assert accuracy["recommended_and_used"] == 2
|
||||
assert accuracy["not_recommended_but_used"] == 1
|
||||
assert accuracy["precision"] == pytest.approx(66.67, rel=0.01)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_tool_accuracy_empty(self, store, mock_redis):
|
||||
"""Test tool accuracy with no data."""
|
||||
mock_redis.zrevrangebyscore.return_value = []
|
||||
|
||||
accuracy = await store.get_tool_accuracy()
|
||||
|
||||
assert accuracy["total_calls"] == 0
|
||||
assert accuracy["precision"] == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close(self, store, mock_redis):
|
||||
"""Test closing the store."""
|
||||
await store.close()
|
||||
mock_redis.aclose.assert_called_once()
|
||||
|
||||
# Client should be None after close
|
||||
assert store._client is None
|
||||
|
||||
|
||||
class TestGlobalBenchmarkStore:
|
||||
"""Test global benchmark store instance."""
|
||||
|
||||
def test_get_benchmark_store(self):
|
||||
"""Test getting global store instance."""
|
||||
store = get_benchmark_store()
|
||||
assert isinstance(store, BenchmarkStore)
|
||||
|
||||
def test_get_benchmark_store_singleton(self):
|
||||
"""Test store is singleton."""
|
||||
store1 = get_benchmark_store()
|
||||
store2 = get_benchmark_store()
|
||||
assert store1 is store2
|
||||
@@ -3,8 +3,6 @@ Tests for tool call tracking.
|
||||
|
||||
Tests capability extraction and recommendation matching.
|
||||
"""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.tool_tracking import ToolCallTracker
|
||||
@@ -35,15 +33,11 @@ class TestToolCallTracker:
|
||||
recommended_capabilities=["librarian", "biographer"]
|
||||
)
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
await tracker.track_call("delegate_to_librarian", 1.0)
|
||||
|
||||
await tracker.track_call("delegate_to_librarian", 1.0)
|
||||
|
||||
# Should NOT log warning since librarian was recommended
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.was_recommended is True
|
||||
# Should record the call
|
||||
assert "delegate_to_librarian" in tracker.actual_calls
|
||||
assert tracker.actual_calls["delegate_to_librarian"] == [1.0]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_track_call_detects_not_recommended(self):
|
||||
@@ -52,14 +46,12 @@ class TestToolCallTracker:
|
||||
recommended_capabilities=["librarian"]
|
||||
)
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
await tracker.track_call("delegate_to_housekeeper", 1.0)
|
||||
|
||||
await tracker.track_call("delegate_to_housekeeper", 1.0)
|
||||
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.was_recommended is False
|
||||
# Should record the call even though not recommended
|
||||
assert "delegate_to_housekeeper" in tracker.actual_calls
|
||||
summary = tracker.get_summary()
|
||||
assert summary["accuracy"]["not_recommended_but_used"] == 1
|
||||
|
||||
def test_get_summary_with_delegation_tools(self):
|
||||
"""Test summary correctly maps delegation tools to capabilities."""
|
||||
@@ -87,15 +79,9 @@ class TestToolCallTracker:
|
||||
"delegate_to_librarian": [1.0],
|
||||
}
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
await tracker.finalize()
|
||||
|
||||
await tracker.finalize()
|
||||
|
||||
# Should record benchmark for unused biographer
|
||||
assert mock_store.return_value.record.called
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.tool_name == "biographer"
|
||||
assert benchmark.was_recommended is True
|
||||
assert benchmark.was_actually_used is False
|
||||
# Summary should show biographer as recommended but unused
|
||||
summary = tracker.get_summary()
|
||||
assert summary["accuracy"]["recommended_and_used"] == 1 # librarian
|
||||
assert summary["accuracy"]["recommended_but_unused"] == 1 # biographer
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
|
||||
|
||||
#!/bin/bash
|
||||
# Tatlock Server Startup Script
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${GREEN}Starting Tatlock server...${NC}"
|
||||
|
||||
# Check if port 8777 is already in use
|
||||
if lsof -Pi :8777 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
|
||||
echo -e "${RED}Error: Port 8777 is already in use${NC}"
|
||||
echo "Run: lsof -i :8777 to see what's using it"
|
||||
echo "Or run: kill \$(lsof -t -i:8777) to stop it"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Activate virtual environment if not already activated
|
||||
if [ -z "$VIRTUAL_ENV" ]; then
|
||||
if [ -d ".venv" ]; then
|
||||
echo -e "${YELLOW}Activating virtual environment...${NC}"
|
||||
source .venv/bin/activate
|
||||
else
|
||||
echo -e "${RED}Error: Virtual environment not found${NC}"
|
||||
echo "Run: python -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
LOGS_DIR="logs"
|
||||
mkdir -p "$LOGS_DIR"
|
||||
|
||||
# Clear/create log file
|
||||
LOG_FILE="$LOGS_DIR/server.log"
|
||||
> "$LOG_FILE"
|
||||
echo -e "${YELLOW}Logs will be written to: ${LOG_FILE}${NC}"
|
||||
|
||||
# Start the server
|
||||
echo -e "${GREEN}Starting uvicorn server on http://localhost:8777${NC}"
|
||||
echo -e "${YELLOW}Press Ctrl+C to stop the server${NC}"
|
||||
echo ""
|
||||
|
||||
uvicorn src.main:app --reload --host 0.0.0.0 --port 8777 2>&1 | tee "$LOG_FILE"
|
||||
Reference in New Issue
Block a user