Compare commits
+11
-4
@@ -8,10 +8,19 @@ API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
API_PREFIX=/v1
|
||||
|
||||
# Ollama Configuration
|
||||
# Ollama Configuration (local - primary backend)
|
||||
OLLAMA_HOST=http://localhost:11434
|
||||
OLLAMA_DEFAULT_MODEL=mistral-nemo:latest
|
||||
OLLAMA_DEFAULT_MODEL=gemma4:e2b
|
||||
OLLAMA_TIMEOUT=120
|
||||
STEWARD_TIMEOUT=60
|
||||
|
||||
# Anthropic Configuration (Claude - cloud fallback)
|
||||
# Set ANTHROPIC_API_KEY to keep the Claude fallback available: it is used
|
||||
# automatically when Ollama is down, or exclusively when PREFER_CLOUD_BACKEND=true
|
||||
# Without an API key, Tatlock uses Ollama only
|
||||
# ANTHROPIC_API_KEY=sk-ant-api03-your-key-here
|
||||
ANTHROPIC_MODEL=claude-sonnet-5
|
||||
PREFER_CLOUD_BACKEND=false
|
||||
|
||||
# SearXNG Configuration
|
||||
SEARXNG_HOST=http://localhost:8087
|
||||
@@ -21,7 +30,6 @@ SEARXNG_TIMEOUT=30
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
REDIS_MEMORY_DB=1
|
||||
REDIS_BENCHMARK_DB=6
|
||||
REDIS_TIMEOUT=5
|
||||
|
||||
# Qdrant Configuration
|
||||
@@ -33,7 +41,6 @@ QDRANT_PORT=6333
|
||||
# - development: DEBUG (maximum verbosity)
|
||||
# - production: WARNING (minimal noise)
|
||||
# Uncomment to override: LOG_LEVEL=INFO
|
||||
ENABLE_BENCHMARKS=true
|
||||
# Note: Log format is auto-selected based on ENVIRONMENT (console for dev, json for production)
|
||||
|
||||
# User Configuration
|
||||
|
||||
@@ -1,10 +1,22 @@
|
||||
name: Build and Push
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
push:
|
||||
tags:
|
||||
- 'v[0-9]*'
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create Gitea Release
|
||||
run: |
|
||||
curl -sf -X POST \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"tag_name": "${{ github.ref_name }}", "name": "Release ${{ github.ref_name }}", "body": "Automated release for ${{ github.ref_name }}"}' \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases"
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
+15
-7
@@ -46,29 +46,37 @@ ENV/
|
||||
.ipynb_checkpoints/
|
||||
*.ipynb
|
||||
|
||||
# Testing & Coverage
|
||||
# Caches (pytest, mypy, ruff)
|
||||
.cache/
|
||||
|
||||
# Build output (coverage, logs)
|
||||
build/
|
||||
|
||||
# Legacy cache/output locations (in case tools fall back)
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
.coverage.*
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
|
||||
# Testing
|
||||
.tox/
|
||||
.nox/
|
||||
*.cover
|
||||
.hypothesis/
|
||||
|
||||
# Type checking
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
.pyre/
|
||||
.pytype/
|
||||
|
||||
# Linting
|
||||
.ruff_cache/
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
logs/*
|
||||
!logs/traces/
|
||||
logs/traces/*
|
||||
!logs/traces/viewer.html
|
||||
*.log
|
||||
|
||||
# Database
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
This document contains instructions and documentation references for AI assistants working with this codebase.
|
||||
|
||||
> **📖 Important**: Before working on this project, read [PHILOSOPHY.md](PHILOSOPHY.md) to understand the system vision, architectural patterns, and design goals. All development should work towards realizing those patterns.
|
||||
> **📖 Important**: Before working on this project, read [docs/philosophy.md](docs/philosophy.md) to understand the system vision, architectural patterns, and design goals. All development should work towards realizing those patterns.
|
||||
# AGENTS.md
|
||||
|
||||
> **Start every session by reading this file.**
|
||||
|
||||
+200
-1
@@ -7,6 +7,191 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.3.0] - 2026-07-13
|
||||
|
||||
### Changed
|
||||
|
||||
- **Local-first backend (claudification rollback)** - Ollama/gemma4 is now the primary backend; Claude remains as fallback. `PREFER_CLOUD_BACKEND` defaults to `false`, Claude is used automatically when the Ollama startup health check fails, and the Steward retries mid-request failures on the other backend in both directions
|
||||
- **Default Claude model `claude-sonnet-5`** - `claude-sonnet-4-20250514` was retired by Anthropic on 2026-06-15 and would 404, leaving the fallback dead
|
||||
- **Dedicated orchestration prompt** - `orchestrate_tool_calls()` now uses a terse tool-execution prompt (`TATLOCK_ORCHESTRATION_PROMPT`); the butler persona prompt suppressed gemma4 tool calling (the model reasoned about the calculator, then answered from memory with wrong arithmetic). Synthesis keeps the persona prompt, so user-visible voice is unchanged
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Startup crash with broken anthropic package** - Anthropic SDK imports in the model selector are now lazy, so an incompatible `anthropic` install degrades to Ollama-only operation instead of crashing the app at import time (root cause of the production outage since April)
|
||||
- **Claude Sonnet 5 rejects sampling parameters** - removed `temperature` from the Steward's direct Claude call and made the Housekeeper's temperature setting backend-conditional via `get_sampling_settings()`
|
||||
- **Pin `anthropic>=0.77,<1.0`** - the April image resolved an anthropic version incompatible with pydantic-ai 1.27
|
||||
- **Steward timeout configurable** - new `STEWARD_TIMEOUT` (default 60s) replaces the hardcoded 30s, which gemma4 chronically exceeded (~35s warm analysis), causing every request to fail or fall back
|
||||
|
||||
### Added
|
||||
|
||||
- **Ollama startup health check** - verifies the server is reachable and `OLLAMA_DEFAULT_MODEL` is pulled; feeds backend resolution and `get_model_info()`
|
||||
- **Contract tests** (`tests/contracts/`, `make test-contracts`) - wire-level tests that send the raw requests the code sends to Ollama (native + OpenAI-compat tool calling), Anthropic (including the pinned temperature-rejection contract), Qdrant, SearXNG, library-desk, and Redis; unreachable services skip, wrong response shapes fail
|
||||
- **Backend resolution unit tests** (`tests/anthropic/`)
|
||||
|
||||
## [2.2.0] - 2026-04-04
|
||||
|
||||
### Changed
|
||||
|
||||
- **Switch default Ollama model to gemma4:e2b** - Replaces mistral-nemo as the local LLM backend; gemma4:e2b has native function calling support, faster tool calling (2-4s vs 15-20s), better parameter accuracy on word problems, and uses less VRAM (8GB vs 9.2GB)
|
||||
|
||||
### Added
|
||||
|
||||
- **Tool calling benchmark script** (`scripts/benchmark_tool_calling.py`) - Compares tool calling accuracy and latency across Ollama models via the Tatlock API
|
||||
|
||||
## [2.1.0] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Streaming SSE compatibility with Open WebUI** - Switch from `exclude_none=True` to `exclude_unset=True` for SSE chunk serialization; `exclude_none` was too aggressive — it stripped `finish_reason: null` from intermediate chunks (which OpenAI includes), while `exclude_unset` correctly omits only fields never passed to the constructor (like `reasoning_content` on content-only chunks) while preserving explicitly-set `finish_reason: null`
|
||||
|
||||
### Changed
|
||||
|
||||
- **Project structure consolidation** - Moved documentation to `docs/`, consolidated all config into `pyproject.toml`, replaced `wakeup.sh`/`pytest.ini`/`requirements*.txt` with `Makefile` + `pyproject.toml`
|
||||
- **CI test gate** - Unit tests now gate release and build jobs in Gitea Actions workflow
|
||||
- **Build output organization** - Tool caches in `.cache/`, generated output (coverage, logs) in `build/`
|
||||
|
||||
## [2.0.5] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Streaming JSON compatibility** - Exclude null fields from streaming chunks using `exclude_none=True`; OpenAI's API omits null fields entirely, and including them (e.g., `content: null`, `reasoning_content: null`) caused parsing issues in Open WebUI
|
||||
|
||||
## [2.0.4] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Open WebUI streaming compatibility** - Replaced `sse_starlette` `EventSourceResponse` with plain `StreamingResponse` for chat completions; `sse_starlette` added `\r\n` line endings and extra SSE fields that Open WebUI couldn't parse
|
||||
|
||||
## [2.0.3] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Steward analysis leaking into responses** - Removed internal routing analysis (`DELEGATE: tatlock_core...`) from user-visible reasoning in both streaming and non-streaming paths
|
||||
|
||||
## [2.0.2] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **tool_choice format incompatibility** - Removed `extra_body` tool_choice hack for Claude backend; PydanticAI handles tool_choice natively for Anthropic, preventing infinite tool call loops
|
||||
- **CI trigger** - Changed workflow trigger from `release:published` to `push:tags:v[0-9]*`
|
||||
|
||||
## [2.0.1] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Expert agent registration failure** - `AnthropicModel` does not accept `api_key` directly; now passes it via `AnthropicProvider`
|
||||
|
||||
## [2.0.0] - 2026-02-05
|
||||
|
||||
### Added
|
||||
|
||||
- **Claude backend support (Claudification Phase 1)** - All agents now prefer Claude over Ollama
|
||||
- New `src/anthropic/` module with model selector and health check
|
||||
- `get_model()` factory returns Claude if available, Ollama as fallback
|
||||
- Startup health check caches Claude API availability
|
||||
- Configuration: `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `PREFER_CLOUD_BACKEND`
|
||||
- 200k token context when using Claude backend
|
||||
|
||||
- **Steward dual-backend support** - Direct API calls to Claude or Ollama
|
||||
- `_call_claude()`: Anthropic Messages API path
|
||||
- `_call_ollama()`: Existing Ollama generate API path (preserved)
|
||||
- Automatic fallback: if Claude call fails mid-request, retries with Ollama
|
||||
|
||||
- **Claudification project tracking** - `PROJECT_CLAUDIFICATION.md` with Phase 1/2 roadmap
|
||||
|
||||
### Changed
|
||||
|
||||
- **All PydanticAI agents refactored to use `get_model()`**:
|
||||
- Tatlock (6 instantiation locations)
|
||||
- Librarian
|
||||
- Biographer
|
||||
- Housekeeper
|
||||
- **`initialize_application()` is now async** - Supports async Claude health check at startup
|
||||
- **Dependencies**: `pydantic-ai-slim[openai,anthropic]` replaces `pydantic-ai-slim[openai]`
|
||||
- **Startup logging** now includes backend selection info (claude/ollama)
|
||||
- **Agent creation logging** now includes backend and model info
|
||||
|
||||
### Removed
|
||||
|
||||
- Stale `tests/core/test_benchmarks.py` (benchmark system was removed in v1.10.0)
|
||||
|
||||
## [1.11.0] - 2025-12-30
|
||||
|
||||
### Added
|
||||
|
||||
- **Paperless document integration** - HybridRAG now includes indexed PDFs and scanned documents from Paperless-ngx
|
||||
- New `include_documents` parameter in `hybrid_search` tool
|
||||
- 📑 icon for document sources in search results
|
||||
- Librarian prompt updated with document awareness
|
||||
|
||||
- **Volatile cache integration** - HybridRAG now includes pre-fetched real-time data
|
||||
- New `include_volatile` parameter in `hybrid_search` tool
|
||||
- ⚡ icon for volatile sources in search results
|
||||
- Supports weather, forecast, news, stock, crypto, sun, air_quality namespaces
|
||||
- Librarian prompt updated with volatile cache awareness (user-configured items only)
|
||||
|
||||
- **Biographer routing in Steward** - Personal memory queries now correctly route to The Biographer
|
||||
- Added explicit routing rules for "where do I live", "what car do I drive", etc.
|
||||
- Added biographer delegation examples to Steward prompt
|
||||
- Location keywords ("live", "where", "home") now trigger profile pre-fetch
|
||||
|
||||
### Changed
|
||||
|
||||
- **LibraryDeskClient.hybrid_search** - Now passes full config including `document_limit`, `volatile_limit`, and enable flags
|
||||
- **Steward guidelines** - Clarified that research queries about TOPICS go to Librarian, queries about USER go to Biographer
|
||||
|
||||
## [1.10.1] - 2025-12-23
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Tatlock's excessive apologizing** - Strengthened personality prompt to prevent unnecessary apologies after successful Librarian delegations. Added explicit "do NOT apologize" instructions to both system prompt and synthesis prompt.
|
||||
|
||||
## [1.10.0] - 2025-12-22
|
||||
|
||||
### Added
|
||||
|
||||
#### Lightweight Request Tracing
|
||||
- **JSON-based tracing system** for local development debugging
|
||||
- Captures full request flow through multi-agent architecture
|
||||
- `Trace` and `Span` dataclasses with automatic timing and nesting
|
||||
- ContextVar-based propagation for async-safe tracing
|
||||
- `trace_span` async context manager for clean instrumentation
|
||||
- Traces written to `logs/traces/{trace_id}.json`
|
||||
- Enabled via `DEBUG=true` environment variable
|
||||
- **Trace Viewer UI** (`logs/traces/viewer.html`)
|
||||
- Standalone HTML viewer with timeline visualization
|
||||
- Filter by status, search by request text
|
||||
- Expandable span details with prompts and responses
|
||||
- **Tracing REST API** (`/traces`)
|
||||
- `GET /traces` - Serve trace viewer UI
|
||||
- `GET /traces/list` - List available traces with filtering
|
||||
- `GET /traces/{trace_id}` - Retrieve specific trace JSON
|
||||
- Only available when `DEBUG=true`
|
||||
- **Full pipeline instrumentation**
|
||||
- Router-level trace start/end with context management
|
||||
- Steward analysis spans in preprocessing
|
||||
- Tatlock orchestrate/synthesize spans
|
||||
- Expert delegation spans (librarian/biographer/housekeeper)
|
||||
- Tool-level spans extracted from PydanticAI messages
|
||||
|
||||
### Changed
|
||||
|
||||
- **Replaced Redis benchmarks with file-based tracing** - Simpler, more useful for debugging
|
||||
- **Context management moved to service layer** - Router simplified, context set in response service
|
||||
- **Server binds to all interfaces** - `wakeup.sh` now uses `0.0.0.0` for network access
|
||||
|
||||
### Removed
|
||||
|
||||
- **Redis benchmark system** (`src/core/benchmarks.py`)
|
||||
- `ENABLE_BENCHMARKS` config setting
|
||||
- `REDIS_BENCHMARK_DB` config setting
|
||||
- `redis_url` property (kept `redis_memory_url`)
|
||||
- Benchmark recording in Steward service and tool tracking
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Librarian fabrication prevention** - Added explicit instructions to never invent data when tools fail or sources are unavailable
|
||||
|
||||
## [1.9.0] - 2025-12-18
|
||||
|
||||
### Changed
|
||||
@@ -779,7 +964,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- CORS middleware
|
||||
- Exception handlers (OpenAI-compatible error format)
|
||||
|
||||
[Unreleased]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.6.0...main
|
||||
[Unreleased]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.1.0...main
|
||||
[2.1.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.0.5...v2.1.0
|
||||
[2.0.5]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.0.0...v2.0.5
|
||||
[2.0.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.11.0...v2.0.0
|
||||
[1.11.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.10.0...v1.11.0
|
||||
[1.10.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.9.0...v1.10.0
|
||||
[1.9.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.6...v1.9.0
|
||||
[1.8.6]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.5...v1.8.6
|
||||
[1.8.5]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.4...v1.8.5
|
||||
[1.8.4]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.3...v1.8.4
|
||||
[1.8.3]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.2...v1.8.3
|
||||
[1.8.2]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.1...v1.8.2
|
||||
[1.8.1]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.8.0...v1.8.1
|
||||
[1.8.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.7.0...v1.8.0
|
||||
[1.7.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.6.0...v1.7.0
|
||||
[1.6.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.5.0...v1.6.0
|
||||
[1.5.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.4.0...v1.5.0
|
||||
[1.4.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.3.3...v1.4.0
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# CLAUDE.md
|
||||
|
||||
Claude Code-specific notes for this project. For general development instructions, architecture, coding standards, and deployment — see [AGENTS.md](AGENTS.md).
|
||||
|
||||
## Setup & Commands
|
||||
|
||||
```bash
|
||||
make setup # Create venv and install all dependencies
|
||||
make test # Unit tests (no external services)
|
||||
make test-integration # Integration tests (needs Claude/Ollama)
|
||||
make test-contracts # Wire-level contract tests against live service boundaries
|
||||
make run # Start dev server on port 8777
|
||||
make lint # Ruff linter + formatter check
|
||||
make typecheck # Mypy
|
||||
make clean # Remove caches and build artifacts
|
||||
```
|
||||
|
||||
Dependencies are in `pyproject.toml` (`[project.dependencies]` and `[project.optional-dependencies.dev]`).
|
||||
|
||||
## Critical Gotchas
|
||||
|
||||
**ASGITransport does NOT trigger FastAPI lifespan events.** The session-scoped `_initialize_app` fixture in `tests/conftest.py` calls `initialize_application()` explicitly via `asyncio.run()`. Without this, the Ollama/Claude health checks never run: `_ollama_available` stays `None` (treated as available, so requests go to Ollama) and `_claude_available` stays `None` (treated as unavailable, so the Claude fallback never engages).
|
||||
|
||||
**AsyncIO scope mismatch.** `asyncio_default_fixture_loop_scope = function` is set in `pyproject.toml`. Session-scoped async fixtures cause `ScopeMismatch` errors. The fix is to use a sync fixture with `asyncio.run()` for session-scoped initialization.
|
||||
|
||||
**The butler persona prompt suppresses local-model tool calling.** With `TATLOCK_SYSTEM_PROMPT` attached, gemma4 reasons about calling the calculator, then answers from memory with wrong arithmetic (a different wrong product each run). `orchestrate_tool_calls()` therefore uses the terse `TATLOCK_ORCHESTRATION_PROMPT`; the persona is applied in `synthesize_from_results()`. Do not reattach the persona prompt to a tool-phase agent. `tool_choice: "required"` via extra_body does NOT force Ollama to call tools — it is advisory at best.
|
||||
|
||||
**Claude Sonnet 5+ rejects sampling parameters.** `temperature`/`top_p`/`top_k` return a 400. Use `get_sampling_settings()` from the model selector instead of passing `ModelSettings(temperature=...)` directly to agents that can run on the Claude fallback. The contract test suite pins this (`make test-contracts`).
|
||||
|
||||
**Integration test timeouts.** Set to 120s to match `OLLAMA_TIMEOUT` config (300s for the pure-Ollama fallback test, which cannot be rescued by Claude). The full local Steward → orchestrate → synthesize flow takes ~2 minutes on gemma4. Steward analysis alone needs ~35s warm — `STEWARD_TIMEOUT` defaults to 60s.
|
||||
|
||||
**`get_benchmark_store` does not exist.** The benchmarking module (`src/core/benchmarks.py`) was never implemented. `scripts/benchmark_analysis.py` also references it and is broken. Do not add mocks for it in tests.
|
||||
|
||||
**Steward tests need household registry.** Use `register_household_members()` (sync) in fixtures, not `initialize_application()` (async). The steward extracts capabilities from the registry.
|
||||
+2
-2
@@ -5,8 +5,8 @@ WORKDIR /app
|
||||
RUN apt-get update && apt-get install -y curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt pyproject.toml ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY pyproject.toml ./
|
||||
RUN pip install --no-cache-dir .
|
||||
|
||||
COPY src/ ./src/
|
||||
|
||||
|
||||
@@ -1,920 +0,0 @@
|
||||
# Tatlock Implementation Roadmap
|
||||
|
||||
> **Reference**: See [PHILOSOPHY.md](PHILOSOPHY.md) for the target architecture and vision
|
||||
|
||||
This document outlines the phased implementation plan to transform the current OpenAI-compatible API into the full Tatlock household butler system.
|
||||
|
||||
## Current State (v1.2.0 - Phase F Complete)
|
||||
|
||||
**What we have**:
|
||||
- ✅ **The Orchestrator** - FastAPI infrastructure layer
|
||||
- OpenAI-compatible API endpoints (Responses API + Chat Completions)
|
||||
- Streaming coordination and conversation management
|
||||
- Response format with reasoning support
|
||||
- Test infrastructure (~400 tests)
|
||||
- ✅ **Two-Tier Architecture**
|
||||
- The Steward analyzes requests and recommends capabilities
|
||||
- Tatlock coordinates execution with scoped tools
|
||||
- Real-time streaming of analysis and reasoning
|
||||
- ✅ **Household Staff**
|
||||
- **Tatlock** (Butler): Primary interface with witty personality
|
||||
- **The Steward**: Request analysis and capability recommendation
|
||||
- **The Librarian**: Research via library-desk HybridRAG + wiki
|
||||
- **The Biographer**: User memory, profiles, preferences, semantic recall
|
||||
- ✅ **Core Tools**
|
||||
- Calculator, Date/Time toolkit, Web search (SearXNG)
|
||||
- ✅ **Memory System**
|
||||
- Direct access layer (memory_service) for fast lookups
|
||||
- Vector storage (Qdrant) for semantic recall
|
||||
- Session cache (Redis) with 24h TTL
|
||||
- Multi-tenancy via ContextVar
|
||||
- ✅ Mock agent (lorem-tester for testing)
|
||||
|
||||
**What we need**:
|
||||
- More household staff (Developer, Secretary, Handyman, Housekeeper)
|
||||
- MCP (Model Context Protocol) integration
|
||||
- Dynamic model switching for specialized tasks
|
||||
- Full multi-tenant database (PostgreSQL)
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Real LLM Integration - PydanticAI + Tools
|
||||
|
||||
**Goal**: Connect to actual language models and establish the base plumbing
|
||||
|
||||
**Note**: Ollama is an external service dependency (already running separately)
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **PydanticAI Integration** ✅
|
||||
- PydanticAI → Ollama connection ✅
|
||||
- Agent creation patterns ✅
|
||||
- Streaming response handling ✅
|
||||
- Error handling and retries ✅
|
||||
|
||||
2. **Convert Tatlock Agent** ✅
|
||||
- Convert Tatlock agent from mock to PydanticAI ✅
|
||||
- British butler personality prompt ✅
|
||||
- Research-oriented mindset ✅
|
||||
- Streaming to reasoning output ✅
|
||||
- Tool calling framework setup ✅
|
||||
|
||||
3. **Permanent Tools** ✅
|
||||
- Calculator: Safe mathematical expression evaluation ✅
|
||||
- Date/Time toolkit: Current time, relative dates, time differences ✅
|
||||
- Web search: SearXNG integration (external service) ✅
|
||||
- Tool registration with PydanticAI ✅
|
||||
|
||||
4. **Testing Infrastructure** ✅
|
||||
- Integration tests with real LLM ✅
|
||||
- Tool functionality tests ✅
|
||||
- Response quality validation ✅
|
||||
- 131 tests, 81.78% coverage ✅
|
||||
|
||||
### Success Criteria
|
||||
- [x] **PydanticAI agents can call Ollama** (mistral-nemo:latest)
|
||||
- [x] **Streaming works end-to-end**
|
||||
- [x] **Tool calling framework functional**
|
||||
- [x] **Permanent tools working** (calculator, date/time, search)
|
||||
- [x] **Tests pass with real LLM**
|
||||
- [ ] Can switch models dynamically (e.g., Codestral for code)
|
||||
|
||||
### Status
|
||||
**✅ MOSTLY COMPLETE** - Tatlock agent functional with permanent tools
|
||||
|
||||
### Remaining Work
|
||||
- Dynamic model switching for specialized tasks (e.g., Codestral for coding)
|
||||
|
||||
### Why First?
|
||||
Without real LLM integration, we can't meaningfully implement the Steward/Butler pattern. Everything else depends on having actual AI agents working.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Orchestration Layer - The Steward
|
||||
|
||||
**Goal**: Implement the first-tier LLM call for tool/agent selection
|
||||
|
||||
**Purpose**: The Steward performs crucial preparatory work before Tatlock engages with a request. By analyzing incoming requests and determining which tools, services, and household staff members will be needed, the Steward creates a curated recommendation that streamlines Tatlock's work and prevents cognitive overload.
|
||||
|
||||
### Core Architecture
|
||||
|
||||
The Steward operates as the first tier in the two-tier request flow:
|
||||
|
||||
```
|
||||
User Request → Orchestrator → Steward Analysis → Recommendations → Tatlock (with scoped tools/agents)
|
||||
```
|
||||
|
||||
**Key Principle**: The Steward narrows the scope to only relevant capabilities, making Tatlock's decision-making cleaner and more focused.
|
||||
|
||||
### Deliverables
|
||||
|
||||
#### 1. Tool & Agent Registry System
|
||||
|
||||
**Purpose**: Centralized catalog of all available capabilities for the Steward to recommend
|
||||
|
||||
**Implementation Details**:
|
||||
- **Registry Module** (`src/core/registry.py`)
|
||||
- Tool registration decorator pattern
|
||||
- Agent registration with capability metadata
|
||||
- Category-based organization (computation, information, automation, communication)
|
||||
- Dynamic tool/agent discovery and loading
|
||||
|
||||
- **Tool Metadata Schema**
|
||||
```python
|
||||
{
|
||||
"name": "calculator",
|
||||
"category": "computation",
|
||||
"description": "Safe mathematical expression evaluation",
|
||||
"capabilities": ["arithmetic", "algebra", "trigonometry"],
|
||||
"cost": "low", # computational cost indicator
|
||||
"requires_network": false
|
||||
}
|
||||
```
|
||||
|
||||
- **Agent Metadata Schema**
|
||||
```python
|
||||
{
|
||||
"name": "developer",
|
||||
"role": "The Developer",
|
||||
"category": "technical",
|
||||
"description": "Software development assistance",
|
||||
"domains": ["code_generation", "debugging", "architecture"],
|
||||
"specialized_model": "codestral", # optional
|
||||
"cost": "high"
|
||||
}
|
||||
```
|
||||
|
||||
- **Registry API**
|
||||
- `get_all_tools()` - List all available tools
|
||||
- `get_all_agents()` - List all expert agents
|
||||
- `get_by_category(category)` - Filter by category
|
||||
- `search_by_capability(query)` - Semantic search (future: vector search)
|
||||
|
||||
**Testing**:
|
||||
- Unit tests for registration and retrieval
|
||||
- Test dynamic loading of new tools/agents
|
||||
- Validate metadata schemas
|
||||
|
||||
#### 2. Steward PydanticAI Agent
|
||||
|
||||
**Purpose**: First-tier LLM that analyzes requests and recommends relevant tools/agents
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Agent Module** (`src/agents/steward.py`)
|
||||
```python
|
||||
from pydantic_ai import Agent, RunContext
|
||||
from pydantic import BaseModel
|
||||
|
||||
class StewardRecommendation(BaseModel):
|
||||
"""Structured output from Steward analysis"""
|
||||
recommended_tools: list[str]
|
||||
recommended_agents: list[str]
|
||||
reasoning: str
|
||||
estimated_complexity: str # "simple", "moderate", "complex"
|
||||
requires_multi_step: bool
|
||||
|
||||
steward = Agent(
|
||||
'ollama:mistral-nemo', # Same base model as Tatlock
|
||||
result_type=StewardRecommendation,
|
||||
system_prompt="""..."""
|
||||
)
|
||||
```
|
||||
|
||||
- **System Prompt Engineering**
|
||||
- Role: Estate steward responsible for efficient household coordination
|
||||
- Task: Analyze requests to determine needed resources
|
||||
- Output: Structured recommendations with reasoning
|
||||
- Constraints: Be conservative (recommend only truly relevant capabilities)
|
||||
- Context: Full registry of available tools and agents
|
||||
|
||||
- **Steward Tools**
|
||||
```python
|
||||
@steward.tool
|
||||
def get_available_capabilities(ctx: RunContext) -> dict:
|
||||
"""Get catalog of all available tools and agents."""
|
||||
return {
|
||||
"tools": registry.get_all_tools(),
|
||||
"agents": registry.get_all_agents()
|
||||
}
|
||||
```
|
||||
|
||||
- **Request Analysis Flow**
|
||||
1. Receive user request
|
||||
2. Query capability registry via tool
|
||||
3. Analyze request for required capabilities
|
||||
4. Generate structured recommendation
|
||||
5. Format as note to Tatlock
|
||||
|
||||
**Testing**:
|
||||
- Test various request types (simple, complex, multi-domain)
|
||||
- Verify recommendations are relevant and not over-inclusive
|
||||
- Test structured output parsing
|
||||
- Validate reasoning quality
|
||||
|
||||
#### 3. Request Preprocessing Pipeline
|
||||
|
||||
**Purpose**: Integration layer that routes requests through Steward before Tatlock
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Preprocessing Module** (`src/core/preprocessing.py`)
|
||||
```python
|
||||
async def preprocess_request(user_request: str) -> EnrichedRequest:
|
||||
"""
|
||||
1. Call Steward for analysis
|
||||
2. Get recommendations
|
||||
3. Enrich original request
|
||||
4. Return scoped context for Tatlock
|
||||
"""
|
||||
# Get Steward analysis
|
||||
steward_result = await steward.run(user_request)
|
||||
recommendations = steward_result.data
|
||||
|
||||
# Create note to Tatlock
|
||||
steward_note = format_steward_note(recommendations)
|
||||
|
||||
# Build scoped tool/agent list
|
||||
scoped_tools = get_scoped_tools(recommendations.recommended_tools)
|
||||
scoped_agents = get_scoped_agents(recommendations.recommended_agents)
|
||||
|
||||
return EnrichedRequest(
|
||||
original_request=user_request,
|
||||
steward_note=steward_note,
|
||||
available_tools=scoped_tools,
|
||||
available_agents=scoped_agents,
|
||||
metadata=recommendations
|
||||
)
|
||||
```
|
||||
|
||||
- **Note Formatting**
|
||||
```
|
||||
=== Internal Note from the Steward ===
|
||||
|
||||
Request Analysis:
|
||||
{steward reasoning}
|
||||
|
||||
Recommended Tools:
|
||||
- calculator: For mathematical computations
|
||||
- web_search: To find current information
|
||||
|
||||
Recommended Household Staff:
|
||||
- The Developer: For code generation assistance
|
||||
|
||||
Estimated Complexity: moderate
|
||||
===================================
|
||||
|
||||
[Original User Request]
|
||||
```
|
||||
|
||||
- **Orchestrator Integration**
|
||||
- Modify `src/responses/service.py` to call preprocessing
|
||||
- Prepend Steward note to request before sending to Tatlock
|
||||
- Limit Tatlock's tool access to recommended tools only
|
||||
- Stream Steward's reasoning to output
|
||||
|
||||
**Testing**:
|
||||
- Integration tests for full preprocessing flow
|
||||
- Test request enrichment format
|
||||
- Verify tool scoping works correctly
|
||||
- Test streaming of Steward reasoning
|
||||
|
||||
#### 4. Real-Time Transparency
|
||||
|
||||
**Purpose**: Stream Steward's analysis to user's reasoning output
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Streaming Integration** (`src/responses/streaming.py`)
|
||||
- Add Steward analysis phase to stream
|
||||
- Format as reasoning item
|
||||
- Include recommendation summary
|
||||
|
||||
- **Example Output to User**:
|
||||
```
|
||||
[Reasoning]
|
||||
Consulting the Steward for resource planning...
|
||||
|
||||
The Steward's Analysis:
|
||||
- Request requires mathematical computation
|
||||
- Need to verify current information via web search
|
||||
- May benefit from Developer's code expertise
|
||||
|
||||
Recommended: calculator, web_search, The Developer
|
||||
|
||||
Proceeding with scoped resources...
|
||||
```
|
||||
|
||||
**Testing**:
|
||||
- Test streaming of Steward analysis
|
||||
- Verify formatting in Open WebUI
|
||||
- Test error handling if Steward fails
|
||||
|
||||
#### 5. Model Efficiency Optimization
|
||||
|
||||
**Purpose**: Ensure the base model stays loaded in VRAM
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Shared Model Configuration**
|
||||
- Both Steward and Tatlock use `ollama:mistral-nemo` by default
|
||||
- Sequential calls (Steward → Tatlock) keep model hot
|
||||
- No reload delays between tiers
|
||||
|
||||
- **Performance Monitoring**
|
||||
- Log response times for Steward calls
|
||||
- Track total request latency (Steward + Tatlock)
|
||||
- Identify optimization opportunities
|
||||
|
||||
**Testing**:
|
||||
- Benchmark Steward → Tatlock call latency
|
||||
- Verify model stays loaded between calls
|
||||
- Test performance under load
|
||||
|
||||
### Implementation Strategy
|
||||
|
||||
#### Week 1-2: Foundation
|
||||
- [ ] Design and implement registry system
|
||||
- [ ] Create tool/agent metadata schemas
|
||||
- [ ] Build registry API with tests
|
||||
- [ ] Migrate existing tools to registry
|
||||
|
||||
#### Week 3-4: Steward Agent
|
||||
- [ ] Create Steward PydanticAI agent
|
||||
- [ ] Engineer system prompt for analysis
|
||||
- [ ] Implement structured recommendation output
|
||||
- [ ] Add registry query tool
|
||||
- [ ] Test with various request types
|
||||
|
||||
#### Week 5-6: Integration
|
||||
- [ ] Build request preprocessing pipeline
|
||||
- [ ] Implement note formatting
|
||||
- [ ] Integrate with Orchestrator
|
||||
- [ ] Add streaming transparency
|
||||
- [ ] Tool scoping for Tatlock
|
||||
|
||||
#### Week 7: Testing & Refinement
|
||||
- [ ] End-to-end integration tests
|
||||
- [ ] Performance optimization
|
||||
- [ ] Prompt refinement based on results
|
||||
- [ ] Documentation and examples
|
||||
|
||||
### Success Criteria
|
||||
|
||||
- [x] **Steward analyzes incoming requests** using PydanticAI agent
|
||||
- [x] **Produces structured recommendations** (tools, agents, reasoning)
|
||||
- [x] **Recommendations formatted as prepended note** to Tatlock
|
||||
- [x] **Tool registry is queryable and extensible** via clean API
|
||||
- [x] **Steward output visible in reasoning stream** for transparency
|
||||
- [x] **Only recommended tools available** to Tatlock (scoped context)
|
||||
- [x] **Base model stays loaded** between Steward and Tatlock calls
|
||||
- [x] **Recommendations are accurate** (not over/under-inclusive)
|
||||
- [x] **Integration tests pass** for full Steward → Tatlock flow
|
||||
|
||||
### Status
|
||||
**✅ COMPLETE** (v0.2.5)
|
||||
|
||||
### Performance Targets
|
||||
|
||||
- **Steward Analysis Time**: < 2 seconds for typical requests
|
||||
- **Total Added Latency**: < 3 seconds including streaming
|
||||
- **Recommendation Accuracy**: > 90% relevance (manual evaluation)
|
||||
- **Model Reload Delay**: 0 seconds (model stays hot)
|
||||
|
||||
### Risk Mitigation
|
||||
|
||||
**Risk**: Steward recommendations too broad (defeats purpose)
|
||||
- Mitigation: Conservative prompt engineering, test with diverse requests, iterate
|
||||
|
||||
**Risk**: Added latency unacceptable to users
|
||||
- Mitigation: Stream Steward reasoning for transparency, optimize prompt, parallel processing where possible
|
||||
|
||||
**Risk**: Tool registry becomes unwieldy
|
||||
- Mitigation: Good categorization, semantic search (future), regular pruning
|
||||
|
||||
**Risk**: Steward and Tatlock models compete for VRAM
|
||||
- Mitigation: Use same base model, sequential calls, monitor memory
|
||||
|
||||
### Future Enhancements (Post-Phase 2)
|
||||
|
||||
- **Semantic Search**: Vector-based capability search instead of metadata lookup
|
||||
- **Learning from Usage**: Track which recommendations work well, adjust over time
|
||||
- **Confidence Scores**: Steward provides confidence for each recommendation
|
||||
- **Request Classification**: Cache classifications for similar requests
|
||||
- **Multi-Model Support**: Allow Steward to recommend specialized models for specific tasks
|
||||
|
||||
### Estimated Effort
|
||||
|
||||
**7-8 weeks** - Core intelligence routing with comprehensive implementation
|
||||
|
||||
### Why Second?
|
||||
|
||||
The Steward is the foundation of the household architecture. Without it, we'd need to expose all tools/agents to Tatlock, creating cognitive overload and poor decision-making. The Steward enables the focused expertise pattern that makes the whole system work.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: The Butler - Tatlock Agent
|
||||
|
||||
**Goal**: Implement the second-tier coordinator with personality within the existing Orchestrator infrastructure
|
||||
|
||||
**Context**: The Orchestrator (FastAPI infrastructure) already exists. This phase implements the real Tatlock PydanticAI agent to replace the current mock agent.
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Butler Agent (Tatlock)**
|
||||
- PydanticAI agent implementation within Orchestrator
|
||||
- Personality prompt engineering (witty British butler)
|
||||
- Tool calling framework
|
||||
- Multi-agent coordination logic
|
||||
|
||||
2. **Scoped Tool Access**
|
||||
- Filter tools based on Steward recommendations
|
||||
- Dynamic tool loading for Butler context
|
||||
- Tool execution framework
|
||||
- Result aggregation
|
||||
|
||||
3. **Real-Time Reasoning Output**
|
||||
- Stream all Butler activities to reasoning output
|
||||
- Tool call progress indicators
|
||||
- Expert agent consultation messages
|
||||
- Wait time transparency
|
||||
|
||||
### Success Criteria
|
||||
- [x] Tatlock receives enriched requests (user + Steward notes)
|
||||
- [x] Only recommended tools are available
|
||||
- [x] Tatlock coordinates multiple tool calls
|
||||
- [x] All actions streamed to reasoning output
|
||||
- [x] Responses have consistent personality
|
||||
- [x] Synthesizes multi-source results coherently
|
||||
|
||||
### Status
|
||||
**✅ COMPLETE** (v1.1.0)
|
||||
|
||||
### Estimated Effort
|
||||
**4-5 weeks** - Complex coordination logic
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Expert Household Staff - Core Agents
|
||||
|
||||
**Goal**: Implement the initial set of domain-specific expert agents
|
||||
|
||||
### Priority Expert Agents
|
||||
|
||||
1. **The Librarian** (Research & Knowledge Management) ✅ **COMPLETE** (v1.1.0)
|
||||
- Research assistance via library-desk HybridRAG
|
||||
- Wiki page management (search, create, update)
|
||||
- Semantic vector search
|
||||
- Knowledge graph queries
|
||||
- Dossier browsing
|
||||
|
||||
2. **The Biographer** (User Memory) ✅ **COMPLETE** (v1.2.0)
|
||||
- User profile management (name, location, timezone)
|
||||
- Preference storage (units, theme)
|
||||
- Semantic memory recall ("What car do I drive?")
|
||||
- Fact storage from conversations
|
||||
- Session context caching
|
||||
|
||||
3. **The Developer** (Software Development) 🔜 **Planned**
|
||||
- Code generation assistance
|
||||
- Debugging support
|
||||
- Documentation generation
|
||||
- Architecture guidance
|
||||
- *Rationale: Directly supports building the system itself*
|
||||
|
||||
4. **The Handyman** (System Maintenance) 🔜 **Planned**
|
||||
- System status queries
|
||||
- Log analysis
|
||||
- Basic troubleshooting
|
||||
- Infrastructure monitoring
|
||||
|
||||
5. **The Secretary** (Scheduling & Organization) 🔜 **Planned**
|
||||
- Calendar integration
|
||||
- Task management
|
||||
- Reminder system
|
||||
- Schedule conflict detection
|
||||
|
||||
6. **The Housekeeper** (Home Automation) 🔜 **Planned**
|
||||
- Home Assistant integration
|
||||
- Device control interface
|
||||
- Status queries
|
||||
- Automation triggers
|
||||
|
||||
### Each Agent Includes
|
||||
- Specialized prompt and personality
|
||||
- Domain-specific tools
|
||||
- MCP integration points (where applicable)
|
||||
- Integration with Butler orchestration
|
||||
|
||||
### Success Criteria
|
||||
- [x] Each agent implemented as separate module
|
||||
- [x] Agents callable via tool framework
|
||||
- [x] Agents use specialized prompts
|
||||
- [x] Results integrate cleanly with Butler
|
||||
- [ ] Can invoke specialized models (e.g., Codestral for Developer)
|
||||
|
||||
### Status
|
||||
**🔶 PARTIAL** - Librarian and Biographer complete, others planned
|
||||
|
||||
### Estimated Effort
|
||||
**6-8 weeks** - Parallel development possible
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: Persistence Layer - Database & Multi-Tenancy
|
||||
|
||||
**Goal**: Add persistent storage and multi-user support when needed
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **PostgreSQL Integration**
|
||||
- Docker compose configuration for PostgreSQL
|
||||
- Database schema design with tenant isolation
|
||||
- Alembic migrations setup
|
||||
- SQLAlchemy models
|
||||
|
||||
2. **Multi-Tenant Architecture**
|
||||
- Tenant identification middleware
|
||||
- Tenant-scoped database sessions
|
||||
- User authentication system (basic)
|
||||
- Per-tenant data isolation
|
||||
|
||||
3. **Core Data Models**
|
||||
- Users and tenants
|
||||
- Conversations and messages (migrate from in-memory)
|
||||
- Agent interactions log
|
||||
- System configuration and preferences
|
||||
|
||||
4. **Migration Strategy**
|
||||
- Gradual migration from in-memory to database
|
||||
- Backward compatibility during transition
|
||||
- Data export/import utilities
|
||||
|
||||
### Success Criteria
|
||||
- [ ] PostgreSQL container running
|
||||
- [ ] Multiple users can authenticate separately
|
||||
- [ ] Each user sees only their own data
|
||||
- [ ] Conversations persist across restarts
|
||||
- [ ] Database migrations work correctly
|
||||
- [ ] Tests verify tenant isolation
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Data layer foundation
|
||||
|
||||
### Why Later?
|
||||
The core orchestration (Steward → Butler → Experts) can work entirely with in-memory state. We only need database persistence when we want conversations to survive restarts and multiple users to have isolated experiences.
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Extended Services Integration
|
||||
|
||||
**Goal**: Connect to additional supporting services
|
||||
|
||||
### Services to Integrate
|
||||
|
||||
1. **Redis (Memory & Caching)** ✅ **COMPLETE** (v1.2.0)
|
||||
- Benchmark storage (db=1)
|
||||
- Memory cache for sessions (db=2)
|
||||
- 24h TTL for session context
|
||||
- Recent entities tracking
|
||||
|
||||
2. **Qdrant (Vector Storage)** ✅ **COMPLETE** (v1.2.0)
|
||||
- Per-user memory collections
|
||||
- 768-dim nomic-embed-text vectors
|
||||
- Semantic search for recall
|
||||
- Type-based filtering
|
||||
|
||||
3. **SearxNG (Web Search)** ✅ **COMPLETE** (v0.2.0)
|
||||
- Search tool integration
|
||||
- Result processing
|
||||
- Privacy-preserving queries
|
||||
|
||||
4. **library-desk (Research API)** ✅ **COMPLETE** (v1.1.0)
|
||||
- HybridRAG search
|
||||
- Wiki management
|
||||
- Knowledge graph queries
|
||||
|
||||
### Success Criteria
|
||||
- [x] Services communicate correctly
|
||||
- [x] Tatlock can invoke web search
|
||||
- [x] Redis used for session data
|
||||
- [x] Qdrant stores user memories
|
||||
- [x] Ollama serves the base model
|
||||
|
||||
### Status
|
||||
**✅ COMPLETE** - All core services integrated
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Infrastructure setup
|
||||
|
||||
---
|
||||
|
||||
## Phase 7: MCP (Model Context Protocol) Integration
|
||||
|
||||
**Goal**: Enable rich tool integrations via MCP
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **MCP Server Framework**
|
||||
- MCP server implementation
|
||||
- Tool registration via MCP
|
||||
- Schema validation
|
||||
- Error handling
|
||||
|
||||
2. **MCP Client in Agents**
|
||||
- PydanticAI MCP integration
|
||||
- Tool discovery from MCP servers
|
||||
- Dynamic tool loading
|
||||
- Result processing
|
||||
|
||||
3. **Initial MCP Tools**
|
||||
- File system operations
|
||||
- Database queries
|
||||
- API integrations
|
||||
- System commands
|
||||
|
||||
### Success Criteria
|
||||
- [ ] MCP server running
|
||||
- [ ] Tools exposed via MCP protocol
|
||||
- [ ] Agents can discover and use MCP tools
|
||||
- [ ] New tools addable without code changes
|
||||
- [ ] MCP tools visible in Steward recommendations
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Standards-based integration
|
||||
|
||||
---
|
||||
|
||||
## Phase 8: Advanced Memory & Context
|
||||
|
||||
**Goal**: Implement sophisticated memory and context management
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Long-Term Memory** ✅ **COMPLETE** (v1.2.0 - Phase F)
|
||||
- Memory service for direct key-based access
|
||||
- Qdrant vector storage for semantic recall
|
||||
- Embedding via nomic-embed-text
|
||||
- The Biographer agent for memory management
|
||||
|
||||
2. **Session Memory** ✅ **COMPLETE** (v1.2.0)
|
||||
- Redis session cache with 24h TTL
|
||||
- Recent entities tracking
|
||||
- Conversation context preservation
|
||||
- Multi-tenancy via ContextVar
|
||||
|
||||
3. **Steward Integration** ✅ **COMPLETE** (v1.2.0)
|
||||
- Memory pre-fetch during request analysis
|
||||
- Profile/preferences included in context
|
||||
- Keyword-based context determination
|
||||
|
||||
4. **Context Management** 🔜 **Future**
|
||||
- Smart context window trimming
|
||||
- Conversation branching
|
||||
- Topic tracking
|
||||
- Memory retrieval integration
|
||||
|
||||
5. **Personalization** 🔜 **Future**
|
||||
- User preference learning
|
||||
- Interaction pattern analysis
|
||||
- Adaptive responses
|
||||
- Custom agent personalities per user
|
||||
|
||||
### Success Criteria
|
||||
- [x] User facts stored in Qdrant with semantic search
|
||||
- [x] Profile and preferences accessible via memory_service
|
||||
- [x] Session context cached in Redis
|
||||
- [x] User preferences affect responses (via Steward pre-fetch)
|
||||
- [ ] Conversations automatically embedded to Qdrant
|
||||
- [ ] Memory improves over time (learning from interactions)
|
||||
|
||||
### Status
|
||||
**🔶 PARTIAL** - Core memory system complete, advanced features planned
|
||||
|
||||
### Estimated Effort
|
||||
**4-5 weeks** - AI/ML heavy (remaining work)
|
||||
|
||||
---
|
||||
|
||||
## Phase 9: Extended Household Staff
|
||||
|
||||
**Goal**: Add specialized agents for additional domains
|
||||
|
||||
### Future Agents
|
||||
|
||||
1. **The Librarian** (Knowledge Management)
|
||||
- Personal documentation indexing
|
||||
- Research assistance
|
||||
- Knowledge base queries
|
||||
- Reference management
|
||||
|
||||
2. **The Accountant** (Financial Tracking)
|
||||
- Expense tracking
|
||||
- Budget monitoring
|
||||
- Financial reports
|
||||
- Transaction categorization
|
||||
|
||||
3. **The Chef** (Meal Planning)
|
||||
- Recipe management
|
||||
- Meal planning
|
||||
- Nutrition tracking
|
||||
- Grocery lists
|
||||
|
||||
4. **Others as Needed**
|
||||
- Domain-specific as requirements emerge
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Each new agent follows household pattern
|
||||
- [ ] Integrates with Steward/Butler flow
|
||||
- [ ] Has appropriate specialized tools
|
||||
- [ ] Documented in PHILOSOPHY.md updates
|
||||
|
||||
### Estimated Effort
|
||||
**Ongoing** - Add as needed
|
||||
|
||||
---
|
||||
|
||||
## Phase 10: User Experience Refinement
|
||||
|
||||
**Goal**: Polish the interaction experience
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Personality Tuning**
|
||||
- Refine Tatlock's wit and tone
|
||||
- Consistent household character
|
||||
- Cultural references appropriate
|
||||
- Humor that doesn't annoy
|
||||
|
||||
2. **Transparency Improvements**
|
||||
- Better progress indicators
|
||||
- Clearer reasoning explanations
|
||||
- Informative wait messages
|
||||
- Error message clarity
|
||||
|
||||
3. **Performance Optimization**
|
||||
- Response time improvements
|
||||
- Model loading optimization
|
||||
- Caching strategies
|
||||
- Streaming smoothness
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Users find Tatlock engaging
|
||||
- [ ] Wait times feel reasonable
|
||||
- [ ] Errors are understandable
|
||||
- [ ] System feels responsive
|
||||
|
||||
### Estimated Effort
|
||||
**Ongoing** - Continuous improvement
|
||||
|
||||
---
|
||||
|
||||
## Phase 11: Production Hardening
|
||||
|
||||
**Goal**: Make the system production-ready for homelab deployment
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Deployment**
|
||||
- Complete docker-compose stack
|
||||
- Environment configuration
|
||||
- Backup strategies
|
||||
- Update procedures
|
||||
|
||||
2. **Monitoring**
|
||||
- Health checks
|
||||
- Performance metrics
|
||||
- Error tracking
|
||||
- Usage analytics
|
||||
|
||||
3. **Security**
|
||||
- Authentication hardening
|
||||
- Rate limiting
|
||||
- Input validation
|
||||
- Audit logging
|
||||
|
||||
4. **Documentation**
|
||||
- Installation guide
|
||||
- Configuration reference
|
||||
- Troubleshooting guide
|
||||
- Architecture documentation
|
||||
|
||||
### Success Criteria
|
||||
- [ ] One-command deployment
|
||||
- [ ] System health is monitorable
|
||||
- [ ] Secure for homelab use
|
||||
- [ ] Well documented
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Production polish
|
||||
|
||||
---
|
||||
|
||||
## Dependencies Between Phases
|
||||
|
||||
```
|
||||
Phase 1 (Ollama + PydanticAI) ← Foundation for all AI
|
||||
↓
|
||||
Phase 2 (Steward)
|
||||
↓
|
||||
Phase 3 (Butler/Tatlock)
|
||||
↓
|
||||
Phase 4 (Expert Agents) ← Phase 7 (MCP) can enhance
|
||||
↓
|
||||
Phase 5 (Database/Multi-Tenancy) ← Can be deferred
|
||||
↓
|
||||
Phase 6 (Extended Services) → Phase 8 (Advanced Memory)
|
||||
↓
|
||||
Phase 9 (Extended Staff) → Phase 10 (UX) → Phase 11 (Production)
|
||||
```
|
||||
|
||||
**Critical Path**: Phases 1 → 2 → 3 → 4 must be sequential
|
||||
**Can Be Deferred**: Phase 5 (Database) until you need persistence
|
||||
**Parallel Opportunities**: Phase 6 and 7 can overlap; Phase 9 and 10 ongoing
|
||||
|
||||
---
|
||||
|
||||
## Overall Timeline Estimate
|
||||
|
||||
**Minimum Viable Household** (Phases 1-4): **15-20 weeks**
|
||||
- Working Steward → Butler → Expert Agents with real LLM
|
||||
- In-memory state (no persistence needed yet)
|
||||
- Core household functional
|
||||
|
||||
**With Persistence** (Phases 1-5): **18-24 weeks**
|
||||
- Add database and multi-tenancy
|
||||
- Conversations survive restarts
|
||||
- Multiple users supported
|
||||
|
||||
**Full-Featured System** (Phases 1-9): **35-45 weeks**
|
||||
- All services integrated
|
||||
- Advanced memory and context
|
||||
- Extended household staff
|
||||
|
||||
**Production-Ready** (All phases): **40-50 weeks**
|
||||
- Polished UX
|
||||
- Hardened for homelab deployment
|
||||
- Fully documented
|
||||
|
||||
*Note: Timeline assumes consistent part-time development effort*
|
||||
|
||||
---
|
||||
|
||||
## Success Metrics
|
||||
|
||||
### Technical
|
||||
- System implements PHILOSOPHY.md patterns
|
||||
- All household roles functional
|
||||
- Multi-tenant isolation verified
|
||||
- Real-time reasoning transparency working
|
||||
- MCP integration complete
|
||||
|
||||
### User Experience
|
||||
- Tatlock feels like interacting with a butler
|
||||
- Wait times are transparent and acceptable
|
||||
- Expert agents provide value in their domains
|
||||
- System is reliable and trustworthy
|
||||
|
||||
### Architecture
|
||||
- Clean separation between household roles
|
||||
- Easy to add new agents/tools
|
||||
- Model efficiency (base model stays loaded)
|
||||
- Scales to household + friends usage
|
||||
|
||||
---
|
||||
|
||||
## Risk Management
|
||||
|
||||
### High Risk Items
|
||||
1. **PydanticAI + Ollama integration complexity**
|
||||
- Mitigation: Prototype early, iterate on connection layer
|
||||
|
||||
2. **Multi-agent coordination complexity**
|
||||
- Mitigation: Start simple, add coordination gradually
|
||||
|
||||
3. **Model performance on homelab hardware**
|
||||
- Mitigation: Model selection, quantization, optimization
|
||||
|
||||
4. **Prompt engineering for personality consistency**
|
||||
- Mitigation: Extensive testing, user feedback, iteration
|
||||
|
||||
### Medium Risk Items
|
||||
- MCP protocol adoption and tooling maturity
|
||||
- Vector embedding quality for memory
|
||||
- Home automation integration variability
|
||||
- User authentication security
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Priority**: Implement The Developer agent for code assistance
|
||||
2. **Integration**: Add Home Assistant integration for The Housekeeper
|
||||
3. **Calendar**: Integrate scheduling service for The Secretary
|
||||
4. **Ongoing**: Add more household staff as needed
|
||||
|
||||
---
|
||||
|
||||
**Document Status**: Active planning document
|
||||
**Created**: 2025-12-06
|
||||
**Last Updated**: 2025-12-13
|
||||
@@ -0,0 +1,51 @@
|
||||
.PHONY: help setup run test test-unit test-integration test-contracts lint typecheck clean
|
||||
|
||||
VENV := .venv
|
||||
PYTHON := $(VENV)/bin/python
|
||||
PIP := $(VENV)/bin/pip
|
||||
PYTEST := $(VENV)/bin/pytest
|
||||
RUFF := $(VENV)/bin/ruff
|
||||
MYPY := $(VENV)/bin/mypy
|
||||
UVICORN := $(VENV)/bin/uvicorn
|
||||
|
||||
HOST := 0.0.0.0
|
||||
PORT := 8777
|
||||
|
||||
help: ## Show this help
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
setup: ## Create venv and install all dependencies
|
||||
python3 -m venv $(VENV)
|
||||
$(PIP) install --upgrade pip
|
||||
$(PIP) install -e ".[dev]"
|
||||
|
||||
run: ## Start the development server on port 8777
|
||||
@mkdir -p build/logs
|
||||
@if lsof -Pi :$(PORT) -sTCP:LISTEN -t >/dev/null 2>&1; then \
|
||||
echo "Error: Port $(PORT) is already in use"; \
|
||||
echo "Run: lsof -i :$(PORT) to see what's using it"; \
|
||||
exit 1; \
|
||||
fi
|
||||
$(UVICORN) src.main:app --reload --host $(HOST) --port $(PORT) 2>&1 | tee build/logs/server.log
|
||||
|
||||
test: ## Run unit tests (no external services needed)
|
||||
$(PYTEST) --ignore=tests/e2e --ignore=tests/integration --ignore=tests/contracts
|
||||
|
||||
test-unit: test ## Alias for test
|
||||
|
||||
test-integration: ## Run integration tests (needs Claude/Ollama)
|
||||
$(PYTEST) tests/agents/test_tatlock_agent.py -v
|
||||
|
||||
test-contracts: ## Wire-level contract tests against live service boundaries
|
||||
$(PYTEST) tests/contracts -v --no-cov
|
||||
|
||||
lint: ## Run ruff linter and formatter check
|
||||
$(RUFF) check src tests
|
||||
$(RUFF) format --check src tests
|
||||
|
||||
typecheck: ## Run mypy type checking
|
||||
$(MYPY) src
|
||||
|
||||
clean: ## Remove build artifacts, caches, and coverage reports
|
||||
rm -rf .cache build
|
||||
find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
||||
@@ -1,6 +1,6 @@
|
||||
# Tatlock - Your Homelab Butler
|
||||
|
||||
> **📖 For the complete system vision and architectural philosophy, see [PHILOSOPHY.md](PHILOSOPHY.md)**
|
||||
> **📖 For the complete system vision and architectural philosophy, see [docs/philosophy.md](docs/philosophy.md)**
|
||||
|
||||
A privacy-first, offline-capable personal assistant system that coordinates specialized AI agents to help with research, development, home automation, and daily organization.
|
||||
|
||||
@@ -58,7 +58,7 @@ A privacy-first, offline-capable personal assistant system that coordinates spec
|
||||
- Error triggers for testing (rate_limit, context_overflow)
|
||||
|
||||
- **Tatlock**: Real PydanticAI agent with butler personality
|
||||
- **LLM Backend**: Ollama (mistral-nemo:latest by default)
|
||||
- **LLM Backend**: Ollama (gemma4:e2b by default, local-first) with optional Claude fallback
|
||||
- **Personality**: Witty British butler, research-oriented
|
||||
- **Core Tools**:
|
||||
- **Calculator**: Safe mathematical expression evaluation
|
||||
@@ -74,7 +74,7 @@ A privacy-first, offline-capable personal assistant system that coordinates spec
|
||||
|
||||
- Python 3.12+ (Python 3.12.11 recommended)
|
||||
- **External Services** (must be running separately):
|
||||
- **Ollama**: LLM inference (mistral-nemo:latest, nomic-embed-text)
|
||||
- **Ollama**: LLM inference (gemma4:e2b, nomic-embed-text)
|
||||
- **Redis**: Caching and session memory
|
||||
- **Qdrant**: Vector storage for The Biographer's memory
|
||||
- **SearXNG**: Web search (optional)
|
||||
@@ -89,12 +89,8 @@ A privacy-first, offline-capable personal assistant system that coordinates spec
|
||||
git clone https://git.schweitz.net/jpmschweitzer/tatlock.git
|
||||
cd tatlock
|
||||
|
||||
# Create virtual environment
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
||||
|
||||
# Install dependencies
|
||||
pip install -r requirements.txt
|
||||
make setup
|
||||
```
|
||||
|
||||
### Run the Server
|
||||
@@ -268,7 +264,10 @@ Interactive documentation available at:
|
||||
pytest
|
||||
|
||||
# Run unit tests only (no external services needed)
|
||||
pytest --ignore=tests/e2e --ignore=tests/integration
|
||||
pytest --ignore=tests/e2e --ignore=tests/integration --ignore=tests/contracts
|
||||
|
||||
# Wire-level contract tests against live service boundaries
|
||||
make test-contracts
|
||||
|
||||
# Run with coverage
|
||||
pytest --cov=src --cov-report=term-missing
|
||||
@@ -307,12 +306,17 @@ Create a `.env` file for custom configuration:
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
|
||||
# Ollama Configuration
|
||||
# Ollama Configuration (primary backend)
|
||||
OLLAMA_HOST=http://localhost:11434
|
||||
OLLAMA_DEFAULT_MODEL=mistral-nemo:latest
|
||||
OLLAMA_DEFAULT_MODEL=gemma4:e2b
|
||||
OLLAMA_EMBEDDING_MODEL=nomic-embed-text
|
||||
OLLAMA_TIMEOUT=120
|
||||
|
||||
# Claude fallback (optional; used when Ollama is down or PREFER_CLOUD_BACKEND=true)
|
||||
# ANTHROPIC_API_KEY=sk-ant-api03-your-key-here
|
||||
ANTHROPIC_MODEL=claude-sonnet-5
|
||||
PREFER_CLOUD_BACKEND=false
|
||||
|
||||
# Redis Configuration
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
@@ -396,8 +400,7 @@ tatlock/
|
||||
│ │ └── multi_tenancy.py # User isolation utilities
|
||||
│ └── main.py # Application entry point
|
||||
├── tests/ # Comprehensive test suite
|
||||
├── PHILOSOPHY.md # System vision and architecture
|
||||
├── IMPLEMENTATION_ROADMAP.md # Development phases
|
||||
├── docs/ # Project documentation
|
||||
├── CHANGELOG.md # Version history
|
||||
└── README.md # This file
|
||||
```
|
||||
@@ -416,8 +419,8 @@ For LLM agent development guidelines and architectural decisions, see [AGENTS.md
|
||||
|
||||
## Documentation
|
||||
|
||||
- **System Philosophy**: [PHILOSOPHY.md](PHILOSOPHY.md) - Vision, goals, and architectural patterns
|
||||
- **User Guide**: This file - Installation, usage, and examples
|
||||
- **System Philosophy**: [docs/philosophy.md](docs/philosophy.md) - Vision, goals, and architectural patterns
|
||||
- **Development Roadmap**: [docs/roadmap.md](docs/roadmap.md) - Open work and planned phases
|
||||
- **Developer Guidelines**: [AGENTS.md](AGENTS.md) - LLM agent development patterns
|
||||
- **Version History**: [CHANGELOG.md](CHANGELOG.md) - Changes and releases
|
||||
|
||||
@@ -432,8 +435,8 @@ For LLM agent development guidelines and architectural decisions, see [AGENTS.md
|
||||
|
||||
## Version
|
||||
|
||||
Current version: **1.3.2** - Biographer tool type hints fix
|
||||
Current version: see [CHANGELOG.md](CHANGELOG.md)
|
||||
|
||||
---
|
||||
|
||||
**Note**: Tatlock is a production-ready homelab butler. All household staff use PydanticAI with Ollama for local LLM inference.
|
||||
**Note**: Tatlock is a production-ready homelab butler. All household staff use PydanticAI with local Ollama inference (gemma4), with an optional Claude cloud fallback.
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
# Claude Integration Plan
|
||||
|
||||
## Overview
|
||||
|
||||
Tatlock uses a bidirectional Claude architecture:
|
||||
- **Scenario A**: Tatlock powered by Claude backend (with Ollama fallback) — **COMPLETE**, then **rolled back to local-first**: Ollama/gemma4 is primary, Claude is retained as fallback (`PREFER_CLOUD_BACKEND=false`)
|
||||
- **Scenario B**: Tatlock exposed as MCP server for external Claude instances — **OPEN**
|
||||
- **Scenario C**: Offline operation via Ollama — **COMPLETE**
|
||||
|
||||
---
|
||||
|
||||
## MCP Server (Expose Tools to Claude) — NOT STARTED
|
||||
|
||||
Create an MCP server that exposes Tatlock's household tools to external Claude instances.
|
||||
|
||||
### New Files
|
||||
|
||||
```
|
||||
src/mcp/
|
||||
├── __init__.py
|
||||
├── server.py # MCP server using mcp Python SDK
|
||||
├── tool_adapters.py # Convert PydanticAI tools → MCP schemas
|
||||
├── auth.py # API key authentication
|
||||
└── transport.py # Streamable HTTP transport
|
||||
```
|
||||
|
||||
### Docker Stack Addition
|
||||
|
||||
```yaml
|
||||
tatlock-mcp:
|
||||
image: git.schweitz.internal/jpmschweitzer/tatlock:latest
|
||||
command: ["python", "-m", "src.mcp.server"]
|
||||
ports:
|
||||
- "8002:8002"
|
||||
environment:
|
||||
- MCP_AUTH_TOKEN=${MCP_AUTH_TOKEN}
|
||||
networks:
|
||||
- docker-dataplane
|
||||
```
|
||||
|
||||
### Claude Desktop Configuration
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"tatlock": {
|
||||
"command": "npx",
|
||||
"args": ["mcp-remote", "https://mcp.schweitz.net/sse", "--header", "Authorization: Bearer ${MCP_AUTH_TOKEN}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Checklist
|
||||
|
||||
- [ ] Create `src/mcp/` module
|
||||
- [ ] Tool adapters (PydanticAI → MCP schema)
|
||||
- [ ] Authentication middleware
|
||||
- [ ] Streamable HTTP transport
|
||||
- [ ] Docker stack configuration
|
||||
|
||||
---
|
||||
|
||||
## Future Phases
|
||||
|
||||
- **LiteLLM Gateway** — Unified endpoint for all models, config-driven routing
|
||||
- **Multi-Provider** — Add OpenAI, Vertex AI, etc.
|
||||
- **Smart Routing** — Context-aware model selection, cost ceiling enforcement
|
||||
|
||||
---
|
||||
|
||||
## Offline Behavior
|
||||
|
||||
| Scenario | Behavior |
|
||||
|----------|----------|
|
||||
| No API key | Use Ollama exclusively |
|
||||
| API unreachable | Use Ollama, log warning |
|
||||
| API rate limited | Fallback to Ollama |
|
||||
|
||||
| Aspect | Claude | Ollama |
|
||||
|--------|--------|--------|
|
||||
| Context | 200k tokens | ~8k tokens |
|
||||
| Latency | 1-3s (network) | 0.5-1s (local) |
|
||||
| Personality | Preserved | Preserved |
|
||||
| Tools | All work | All work |
|
||||
| Cost | API charges | Free |
|
||||
|
||||
---
|
||||
|
||||
## Related Repo Handovers
|
||||
|
||||
Handover documents created in each repo: `PROJECT_CLAUDIFICATION_HANDOVER.md`
|
||||
|
||||
### Open Items
|
||||
|
||||
- **library-desk**: Review HybridRAG response size limits, smart_create endpoint, response formats
|
||||
- **core-api**: Review list_devices response format, error messages, rate limiting
|
||||
- **portainer-core**: Update stack with new env vars, configure secrets, update CONTAINERS.md
|
||||
- **webber**: Review content truncation limits, extraction quality
|
||||
- **tatlock-ui**: Test streaming with Claude backend, conversation history, tool call display
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
# Tatlock Implementation Roadmap
|
||||
|
||||
> **Reference**: See [philosophy.md](philosophy.md) for the target architecture and vision
|
||||
|
||||
This document tracks open/planned work. Completed phases have been removed.
|
||||
|
||||
## Current State (v2.0.5)
|
||||
|
||||
**What we have**:
|
||||
- OpenAI-compatible API (Responses API + Chat Completions)
|
||||
- Two-tier architecture (Steward → Tatlock)
|
||||
- Household staff: Tatlock (Butler), Steward, Librarian, Biographer
|
||||
- Core tools: Calculator, Date/Time, Web search (SearXNG)
|
||||
- Memory system: Qdrant (vector), Redis (session cache), multi-tenancy via ContextVar
|
||||
- Dual backend: Ollama/gemma4 (primary) + Claude (fallback)
|
||||
- 439 tests with good coverage
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Expert Household Staff — Remaining Agents
|
||||
|
||||
**Goal**: Implement remaining domain-specific expert agents
|
||||
|
||||
### Planned Agents
|
||||
|
||||
1. **The Developer** (Software Development)
|
||||
- Code generation assistance
|
||||
- Debugging support
|
||||
- Documentation generation
|
||||
- Architecture guidance
|
||||
|
||||
2. **The Handyman** (System Maintenance)
|
||||
- System status queries
|
||||
- Log analysis
|
||||
- Basic troubleshooting
|
||||
- Infrastructure monitoring
|
||||
|
||||
3. **The Secretary** (Scheduling & Organization)
|
||||
- Calendar integration
|
||||
- Task management
|
||||
- Reminder system
|
||||
- Schedule conflict detection
|
||||
|
||||
4. **The Housekeeper** (Home Automation)
|
||||
- Home Assistant integration
|
||||
- Device control interface
|
||||
- Status queries
|
||||
- Automation triggers
|
||||
|
||||
### Each Agent Includes
|
||||
- Specialized prompt and personality
|
||||
- Domain-specific tools
|
||||
- MCP integration points (where applicable)
|
||||
- Integration with Butler orchestration
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Each agent implemented as separate module
|
||||
- [ ] Agents callable via tool framework
|
||||
- [ ] Can invoke specialized models (e.g., Codestral for Developer)
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: Persistence Layer — Database & Multi-Tenancy
|
||||
|
||||
**Goal**: Add persistent storage and multi-user support
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **PostgreSQL Integration**
|
||||
- Docker compose configuration
|
||||
- Database schema with tenant isolation
|
||||
- Alembic migrations
|
||||
- SQLAlchemy models
|
||||
|
||||
2. **Multi-Tenant Architecture**
|
||||
- Tenant identification middleware
|
||||
- Tenant-scoped database sessions
|
||||
- User authentication system
|
||||
- Per-tenant data isolation
|
||||
|
||||
3. **Core Data Models**
|
||||
- Users and tenants
|
||||
- Conversations and messages (migrate from in-memory)
|
||||
- Agent interactions log
|
||||
- System configuration and preferences
|
||||
|
||||
### Success Criteria
|
||||
- [ ] PostgreSQL container running
|
||||
- [ ] Multiple users authenticate separately
|
||||
- [ ] Each user sees only their own data
|
||||
- [ ] Conversations persist across restarts
|
||||
- [ ] Database migrations work correctly
|
||||
|
||||
---
|
||||
|
||||
## Phase 7: MCP (Model Context Protocol) Integration
|
||||
|
||||
**Goal**: Enable rich tool integrations via MCP
|
||||
|
||||
See also [claude-integration.md](claude-integration.md) for MCP server implementation details.
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **MCP Server Framework**
|
||||
- MCP server implementation
|
||||
- Tool registration via MCP
|
||||
- Schema validation
|
||||
- Error handling
|
||||
|
||||
2. **MCP Client in Agents**
|
||||
- PydanticAI MCP integration
|
||||
- Tool discovery from MCP servers
|
||||
- Dynamic tool loading
|
||||
|
||||
3. **Initial MCP Tools**
|
||||
- File system operations
|
||||
- Database queries
|
||||
- API integrations
|
||||
- System commands
|
||||
|
||||
### Success Criteria
|
||||
- [ ] MCP server running
|
||||
- [ ] Tools exposed via MCP protocol
|
||||
- [ ] Agents can discover and use MCP tools
|
||||
- [ ] New tools addable without code changes
|
||||
- [ ] MCP tools visible in Steward recommendations
|
||||
|
||||
---
|
||||
|
||||
## Phase 8: Advanced Memory & Context — Remaining Work
|
||||
|
||||
**Goal**: Implement sophisticated context management and personalization
|
||||
|
||||
### Open Deliverables
|
||||
|
||||
1. **Context Management**
|
||||
- Smart context window trimming
|
||||
- Conversation branching
|
||||
- Topic tracking
|
||||
|
||||
2. **Personalization**
|
||||
- User preference learning
|
||||
- Interaction pattern analysis
|
||||
- Adaptive responses
|
||||
- Custom agent personalities per user
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Conversations automatically embedded to Qdrant
|
||||
- [ ] Memory improves over time (learning from interactions)
|
||||
|
||||
---
|
||||
|
||||
## Phase 9: Extended Household Staff
|
||||
|
||||
**Goal**: Add specialized agents for additional domains
|
||||
|
||||
### Future Agents
|
||||
- **The Accountant** — Expense tracking, budgets, financial reports
|
||||
- **The Chef** — Meal planning, recipes, nutrition tracking
|
||||
- Others as needs emerge
|
||||
|
||||
---
|
||||
|
||||
## Phase 10: User Experience Refinement
|
||||
|
||||
**Goal**: Polish the interaction experience
|
||||
|
||||
- Personality tuning and consistency
|
||||
- Better progress indicators
|
||||
- Response time improvements
|
||||
- Streaming smoothness
|
||||
|
||||
---
|
||||
|
||||
## Phase 11: Production Hardening
|
||||
|
||||
**Goal**: Make the system production-ready for homelab deployment
|
||||
|
||||
- Complete docker-compose stack
|
||||
- Health checks and monitoring
|
||||
- Authentication hardening and rate limiting
|
||||
- Installation and troubleshooting documentation
|
||||
|
||||
---
|
||||
|
||||
## Dependencies
|
||||
|
||||
```
|
||||
Phase 4 (Remaining Agents)
|
||||
↓
|
||||
Phase 5 (Database/Multi-Tenancy) ← Can be deferred
|
||||
↓
|
||||
Phase 7 (MCP) → Phase 8 (Advanced Memory)
|
||||
↓
|
||||
Phase 9 (Extended Staff) → Phase 10 (UX) → Phase 11 (Production)
|
||||
```
|
||||
|
||||
**Can Be Deferred**: Phase 5 until you need persistence
|
||||
**Parallel Opportunities**: Phases 7 and 8 can overlap; 9 and 10 ongoing
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Implement The Developer agent for code assistance
|
||||
2. Add Home Assistant integration for The Housekeeper
|
||||
3. Integrate scheduling service for The Secretary
|
||||
4. MCP server for external Claude access
|
||||
File diff suppressed because it is too large
Load Diff
+60
-3
@@ -4,17 +4,69 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "tatlock"
|
||||
version = "1.9.0"
|
||||
version = "2.3.0"
|
||||
description = "OpenAI-compatible API with Ollama backend"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = []
|
||||
dependencies = [
|
||||
"fastapi>=0.123,<0.124",
|
||||
"uvicorn[standard]>=0.38,<0.39",
|
||||
"pydantic>=2.11,<2.13",
|
||||
"pydantic-settings>=2.12,<2.13",
|
||||
"pydantic-ai-slim[openai,anthropic]>=1.27,<1.28",
|
||||
"anthropic>=0.77,<1.0",
|
||||
"httpx>=0.28,<0.29",
|
||||
"sse-starlette>=3.0,<3.1",
|
||||
"python-dotenv>=1.2,<1.3",
|
||||
"starlette>=0.45,<0.46",
|
||||
"redis[hiredis]>=5.2,<6.0",
|
||||
"qdrant-client>=1.12,<2.0",
|
||||
"structlog>=24.1,<25.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.3,<8.4",
|
||||
"pytest-asyncio>=0.25,<0.26",
|
||||
"pytest-cov>=6.0,<6.1",
|
||||
"pytest-mock>=3.14,<3.15",
|
||||
"ruff>=0.8,<0.9",
|
||||
"mypy>=1.14,<1.15",
|
||||
"faker>=34.0,<35.0",
|
||||
"coverage[toml]>=7.7,<7.8",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
python_classes = ["Test*"]
|
||||
python_functions = ["test_*"]
|
||||
asyncio_mode = "auto"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
cache_dir = ".cache/pytest"
|
||||
markers = [
|
||||
"unit: Unit tests",
|
||||
"integration: Integration tests",
|
||||
"slow: Slow running tests",
|
||||
"contract: Wire-level contract tests against live service boundaries",
|
||||
]
|
||||
addopts = [
|
||||
"--verbose",
|
||||
"--strict-markers",
|
||||
"--tb=short",
|
||||
"--cov=src",
|
||||
"--cov-report=term-missing",
|
||||
"--cov-report=html:build/coverage/html",
|
||||
"--cov-report=xml:build/coverage/coverage.xml",
|
||||
"--cov-branch",
|
||||
]
|
||||
filterwarnings = [
|
||||
"ignore::DeprecationWarning",
|
||||
]
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["src"]
|
||||
branch = true
|
||||
data_file = "build/coverage/.coverage"
|
||||
omit = [
|
||||
"*/tests/*",
|
||||
"*/__pycache__/*",
|
||||
@@ -37,11 +89,15 @@ exclude_lines = [
|
||||
]
|
||||
|
||||
[tool.coverage.html]
|
||||
directory = "htmlcov"
|
||||
directory = "build/coverage/html"
|
||||
|
||||
[tool.coverage.xml]
|
||||
output = "build/coverage/coverage.xml"
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py312"
|
||||
cache-dir = ".cache/ruff"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
@@ -64,6 +120,7 @@ ignore = [
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.12"
|
||||
cache_dir = ".cache/mypy"
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
disallow_untyped_defs = true
|
||||
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
[pytest]
|
||||
testpaths = tests
|
||||
python_files = test_*.py
|
||||
python_classes = Test*
|
||||
python_functions = test_*
|
||||
asyncio_mode = auto
|
||||
asyncio_default_fixture_loop_scope = function
|
||||
|
||||
# Markers
|
||||
markers =
|
||||
unit: Unit tests
|
||||
integration: Integration tests
|
||||
slow: Slow running tests
|
||||
|
||||
# Coverage options (overridden by pyproject.toml)
|
||||
addopts =
|
||||
--verbose
|
||||
--strict-markers
|
||||
--tb=short
|
||||
--cov=src
|
||||
--cov-report=term-missing
|
||||
--cov-report=html
|
||||
--cov-report=xml
|
||||
--cov-branch
|
||||
|
||||
# Ignore warnings from dependencies
|
||||
filterwarnings =
|
||||
ignore::DeprecationWarning
|
||||
@@ -1,25 +0,0 @@
|
||||
# Development and Testing Dependencies
|
||||
# Install with: pip install -r requirements.txt -r requirements-dev.txt
|
||||
|
||||
# Testing Framework
|
||||
# Latest pytest with async support
|
||||
pytest>=8.3,<8.4
|
||||
pytest-asyncio>=0.25,<0.26
|
||||
pytest-cov>=6.0,<6.1
|
||||
|
||||
# Test client for FastAPI
|
||||
httpx>=0.28,<0.29 # Already in requirements.txt but needed for test client
|
||||
|
||||
# Code Quality
|
||||
# Linting and formatting
|
||||
ruff>=0.8,<0.9
|
||||
|
||||
# Type checking
|
||||
mypy>=1.14,<1.15
|
||||
|
||||
# Testing utilities
|
||||
pytest-mock>=3.14,<3.15
|
||||
faker>=34.0,<35.0
|
||||
|
||||
# Coverage reporting
|
||||
coverage[toml]>=7.7,<7.8
|
||||
@@ -1,61 +0,0 @@
|
||||
# Core FastAPI framework and server
|
||||
# FastAPI: Modern, fast web framework for building APIs
|
||||
# Latest: 0.123.9 (Dec 4, 2025) - No known CVEs
|
||||
fastapi>=0.123,<0.124
|
||||
|
||||
# ASGI server for running FastAPI
|
||||
# Latest: 0.38.0 (Oct 18, 2025) - No known CVEs
|
||||
# Note: Old versions had CVE-2020-7694/7695, but 0.38.0 is secure
|
||||
uvicorn[standard]>=0.38,<0.39
|
||||
|
||||
# Additional dependencies
|
||||
# Pydantic for data validation (comes with pydantic-ai but pinning explicitly)
|
||||
# Updated to >=2.11 due to ag-ui-protocol dependency requirement
|
||||
# Latest: 2.12.4 (Nov 5, 2025) - No known CVEs
|
||||
pydantic>=2.11,<2.13
|
||||
|
||||
# Pydantic settings for configuration management
|
||||
# Required explicitly since pydantic-ai-slim doesn't include it
|
||||
# Latest: 2.12.0 (Dec 2025) - No known CVEs
|
||||
pydantic-settings>=2.12,<2.13
|
||||
|
||||
# AI/LLM integration
|
||||
# PydanticAI: Agent framework for using Pydantic with LLMs
|
||||
# Using slim version with only openai extra (Ollama uses OpenAI-compatible API)
|
||||
# This avoids installing SDKs for anthropic, cohere, google, groq, huggingface, etc.
|
||||
# See DEPENDENCY_SLIM.md for rollback instructions if this breaks
|
||||
pydantic-ai-slim[openai]>=1.27,<1.28
|
||||
|
||||
# HTTP client for Ollama communication
|
||||
# Latest: 0.28.1 - No known CVEs
|
||||
httpx>=0.28,<0.29
|
||||
|
||||
# Server-Sent Events for streaming responses
|
||||
# Required for OpenAI-compatible streaming endpoints
|
||||
# Latest: 3.0.2 (Oct 30, 2025) - No known CVEs
|
||||
sse-starlette>=3.0,<3.1
|
||||
|
||||
# Configuration management
|
||||
# Latest: 1.2.1 (Oct 26, 2025) - No known CVEs
|
||||
python-dotenv>=1.2,<1.3
|
||||
|
||||
# ASGI toolkit (dependency of FastAPI, pinning for security)
|
||||
starlette>=0.45,<0.46
|
||||
|
||||
# Redis for performance benchmarking and caching
|
||||
# Latest: 5.2.1 (Dec 5, 2025) - No known CVEs
|
||||
# hiredis: C parser for better performance
|
||||
redis[hiredis]>=5.2,<6.0
|
||||
|
||||
# Qdrant vector database client for memory storage
|
||||
# Latest: 1.12.1 (Dec 2025) - No known CVEs
|
||||
qdrant-client>=1.12,<2.0
|
||||
|
||||
# Structured logging for observability
|
||||
# Latest: 24.4.0 (Aug 22, 2024) - No known CVEs
|
||||
structlog>=24.1,<25.0
|
||||
|
||||
# Note on version locking strategy:
|
||||
# Using >=X.Y,<X.(Y+1) format to lock to minor versions
|
||||
# This protects against supply chain attacks while allowing patch updates
|
||||
# Update regularly and review changelogs before upgrading minor versions
|
||||
@@ -0,0 +1,542 @@
|
||||
"""
|
||||
Benchmark tool calling across different Ollama models via Tatlock API.
|
||||
|
||||
Sends test prompts through the full Tatlock pipeline (Steward -> Orchestration
|
||||
-> Synthesis) and records tool selection accuracy, latency, and response quality.
|
||||
|
||||
Between models, swaps OLLAMA_DEFAULT_MODEL in .env and waits for uvicorn
|
||||
auto-reload. Requires the server to be running via ./wakeup.sh.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python scripts/benchmark_tool_calling.py
|
||||
.venv/bin/python scripts/benchmark_tool_calling.py --models "gemma4:e4b,gemma4:e2b"
|
||||
.venv/bin/python scripts/benchmark_tool_calling.py --iterations 3
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
API_BASE = "http://localhost:8777"
|
||||
CHAT_URL = f"{API_BASE}/v1/chat/completions"
|
||||
HEALTH_URL = f"{API_BASE}/health"
|
||||
OLLAMA_URL = "http://localhost:11434"
|
||||
ENV_PATH = Path(__file__).parent.parent / ".env"
|
||||
|
||||
DEFAULT_MODELS = ["mistral-nemo-large:latest", "gemma4:e4b", "gemma4:e2b"]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test scenarios
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Scenario:
|
||||
name: str
|
||||
prompt: str
|
||||
expected_tool: str | None # None = no tool expected
|
||||
# Patterns to check in the response text for indirect tool-use evidence
|
||||
success_patterns: list[str] = field(default_factory=list)
|
||||
category: str = "basic"
|
||||
|
||||
|
||||
SCENARIOS = [
|
||||
# --- Should call calculate_math ---
|
||||
Scenario(
|
||||
name="Simple arithmetic",
|
||||
prompt="What is 144 divided by 12?",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["12"],
|
||||
category="calculator",
|
||||
),
|
||||
Scenario(
|
||||
name="Square root",
|
||||
prompt="What's the square root of 256?",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["16"],
|
||||
category="calculator",
|
||||
),
|
||||
Scenario(
|
||||
name="Complex math",
|
||||
prompt="Calculate pi times the square of 5",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["78.5"], # pi * 25 ≈ 78.54
|
||||
category="calculator",
|
||||
),
|
||||
Scenario(
|
||||
name="Word problem",
|
||||
prompt="If I have 3 bags with 17 apples each and I eat 4, how many apples do I have?",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["47"],
|
||||
category="calculator",
|
||||
),
|
||||
|
||||
# --- Should call get_current_time ---
|
||||
Scenario(
|
||||
name="Current date",
|
||||
prompt="What's today's date?",
|
||||
expected_tool="get_current_time",
|
||||
success_patterns=["2026"], # Should contain current year
|
||||
category="datetime",
|
||||
),
|
||||
Scenario(
|
||||
name="Current time",
|
||||
prompt="What time is it right now?",
|
||||
expected_tool="get_current_time",
|
||||
success_patterns=[":"], # Time format contains colons
|
||||
category="datetime",
|
||||
),
|
||||
|
||||
# --- Should call calculate_date_offset ---
|
||||
Scenario(
|
||||
name="Relative date past",
|
||||
prompt="What was the date 2 weeks ago?",
|
||||
expected_tool="calculate_date_offset",
|
||||
success_patterns=["2026"],
|
||||
category="datetime",
|
||||
),
|
||||
|
||||
# --- Should call calculate_time_difference ---
|
||||
Scenario(
|
||||
name="Date difference",
|
||||
prompt="How many days between January 1st 2025 and March 15th 2025?",
|
||||
expected_tool="calculate_time_difference",
|
||||
success_patterns=["73", "74"], # 73 or 74 days
|
||||
category="datetime",
|
||||
),
|
||||
|
||||
# --- Should NOT call any tool ---
|
||||
Scenario(
|
||||
name="Greeting",
|
||||
prompt="Hello! How are you?",
|
||||
expected_tool=None,
|
||||
success_patterns=["sir"], # Butler personality
|
||||
category="no_tool",
|
||||
),
|
||||
Scenario(
|
||||
name="Knowledge question",
|
||||
prompt="What is the capital of France?",
|
||||
expected_tool=None,
|
||||
success_patterns=["Paris"],
|
||||
category="no_tool",
|
||||
),
|
||||
Scenario(
|
||||
name="Opinion request",
|
||||
prompt="What do you think about rainy days?",
|
||||
expected_tool=None,
|
||||
category="no_tool",
|
||||
),
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunResult:
|
||||
scenario: str
|
||||
model: str
|
||||
iteration: int
|
||||
latency: float
|
||||
response_text: str
|
||||
has_correct_answer: bool
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelStats:
|
||||
model: str
|
||||
results: list[RunResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return len(self.results)
|
||||
|
||||
@property
|
||||
def errors(self) -> int:
|
||||
return sum(1 for r in self.results if r.error)
|
||||
|
||||
@property
|
||||
def accuracy(self) -> float:
|
||||
valid = [r for r in self.results if not r.error]
|
||||
if not valid:
|
||||
return 0
|
||||
return sum(1 for r in valid if r.has_correct_answer) / len(valid) * 100
|
||||
|
||||
@property
|
||||
def avg_latency(self) -> float:
|
||||
lats = [r.latency for r in self.results if not r.error]
|
||||
return statistics.mean(lats) if lats else 0
|
||||
|
||||
@property
|
||||
def p95_latency(self) -> float:
|
||||
lats = sorted(r.latency for r in self.results if not r.error)
|
||||
if not lats:
|
||||
return 0
|
||||
return lats[min(int(len(lats) * 0.95), len(lats) - 1)]
|
||||
|
||||
@property
|
||||
def max_latency(self) -> float:
|
||||
lats = [r.latency for r in self.results if not r.error]
|
||||
return max(lats) if lats else 0
|
||||
|
||||
def category_accuracy(self, category: str) -> float:
|
||||
cat_scenarios = {s.name for s in SCENARIOS if s.category == category}
|
||||
valid = [r for r in self.results if not r.error and r.scenario in cat_scenarios]
|
||||
if not valid:
|
||||
return 0
|
||||
return sum(1 for r in valid if r.has_correct_answer) / len(valid) * 100
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# .env manipulation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def swap_model_in_env(model_name: str):
|
||||
"""Swap OLLAMA_DEFAULT_MODEL in .env file."""
|
||||
content = ENV_PATH.read_text()
|
||||
content = re.sub(
|
||||
r'^OLLAMA_DEFAULT_MODEL=.*$',
|
||||
f'OLLAMA_DEFAULT_MODEL={model_name}',
|
||||
content,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
ENV_PATH.write_text(content)
|
||||
print(f" .env updated: OLLAMA_DEFAULT_MODEL={model_name}")
|
||||
|
||||
|
||||
async def wait_for_server_reload(client: httpx.AsyncClient, timeout: float = 30):
|
||||
"""Wait for uvicorn to auto-reload after .env change."""
|
||||
# Give uvicorn a moment to detect the file change
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Poll health endpoint
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
r = await client.get(HEALTH_URL, timeout=5)
|
||||
if r.status_code == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(1)
|
||||
|
||||
raise TimeoutError("Server did not come back after reload")
|
||||
|
||||
|
||||
async def warm_up_ollama_model(client: httpx.AsyncClient, model_name: str):
|
||||
"""Send a throwaway request to load the model into VRAM."""
|
||||
print(f" Warming up {model_name} in Ollama...", end=" ", flush=True)
|
||||
try:
|
||||
r = await client.post(
|
||||
f"{OLLAMA_URL}/api/generate",
|
||||
json={"model": model_name, "prompt": "hi", "stream": False},
|
||||
timeout=120,
|
||||
)
|
||||
r.raise_for_status()
|
||||
duration = r.json().get("total_duration", 0) / 1e9
|
||||
print(f"OK ({duration:.1f}s)")
|
||||
except Exception as e:
|
||||
print(f"WARN: {e}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core benchmark logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def run_scenario(
|
||||
client: httpx.AsyncClient,
|
||||
scenario: Scenario,
|
||||
model: str,
|
||||
iteration: int,
|
||||
) -> RunResult:
|
||||
"""Run a single scenario through the Tatlock API."""
|
||||
payload = {
|
||||
"model": "Tatlock",
|
||||
"messages": [{"role": "user", "content": scenario.prompt}],
|
||||
}
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r = await client.post(CHAT_URL, json=payload, timeout=120)
|
||||
latency = time.monotonic() - start
|
||||
|
||||
if r.status_code != 200:
|
||||
return RunResult(
|
||||
scenario=scenario.name,
|
||||
model=model,
|
||||
iteration=iteration,
|
||||
latency=latency,
|
||||
response_text="",
|
||||
has_correct_answer=False,
|
||||
error=f"HTTP {r.status_code}: {r.text[:100]}",
|
||||
)
|
||||
|
||||
data = r.json()
|
||||
response_text = data["choices"][0]["message"]["content"]
|
||||
|
||||
# Check if the response contains expected patterns
|
||||
has_correct = True
|
||||
if scenario.success_patterns:
|
||||
has_correct = any(
|
||||
p.lower() in response_text.lower()
|
||||
for p in scenario.success_patterns
|
||||
)
|
||||
|
||||
return RunResult(
|
||||
scenario=scenario.name,
|
||||
model=model,
|
||||
iteration=iteration,
|
||||
latency=latency,
|
||||
response_text=response_text,
|
||||
has_correct_answer=has_correct,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
latency = time.monotonic() - start
|
||||
return RunResult(
|
||||
scenario=scenario.name,
|
||||
model=model,
|
||||
iteration=iteration,
|
||||
latency=latency,
|
||||
response_text="",
|
||||
has_correct_answer=False,
|
||||
error=str(e)[:200],
|
||||
)
|
||||
|
||||
|
||||
async def benchmark_model(
|
||||
client: httpx.AsyncClient,
|
||||
model_name: str,
|
||||
iterations: int,
|
||||
) -> ModelStats:
|
||||
"""Run all scenarios for a single model."""
|
||||
stats = ModelStats(model=model_name)
|
||||
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f" Model: {model_name}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
# Swap model in .env
|
||||
swap_model_in_env(model_name)
|
||||
|
||||
# Warm up model in Ollama BEFORE server reload picks it up
|
||||
await warm_up_ollama_model(client, model_name)
|
||||
|
||||
# Wait for server to reload with new model
|
||||
print(" Waiting for server reload...", end=" ", flush=True)
|
||||
await wait_for_server_reload(client)
|
||||
print("OK")
|
||||
|
||||
# Run a throwaway request through the full pipeline to warm up
|
||||
print(" Warming up pipeline...", end=" ", flush=True)
|
||||
try:
|
||||
await client.post(
|
||||
CHAT_URL,
|
||||
json={"model": "Tatlock", "messages": [{"role": "user", "content": "hi"}]},
|
||||
timeout=120,
|
||||
)
|
||||
print("OK")
|
||||
except Exception as e:
|
||||
print(f"WARN: {e}")
|
||||
|
||||
for iteration in range(iterations):
|
||||
if iterations > 1:
|
||||
print(f"\n --- Iteration {iteration + 1}/{iterations} ---")
|
||||
|
||||
for scenario in SCENARIOS:
|
||||
result = await run_scenario(client, scenario, model_name, iteration)
|
||||
stats.results.append(result)
|
||||
|
||||
# Display
|
||||
if result.error:
|
||||
print(
|
||||
f" [ERR ] {scenario.name:30s} {result.latency:5.1f}s "
|
||||
f"{result.error[:60]}"
|
||||
)
|
||||
elif result.has_correct_answer:
|
||||
preview = result.response_text[:60].replace("\n", " ")
|
||||
print(f" [OK ] {scenario.name:30s} {result.latency:5.1f}s {preview}")
|
||||
else:
|
||||
preview = result.response_text[:60].replace("\n", " ")
|
||||
print(f" [MISS] {scenario.name:30s} {result.latency:5.1f}s {preview}")
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def print_comparison(all_stats: list[ModelStats]):
|
||||
"""Print side-by-side comparison table."""
|
||||
print("\n" + "=" * 80)
|
||||
print(" COMPARISON SUMMARY")
|
||||
print("=" * 80)
|
||||
|
||||
col_width = max(len(s.model) for s in all_stats) + 2
|
||||
label_width = 32
|
||||
|
||||
header = f"{'Metric':<{label_width}}"
|
||||
for s in all_stats:
|
||||
header += f" {s.model:>{col_width}}"
|
||||
print(f"\n{header}")
|
||||
print("-" * (label_width + (col_width + 2) * len(all_stats)))
|
||||
|
||||
# Answer accuracy
|
||||
row = f"{'Correct answer rate':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.accuracy:>{col_width - 1}.1f}%"
|
||||
print(row)
|
||||
|
||||
# Latency
|
||||
row = f"{'Avg latency':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.avg_latency:>{col_width - 1}.1f}s"
|
||||
print(row)
|
||||
|
||||
row = f"{'P95 latency':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.p95_latency:>{col_width - 1}.1f}s"
|
||||
print(row)
|
||||
|
||||
row = f"{'Max latency':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.max_latency:>{col_width - 1}.1f}s"
|
||||
print(row)
|
||||
|
||||
# Errors
|
||||
row = f"{'Errors':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.errors:>{col_width}}"
|
||||
print(row)
|
||||
|
||||
# Per-category
|
||||
categories = sorted(set(sc.category for sc in SCENARIOS))
|
||||
print(f"\n{'Per-category accuracy':<{label_width}}")
|
||||
print("-" * (label_width + (col_width + 2) * len(all_stats)))
|
||||
for cat in categories:
|
||||
row = f" {cat:<{label_width - 2}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.category_accuracy(cat):>{col_width - 1}.1f}%"
|
||||
print(row)
|
||||
|
||||
# Mismatches
|
||||
print(f"\n{'Missed answers':<50}")
|
||||
print("-" * 80)
|
||||
any_miss = False
|
||||
for scenario in SCENARIOS:
|
||||
misses = []
|
||||
for s in all_stats:
|
||||
sc_results = [r for r in s.results if r.scenario == scenario.name]
|
||||
fails = [r for r in sc_results if not r.has_correct_answer and not r.error]
|
||||
if fails:
|
||||
preview = fails[0].response_text[:50].replace("\n", " ")
|
||||
misses.append(f"{s.model}: \"{preview}\"")
|
||||
if misses:
|
||||
any_miss = True
|
||||
print(f" {scenario.name}")
|
||||
for m in misses:
|
||||
print(f" {m}")
|
||||
|
||||
if not any_miss:
|
||||
print(" (none)")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
|
||||
|
||||
def save_results(all_stats: list[ModelStats], output_path: Path):
|
||||
"""Save detailed results to JSON."""
|
||||
data = {}
|
||||
for stats in all_stats:
|
||||
data[stats.model] = {
|
||||
"summary": {
|
||||
"accuracy": stats.accuracy,
|
||||
"avg_latency": round(stats.avg_latency, 2),
|
||||
"p95_latency": round(stats.p95_latency, 2),
|
||||
"max_latency": round(stats.max_latency, 2),
|
||||
"errors": stats.errors,
|
||||
"total_runs": stats.total,
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"scenario": r.scenario,
|
||||
"iteration": r.iteration,
|
||||
"latency": round(r.latency, 3),
|
||||
"has_correct_answer": r.has_correct_answer,
|
||||
"response_text": r.response_text,
|
||||
"error": r.error,
|
||||
}
|
||||
for r in stats.results
|
||||
],
|
||||
}
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(data, indent=2))
|
||||
print(f"\nDetailed results saved to: {output_path}")
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description="Benchmark tool calling across Ollama models via Tatlock API")
|
||||
parser.add_argument(
|
||||
"--iterations", type=int, default=1,
|
||||
help="Iterations per model (default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--models", type=str, default=",".join(DEFAULT_MODELS),
|
||||
help=f"Comma-separated models (default: {','.join(DEFAULT_MODELS)})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", type=str, default="logs/benchmark_results.json",
|
||||
help="JSON output path (default: logs/benchmark_results.json)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
models = [m.strip() for m in args.models.split(",")]
|
||||
|
||||
# Verify server is running
|
||||
async with httpx.AsyncClient() as client:
|
||||
try:
|
||||
r = await client.get(HEALTH_URL, timeout=5)
|
||||
r.raise_for_status()
|
||||
print("Server is running.")
|
||||
except Exception:
|
||||
print("ERROR: Server not running. Start it with ./wakeup.sh first.")
|
||||
return
|
||||
|
||||
print("=" * 70)
|
||||
print(" Tool Calling Benchmark (via Tatlock API)")
|
||||
print("=" * 70)
|
||||
print(f" Models: {', '.join(models)}")
|
||||
print(f" Scenarios: {len(SCENARIOS)}")
|
||||
print(f" Iterations: {args.iterations}")
|
||||
print(f" Total runs: {len(SCENARIOS) * args.iterations * len(models)}")
|
||||
|
||||
# Remember original model to restore after benchmark
|
||||
original_env = ENV_PATH.read_text()
|
||||
|
||||
all_stats = []
|
||||
async with httpx.AsyncClient() as client:
|
||||
for model in models:
|
||||
stats = await benchmark_model(client, model, args.iterations)
|
||||
all_stats.append(stats)
|
||||
|
||||
# Restore original .env
|
||||
ENV_PATH.write_text(original_env)
|
||||
print(f"\n .env restored to original")
|
||||
|
||||
print_comparison(all_stats)
|
||||
save_results(all_stats, Path(args.output))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -102,16 +102,10 @@ _biographer_agent: Optional[Agent[None, str]] = None
|
||||
|
||||
def _create_biographer_agent() -> Agent[None, str]:
|
||||
"""Create The Biographer PydanticAI agent."""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
# Create Ollama model with sanitized provider
|
||||
# (fixes 'content: null' issue with tool calls)
|
||||
model = OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
agent: Agent[None, str] = Agent(
|
||||
model=model,
|
||||
@@ -131,9 +125,12 @@ def _create_biographer_agent() -> Agent[None, str]:
|
||||
# Register management tools
|
||||
agent.tool_plain(forget_memory)
|
||||
|
||||
from src.anthropic.model_selector import get_model_info
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"biographer_agent_created",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
tool_count=6,
|
||||
)
|
||||
|
||||
|
||||
+145
-84
@@ -13,6 +13,7 @@ from enum import Enum
|
||||
from typing import AsyncGenerator, Callable, Optional, Any
|
||||
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import trace_span, SpanType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -239,38 +240,58 @@ async def delegate_to_librarian(
|
||||
has_context=bool(context),
|
||||
)
|
||||
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_librarian(task=task, context=context)
|
||||
async with trace_span(
|
||||
"delegate_to_librarian",
|
||||
SpanType.EXPERT,
|
||||
metadata={
|
||||
"expert": "librarian",
|
||||
"task_preview": task[:100],
|
||||
"has_context": bool(context),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_librarian(task=task, context=context)
|
||||
|
||||
logger.info(
|
||||
"delegation_to_librarian_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
logger.info(
|
||||
"delegation_to_librarian_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
if span:
|
||||
span.metadata["success"] = True
|
||||
span.metadata["output_length"] = len(output)
|
||||
span.details["task"] = task
|
||||
span.details["context"] = context[:500] if context else None
|
||||
span.details["result_preview"] = output[:1000]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_librarian_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_librarian_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if span:
|
||||
span.metadata["success"] = False
|
||||
span.details["error"] = str(e)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="librarian",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
async def delegate_to_biographer(
|
||||
@@ -317,38 +338,58 @@ async def delegate_to_biographer(
|
||||
has_context=bool(context),
|
||||
)
|
||||
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_biographer(task=task, context=context)
|
||||
async with trace_span(
|
||||
"delegate_to_biographer",
|
||||
SpanType.EXPERT,
|
||||
metadata={
|
||||
"expert": "biographer",
|
||||
"task_preview": task[:100],
|
||||
"has_context": bool(context),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_biographer(task=task, context=context)
|
||||
|
||||
logger.info(
|
||||
"delegation_to_biographer_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
logger.info(
|
||||
"delegation_to_biographer_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
if span:
|
||||
span.metadata["success"] = True
|
||||
span.metadata["output_length"] = len(output)
|
||||
span.details["task"] = task
|
||||
span.details["context"] = context[:500] if context else None
|
||||
span.details["result_preview"] = output[:1000]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_biographer_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_biographer_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if span:
|
||||
span.metadata["success"] = False
|
||||
span.details["error"] = str(e)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="biographer",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
async def delegate_to_housekeeper(
|
||||
@@ -394,38 +435,58 @@ async def delegate_to_housekeeper(
|
||||
has_context=bool(context),
|
||||
)
|
||||
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_housekeeper(task=task, context=context)
|
||||
async with trace_span(
|
||||
"delegate_to_housekeeper",
|
||||
SpanType.EXPERT,
|
||||
metadata={
|
||||
"expert": "housekeeper",
|
||||
"task_preview": task[:100],
|
||||
"has_context": bool(context),
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
# Use run() not run_stream() - avoids Ollama bug
|
||||
output = await run_housekeeper(task=task, context=context)
|
||||
|
||||
logger.info(
|
||||
"delegation_to_housekeeper_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
logger.info(
|
||||
"delegation_to_housekeeper_completed",
|
||||
task=task[:50],
|
||||
output_length=len(output),
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
if span:
|
||||
span.metadata["success"] = True
|
||||
span.metadata["output_length"] = len(output)
|
||||
span.details["task"] = task
|
||||
span.details["context"] = context[:500] if context else None
|
||||
span.details["result_preview"] = output[:1000]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_housekeeper_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=True,
|
||||
output=output,
|
||||
)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"delegation_to_housekeeper_error",
|
||||
task=task[:50],
|
||||
error=str(e),
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if span:
|
||||
span.metadata["success"] = False
|
||||
span.details["error"] = str(e)
|
||||
|
||||
return DelegationResult(
|
||||
expert_name="housekeeper",
|
||||
task=task,
|
||||
success=False,
|
||||
output="",
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@@ -103,16 +103,10 @@ _housekeeper_agent: Optional[Agent[None, str]] = None
|
||||
|
||||
def _create_housekeeper_agent() -> Agent[None, str]:
|
||||
"""Create the Housekeeper PydanticAI agent."""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
# Create Ollama model with sanitized provider
|
||||
# (fixes 'content: null' issue with tool calls)
|
||||
model = OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
agent: Agent[None, str] = Agent(
|
||||
model=model,
|
||||
@@ -145,9 +139,12 @@ def _create_housekeeper_agent() -> Agent[None, str]:
|
||||
# Register history tools
|
||||
agent.tool_plain(get_history)
|
||||
|
||||
from src.anthropic.model_selector import get_model_info
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"housekeeper_agent_created",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
tool_count=13,
|
||||
)
|
||||
|
||||
@@ -207,13 +204,13 @@ async def run_housekeeper(
|
||||
)
|
||||
|
||||
try:
|
||||
# Use temperature 0.1 for slight exploration
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
# Temperature 0.1 for slight exploration (skipped on Claude backend)
|
||||
from src.anthropic.model_selector import get_sampling_settings
|
||||
|
||||
result = await agent.run(
|
||||
prompt,
|
||||
message_history=message_history,
|
||||
model_settings=ModelSettings(temperature=0.1),
|
||||
model_settings=get_sampling_settings(0.1),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -269,13 +266,13 @@ async def run_housekeeper_stream(
|
||||
)
|
||||
|
||||
try:
|
||||
# Use temperature 0.1 for slight exploration
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
# Temperature 0.1 for slight exploration (skipped on Claude backend)
|
||||
from src.anthropic.model_selector import get_sampling_settings
|
||||
|
||||
async with agent.run_stream(
|
||||
prompt,
|
||||
message_history=message_history,
|
||||
model_settings=ModelSettings(temperature=0.1),
|
||||
model_settings=get_sampling_settings(0.1),
|
||||
) as response:
|
||||
async for delta in response.stream_text(delta=True):
|
||||
yield delta
|
||||
|
||||
@@ -39,7 +39,14 @@ Your role is to help users find, understand, synthesize, and manage information
|
||||
- The personal wiki (Wiki.js) containing documentation and notes
|
||||
- The knowledge graph (Neo4j) with entities and relationships
|
||||
- Vector embeddings (Qdrant) for semantic search
|
||||
- Web search (SearXNG) for current information
|
||||
- Paperless documents (📑) - indexed PDFs, scanned documents, invoices, receipts from the user's document archive
|
||||
- Volatile cache (⚡) - pre-fetched real-time data for user-relevant locations and items:
|
||||
- weather/forecast: conditions and forecasts for user's configured cities
|
||||
- news: headlines from user's preferred sources
|
||||
- stock/crypto: quotes for user's watched symbols
|
||||
- sun/air_quality: data for user's locations
|
||||
- Note: volatile data may not exist for arbitrary queries - falls back to web search
|
||||
- Web search (SearXNG) for current information not available in cache
|
||||
|
||||
## Your Personality
|
||||
- Scholarly and thorough in your research
|
||||
@@ -60,7 +67,13 @@ Your role is to help users find, understand, synthesize, and manage information
|
||||
- Use for: comparing multiple sources, gathering info from several pages
|
||||
|
||||
### Internal Research Tools
|
||||
- **hybrid_search**: Your primary research tool - searches wiki, graph, and web at once
|
||||
- **hybrid_search**: Your primary research tool - searches ALL sources at once:
|
||||
- Wiki pages (vector similarity)
|
||||
- Knowledge graph (entity relationships)
|
||||
- Paperless documents (📑 indexed PDFs, scans)
|
||||
- Volatile cache (⚡ weather, news, stocks - when available)
|
||||
- Web search (current information)
|
||||
Results are fused and re-ranked by relevance. Volatile data gets priority when fresh.
|
||||
- **search_wiki**: Find specific wiki pages by keyword
|
||||
- **semantic_search**: Find conceptually similar content
|
||||
- **explore_knowledge_graph** / **find_related_entities**: Discover connections
|
||||
@@ -114,6 +127,14 @@ Your responses are returned to Tatlock (the butler) who will synthesize them int
|
||||
- Note any gaps in available information
|
||||
- Be concise but thorough - Tatlock will format the final response
|
||||
- Structure your findings clearly so they can be easily integrated with other responses
|
||||
|
||||
## CRITICAL: Never Fabricate Information
|
||||
If a tool fails or you cannot access a data source:
|
||||
- Say "I was unable to retrieve [information type]" - be specific about what failed
|
||||
- Do NOT provide placeholder, template, or made-up data
|
||||
- Do NOT say "Here's what I would have said" or "Here's a sample response"
|
||||
- Do NOT invent specific numbers, dates, or facts when the actual data is unavailable
|
||||
- It is better to return no information than to return fabricated information
|
||||
"""
|
||||
|
||||
# Lazy initialization to avoid connection issues during imports
|
||||
@@ -122,16 +143,10 @@ _librarian_agent: Optional[Agent[None, str]] = None
|
||||
|
||||
def _create_librarian_agent() -> Agent[None, str]:
|
||||
"""Create the Librarian PydanticAI agent."""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
# Create Ollama model with sanitized provider
|
||||
# (fixes 'content: null' issue with tool calls)
|
||||
model = OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
agent: Agent[None, str] = Agent(
|
||||
model=model,
|
||||
@@ -161,9 +176,12 @@ def _create_librarian_agent() -> Agent[None, str]:
|
||||
agent.tool_plain(update_wiki_page)
|
||||
agent.tool_plain(smart_create_wiki_page)
|
||||
|
||||
from src.anthropic.model_selector import get_model_info
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"librarian_agent_created",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
tool_count=14, # 7 research + 3 web + 1 wiki read + 3 wiki write
|
||||
)
|
||||
|
||||
|
||||
@@ -225,18 +225,22 @@ class LibraryDeskClient:
|
||||
vector_limit: int = 10,
|
||||
graph_limit: int = 10,
|
||||
web_limit: int = 5,
|
||||
document_limit: int = 5,
|
||||
volatile_limit: int = 3,
|
||||
enable_reranking: bool = True,
|
||||
final_result_count: int = 10,
|
||||
) -> HybridRAGResponse:
|
||||
"""
|
||||
Execute HybridRAG search combining vector, graph, and web results.
|
||||
Execute HybridRAG search combining vector, graph, documents, volatile, and web.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
user: User identifier for multi-tenancy (defaults to request context)
|
||||
vector_limit: Max results from vector search
|
||||
vector_limit: Max results from vector search (wiki pages)
|
||||
graph_limit: Max results from graph search
|
||||
web_limit: Max results from web search
|
||||
web_limit: Max results from web search (0 to disable)
|
||||
document_limit: Max results from Paperless documents (0 to disable)
|
||||
volatile_limit: Max results from volatile cache (0 to disable)
|
||||
enable_reranking: Whether to rerank with LLM
|
||||
final_result_count: Number of final results after fusion
|
||||
|
||||
@@ -252,6 +256,11 @@ class LibraryDeskClient:
|
||||
"vector_limit": vector_limit,
|
||||
"graph_limit": graph_limit,
|
||||
"web_limit": web_limit,
|
||||
"document_limit": document_limit,
|
||||
"volatile_limit": volatile_limit,
|
||||
"enable_documents": document_limit > 0,
|
||||
"enable_volatile": volatile_limit > 0,
|
||||
"enable_web": web_limit > 0,
|
||||
"enable_reranking": enable_reranking,
|
||||
"final_result_count": final_result_count,
|
||||
},
|
||||
|
||||
@@ -17,20 +17,26 @@ logger = get_logger(__name__)
|
||||
async def hybrid_search(
|
||||
query: str,
|
||||
include_web: bool = True,
|
||||
include_documents: bool = True,
|
||||
include_volatile: bool = True,
|
||||
) -> str:
|
||||
"""
|
||||
Search across all knowledge sources using HybridRAG.
|
||||
|
||||
This is the primary research tool, combining:
|
||||
- Vector search (semantic similarity over documents)
|
||||
- Vector search (semantic similarity over wiki pages)
|
||||
- Knowledge graph (entities and relationships)
|
||||
- Paperless documents (📑 indexed PDFs, scans, invoices)
|
||||
- Volatile cache (⚡ weather, news, stocks - for user's configured items)
|
||||
- Web search (current information from SearXNG)
|
||||
|
||||
Results are fused and re-ranked by relevance.
|
||||
Results are fused and re-ranked by relevance. Volatile data gets priority when fresh.
|
||||
|
||||
Args:
|
||||
query: Natural language research query
|
||||
include_web: Whether to include web results (default: True)
|
||||
include_documents: Whether to include Paperless documents (default: True)
|
||||
include_volatile: Whether to include volatile cache data (default: True)
|
||||
|
||||
Returns:
|
||||
Formatted search results with sources and context
|
||||
@@ -38,12 +44,16 @@ async def hybrid_search(
|
||||
Examples:
|
||||
hybrid_search("How does Docker orchestration work with Kubernetes?")
|
||||
hybrid_search("What projects use Neo4j?", include_web=False)
|
||||
hybrid_search("Find my electricity invoices", include_web=False, include_volatile=False)
|
||||
hybrid_search("What's the weather in Rotterdam?") # May hit volatile cache
|
||||
"""
|
||||
try:
|
||||
async with LibraryDeskClient() as client:
|
||||
response = await client.hybrid_search(
|
||||
query=query,
|
||||
web_limit=5 if include_web else 0,
|
||||
document_limit=5 if include_documents else 0,
|
||||
volatile_limit=3 if include_volatile else 0,
|
||||
)
|
||||
|
||||
if not response.results:
|
||||
@@ -70,6 +80,8 @@ async def hybrid_search(
|
||||
"vector": "📄",
|
||||
"graph": "🔗",
|
||||
"web": "🌐",
|
||||
"document": "📑",
|
||||
"volatile": "⚡",
|
||||
}.get(result.source, "•")
|
||||
|
||||
output_parts.append(
|
||||
|
||||
+125
-30
@@ -5,11 +5,13 @@ The Steward analyzes incoming requests, identifies relevant household
|
||||
capabilities, and provides focused recommendations to Tatlock (the Butler).
|
||||
This creates a two-tier architecture that prevents cognitive overload.
|
||||
|
||||
Uses plain text output (not JSON) for reliability with Ollama models.
|
||||
Uses plain text output (not JSON) for reliability. Supports both Claude
|
||||
(preferred) and Ollama (fallback) backends via direct API calls.
|
||||
"""
|
||||
import httpx
|
||||
from typing import Optional
|
||||
|
||||
from src.anthropic.model_selector import get_model_info, is_claude_available, resolve_backend
|
||||
from src.core.config import config
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
@@ -56,14 +58,22 @@ USER QUERY: {query}
|
||||
GUIDELINES:
|
||||
- Be conservative - only recommend truly necessary capabilities
|
||||
- Simple greetings/chat → no capabilities needed (conversational response only)
|
||||
- Questions about prior conversation ("what did I say", "my name", "what we discussed") → no capabilities (Tatlock has full history)
|
||||
- Questions about prior conversation ("what did I say", "what we discussed") → no capabilities (Tatlock has full history)
|
||||
- Math/calculations → tatlock_core
|
||||
- Time/date queries → tatlock_core
|
||||
- PERSONAL MEMORY queries → biographer to recall (ALWAYS use for questions about the user themselves):
|
||||
- "where do I live", "what's my location", "my address" → biographer to recall location
|
||||
- "what's my name", "who am I" → biographer to recall name
|
||||
- "what car do I drive", "my vehicle" → biographer to recall car
|
||||
- "what do you know about me", "what have I told you" → biographer to recall or list_memories
|
||||
- "remember that I...", "store that..." → biographer to store_insight
|
||||
- "forget my...", "delete..." → biographer to forget_memory
|
||||
- "my timezone", "my preferences" → biographer to recall preferences
|
||||
- Web searches, weather, news, current information → librarian with search_web
|
||||
- Read a URL or article → librarian with read_url
|
||||
- Wiki creation ("create a page about X", "add X to wiki") → librarian with smart_create
|
||||
- Wiki updates ("update the page", "add to dossier") → librarian with update
|
||||
- Research queries ("find info", "what do we know about", "search for") → librarian with hybrid_search
|
||||
- Research queries about TOPICS (not about the user) → librarian with hybrid_search
|
||||
- In-depth research, knowledge synthesis, document lookup → librarian with hybrid_search
|
||||
- If conversation history is relevant, note which previous turns matter
|
||||
- Assess complexity: simple (1 tool), moderate (2-3 tools), complex (multiple steps)
|
||||
@@ -75,6 +85,10 @@ COMPLEXITY: [simple/moderate/complex]
|
||||
CONTEXT: [any relevant conversation context, or "none"]
|
||||
|
||||
EXAMPLES:
|
||||
- "DELEGATE: biographer to recall the user's location" (for "where do I live?")
|
||||
- "DELEGATE: biographer to recall the user's car" (for "what car do I drive?")
|
||||
- "DELEGATE: biographer to list_memories about the user" (for "what do you know about me?")
|
||||
- "DELEGATE: biographer to store_insight about user's pet" (for "remember that I have a dog named Max")
|
||||
- "DELEGATE: librarian to search_web for tomorrow's weather forecast"
|
||||
- "DELEGATE: librarian to create a wiki page about CI/CD pipelines"
|
||||
- "DELEGATE: librarian to hybrid_search for information about Docker networking"
|
||||
@@ -93,22 +107,75 @@ class StewardAgent:
|
||||
Analyzes requests with full conversation context and recommends
|
||||
which household capabilities the Butler should use.
|
||||
|
||||
Uses plain text output for reliability with Ollama models.
|
||||
Uses plain text output for reliability. Supports both Claude
|
||||
(preferred) and Ollama (fallback) backends via direct API calls.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Steward with Ollama model (same as Tatlock for VRAM efficiency)."""
|
||||
"""Initialize Steward with backend selection based on availability."""
|
||||
# Ollama config (primary)
|
||||
self.ollama_host = str(config.OLLAMA_HOST).rstrip('/')
|
||||
self.model_name = config.OLLAMA_DEFAULT_MODEL
|
||||
self.timeout = 30.0 # 30 second timeout for analysis
|
||||
self.ollama_model = config.OLLAMA_DEFAULT_MODEL
|
||||
|
||||
# Claude config (fallback)
|
||||
self.claude_model = config.ANTHROPIC_MODEL
|
||||
self._anthropic_client = None
|
||||
|
||||
# Determine which backend to use (Ollama-first, Claude when
|
||||
# preferred via config or when Ollama is down)
|
||||
self._use_claude = resolve_backend() == "claude"
|
||||
|
||||
self.timeout = float(config.STEWARD_TIMEOUT)
|
||||
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"steward_agent_created",
|
||||
ollama_host=self.ollama_host,
|
||||
model=self.model_name,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
def _get_anthropic_client(self):
|
||||
"""Get or create Anthropic client (lazy initialization)."""
|
||||
if self._anthropic_client is None:
|
||||
from anthropic import AsyncAnthropic
|
||||
self._anthropic_client = AsyncAnthropic(api_key=config.ANTHROPIC_API_KEY)
|
||||
return self._anthropic_client
|
||||
|
||||
async def _call_claude(self, system_prompt: str, user_message: str) -> str:
|
||||
"""Call Claude API directly for plain text generation."""
|
||||
client = self._get_anthropic_client()
|
||||
|
||||
# No temperature: rejected by Claude Sonnet 5+ (sampling params deprecated)
|
||||
response = await client.messages.create(
|
||||
model=self.claude_model,
|
||||
max_tokens=1024,
|
||||
system=system_prompt,
|
||||
messages=[{"role": "user", "content": user_message}],
|
||||
)
|
||||
|
||||
return response.content[0].text.strip()
|
||||
|
||||
async def _call_ollama(self, prompt: str) -> str:
|
||||
"""Call Ollama API directly for plain text generation."""
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.ollama_host}/api/generate",
|
||||
json={
|
||||
"model": self.ollama_model,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.3, # Lower = more consistent
|
||||
"top_p": 0.9
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
return result["response"].strip()
|
||||
|
||||
async def analyze(
|
||||
self,
|
||||
query: str,
|
||||
@@ -117,6 +184,8 @@ class StewardAgent:
|
||||
"""
|
||||
Analyze query and return plain text recommendation.
|
||||
|
||||
Uses Claude if available, falls back to Ollama.
|
||||
|
||||
Args:
|
||||
query: User's query to analyze
|
||||
conversation_history: Previous conversation turns
|
||||
@@ -132,35 +201,61 @@ class StewardAgent:
|
||||
history = conversation_history or []
|
||||
prompt = build_steward_prompt(query, history)
|
||||
|
||||
logger.debug("steward_calling_ollama", query_preview=query[:100])
|
||||
backend = "claude" if self._use_claude else "ollama"
|
||||
logger.debug(
|
||||
"steward_calling_llm",
|
||||
backend=backend,
|
||||
query_preview=query[:100],
|
||||
)
|
||||
|
||||
# Call Ollama API directly (more reliable than PydanticAI for plain text)
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.post(
|
||||
f"{self.ollama_host}/api/generate",
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {
|
||||
"temperature": 0.3, # Lower = more consistent
|
||||
"top_p": 0.9
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
|
||||
analysis_text = result["response"].strip()
|
||||
try:
|
||||
if self._use_claude:
|
||||
# For Claude, split into system + user message
|
||||
# The prompt contains both, but Claude prefers explicit system
|
||||
analysis_text = await self._call_claude(
|
||||
system_prompt="You are the Steward of the household, advising the Butler (Tatlock) on which capabilities to use. Be concise and specific.",
|
||||
user_message=prompt,
|
||||
)
|
||||
else:
|
||||
analysis_text = await self._call_ollama(prompt)
|
||||
|
||||
logger.debug(
|
||||
"steward_analysis_received",
|
||||
text_preview=analysis_text[:150]
|
||||
backend=backend,
|
||||
text_preview=analysis_text[:150],
|
||||
)
|
||||
|
||||
return analysis_text
|
||||
|
||||
except Exception as e:
|
||||
# Mid-request fallback: retry on the other backend when possible
|
||||
if self._use_claude:
|
||||
logger.warning(
|
||||
"steward_claude_fallback",
|
||||
error=str(e),
|
||||
)
|
||||
analysis_text = await self._call_ollama(prompt)
|
||||
fallback_backend = "ollama_fallback"
|
||||
elif is_claude_available():
|
||||
logger.warning(
|
||||
"steward_ollama_fallback",
|
||||
error=str(e),
|
||||
)
|
||||
analysis_text = await self._call_claude(
|
||||
system_prompt="You are the Steward of the household, advising the Butler (Tatlock) on which capabilities to use. Be concise and specific.",
|
||||
user_message=prompt,
|
||||
)
|
||||
fallback_backend = "claude_fallback"
|
||||
else:
|
||||
raise
|
||||
|
||||
logger.debug(
|
||||
"steward_analysis_received",
|
||||
backend=fallback_backend,
|
||||
text_preview=analysis_text[:150],
|
||||
)
|
||||
return analysis_text
|
||||
|
||||
|
||||
# Global Steward instance
|
||||
_steward_agent = None
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""
|
||||
Steward service layer.
|
||||
|
||||
Provides high-level interface for request analysis with logging,
|
||||
benchmarking, and error handling.
|
||||
Provides high-level interface for request analysis with logging
|
||||
and error handling.
|
||||
|
||||
Parses plain text recommendations into structured data.
|
||||
Includes memory pre-fetch for user context injection.
|
||||
@@ -10,7 +10,6 @@ Includes memory pre-fetch for user context injection.
|
||||
import re
|
||||
from typing import Any, Optional
|
||||
|
||||
from src.core.benchmarks import PerformanceBenchmark, get_benchmark_store
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger, log_operation
|
||||
from src.core.memory_service import memory_service
|
||||
@@ -237,7 +236,9 @@ async def _prefetch_memory_context(user_request: str) -> dict[str, Any]:
|
||||
# Location-related queries
|
||||
if any(word in request_lower for word in [
|
||||
"weather", "temperature", "forecast", "nearby", "local",
|
||||
"directions", "distance", "map", "here"
|
||||
"directions", "distance", "map", "here",
|
||||
# Direct location questions
|
||||
"live", "where", "home", "reside", "location", "address",
|
||||
]):
|
||||
profile_keys.append("location")
|
||||
|
||||
@@ -285,8 +286,7 @@ async def analyze_request(
|
||||
This is the main entry point for Steward analysis. It:
|
||||
1. Calls the Steward agent with full conversation history
|
||||
2. Logs the operation with timing
|
||||
3. Records performance benchmarks to Redis
|
||||
4. Returns structured recommendations
|
||||
3. Returns structured recommendations
|
||||
|
||||
Args:
|
||||
user_request: The current user message to analyze
|
||||
@@ -365,23 +365,6 @@ async def analyze_request(
|
||||
reasoning=analysis_text[:200], # First 200 chars
|
||||
)
|
||||
|
||||
# Record performance benchmark
|
||||
if log_ctx.get("duration_seconds"):
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="steward_analysis",
|
||||
duration_seconds=log_ctx["duration_seconds"],
|
||||
success=True,
|
||||
recommendation_count=len(recommendation.recommended_capabilities),
|
||||
confidence=None, # Could add confidence scoring in future
|
||||
conversation_id=conversation_id,
|
||||
metadata={
|
||||
"complexity": recommendation.estimated_complexity,
|
||||
"has_context": recommendation.conversation_context.has_previous_context,
|
||||
"missing_capabilities": recommendation.missing_capabilities is not None,
|
||||
},
|
||||
)
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
return recommendation
|
||||
|
||||
except Exception as e:
|
||||
|
||||
+114
-75
@@ -20,6 +20,11 @@ from src.agents.tatlock_core.tools import (
|
||||
)
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import (
|
||||
start_span, end_span, get_current_span,
|
||||
add_tool_spans_from_messages,
|
||||
SpanType, SpanStatus,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -42,7 +47,16 @@ def generate_id() -> str:
|
||||
# System prompt defining Tatlock's personality
|
||||
TATLOCK_SYSTEM_PROMPT = """You are Tatlock, a helpful personal assistant with the demeanor of a British butler.
|
||||
|
||||
Address users as "sir" and maintain a formal yet personable tone. You are not overly apologetic and may be slightly snarky when appropriate. If an opportunity for a pun presents itself, you cannot resist.
|
||||
## Personality
|
||||
|
||||
Address users as "sir". Be confident, direct, and efficient - you are an unflappable English butler who gets things done. Dry wit and puns are encouraged.
|
||||
|
||||
**CRITICAL - Do NOT:**
|
||||
- Apologize unless you genuinely made an error
|
||||
- Say "Apologies for any confusion" or "Allow me to rectify" when nothing went wrong
|
||||
- Preface successful results with caveats or apologies
|
||||
|
||||
When presenting findings: lead with the answer, be concise, skip the preamble.
|
||||
|
||||
You coordinate with various household staff (expert agents) to provide comprehensive assistance across:
|
||||
- Research and knowledge work
|
||||
@@ -115,6 +129,22 @@ or
|
||||
"""
|
||||
|
||||
|
||||
# Tool-phase prompt for orchestrate_tool_calls(). The butler personality prompt
|
||||
# suppresses tool calling on small local models (gemma4 reasons about the tool,
|
||||
# then answers from memory with wrong arithmetic), so the orchestration phase
|
||||
# uses a terse operator prompt; synthesize_from_results() applies the persona.
|
||||
TATLOCK_ORCHESTRATION_PROMPT = """You are the tool-execution phase of Tatlock, \
|
||||
a butler assistant. Your only job is to gather accurate results by calling the \
|
||||
provided tools.
|
||||
|
||||
- ALWAYS use tools for the task - never answer from memory and never do mental math.
|
||||
- Mathematics: call the calculate tool, even for trivial arithmetic.
|
||||
- Dates and times: call the date/time tools, never guess.
|
||||
- When the instructions say DELEGATE to an agent, call the matching delegate_to_* tool.
|
||||
- After the tool results arrive, reply with a one-line factual summary of the results. \
|
||||
A later step writes the polished reply, so do not add personality."""
|
||||
|
||||
|
||||
class TatlockAgent(AgentInterface):
|
||||
"""
|
||||
Tatlock - The Butler agent using PydanticAI with Ollama.
|
||||
@@ -124,10 +154,7 @@ class TatlockAgent(AgentInterface):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Tatlock configuration (lazy agent creation)."""
|
||||
# Store Ollama configuration
|
||||
self.ollama_host = str(config.OLLAMA_HOST)
|
||||
self.model_name = config.OLLAMA_DEFAULT_MODEL
|
||||
"""Initialize Tatlock (lazy agent creation)."""
|
||||
self._agent = None # Lazy initialization
|
||||
|
||||
def _ensure_agent(self):
|
||||
@@ -135,30 +162,21 @@ class TatlockAgent(AgentInterface):
|
||||
if self._agent is not None:
|
||||
return
|
||||
|
||||
from src.anthropic.model_selector import get_model, get_model_info
|
||||
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"tatlock_agent_initializing",
|
||||
ollama_host=self.ollama_host,
|
||||
model=self.model_name,
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
)
|
||||
|
||||
# Import required classes for Ollama configuration
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
# Get best available model (Claude if available, else Ollama)
|
||||
model = get_model()
|
||||
|
||||
# PydanticAI expects Ollama base URL to end with /v1
|
||||
# Remove trailing slash from ollama_host if present
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
# Create Ollama model with provider
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
|
||||
# Create PydanticAI agent with Ollama model
|
||||
# Create PydanticAI agent
|
||||
self._agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
)
|
||||
|
||||
@@ -433,7 +451,7 @@ class TatlockAgent(AgentInterface):
|
||||
steward_note: Note from Steward (prepended to request, invisible to user)
|
||||
scoped_tools: List of tool definitions from household registry
|
||||
message_history: Conversation history in PydanticAI format
|
||||
tool_tracker: Optional tool call tracker for benchmarking
|
||||
tool_tracker: Optional tool call tracker for analysis
|
||||
|
||||
Returns:
|
||||
str: Tatlock's response text
|
||||
@@ -447,8 +465,7 @@ class TatlockAgent(AgentInterface):
|
||||
... tool_tracker=tracker,
|
||||
... )
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_run_with_scoped_tools",
|
||||
@@ -459,18 +476,12 @@ class TatlockAgent(AgentInterface):
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
# This ensures Tatlock can ONLY use tools recommended by the Steward
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools
|
||||
# Tools from household registry are already PydanticAI Tool objects
|
||||
scoped_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
tools=scoped_tools, # Pass tools directly to Agent constructor
|
||||
)
|
||||
@@ -499,13 +510,13 @@ class TatlockAgent(AgentInterface):
|
||||
)
|
||||
|
||||
# Run with scoped tools and tracker
|
||||
# Force tool_choice: required to make LLM actually call tools
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
# Force tool_choice to make LLM actually call tools
|
||||
from src.anthropic.model_selector import get_tool_choice_settings
|
||||
result = await scoped_agent.run(
|
||||
enriched_message,
|
||||
message_history=pydantic_history if pydantic_history else None,
|
||||
deps=tool_tracker,
|
||||
model_settings=ModelSettings(extra_body={"tool_choice": "required"})
|
||||
model_settings=get_tool_choice_settings(),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -541,8 +552,7 @@ class TatlockAgent(AgentInterface):
|
||||
Yields:
|
||||
Text chunks from the streaming response
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_run_with_scoped_tools_stream",
|
||||
@@ -552,17 +562,11 @@ class TatlockAgent(AgentInterface):
|
||||
)
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools
|
||||
scoped_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
tools=scoped_tools,
|
||||
)
|
||||
@@ -627,7 +631,7 @@ class TatlockAgent(AgentInterface):
|
||||
steward_note: Note from Steward (invisible to user)
|
||||
scoped_tools: List of tool definitions from household registry
|
||||
message_history: Conversation history
|
||||
tool_tracker: Optional tool call tracker for benchmarking
|
||||
tool_tracker: Optional tool call tracker for analysis
|
||||
|
||||
Returns:
|
||||
dict with:
|
||||
@@ -636,9 +640,6 @@ class TatlockAgent(AgentInterface):
|
||||
- tool_outputs: Dict mapping tool names to their outputs
|
||||
- raw_output: The agent's raw text output
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
from pydantic_ai.messages import (
|
||||
ModelRequest,
|
||||
ModelResponse,
|
||||
@@ -647,6 +648,7 @@ class TatlockAgent(AgentInterface):
|
||||
ToolCallPart,
|
||||
ToolReturnPart,
|
||||
)
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_orchestrate_tool_calls",
|
||||
@@ -655,19 +657,23 @@ class TatlockAgent(AgentInterface):
|
||||
history_length=len(message_history),
|
||||
)
|
||||
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
# Start tracing span for orchestration phase
|
||||
orchestrate_span = start_span(
|
||||
"tatlock_orchestrate",
|
||||
SpanType.TATLOCK,
|
||||
metadata={
|
||||
"scoped_tool_count": len(scoped_tools),
|
||||
"tool_names": [getattr(t, '__name__', str(t)) for t in scoped_tools[:5]],
|
||||
},
|
||||
)
|
||||
|
||||
# Create agent with scoped tools
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools, using the tool-phase prompt
|
||||
scoped_agent = Agent(
|
||||
ollama_model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
model,
|
||||
system_prompt=TATLOCK_ORCHESTRATION_PROMPT,
|
||||
tools=scoped_tools,
|
||||
)
|
||||
|
||||
@@ -693,11 +699,12 @@ class TatlockAgent(AgentInterface):
|
||||
)
|
||||
|
||||
# Run with scoped tools and tracker
|
||||
from src.anthropic.model_selector import get_tool_choice_settings
|
||||
result = await scoped_agent.run(
|
||||
enriched_message,
|
||||
message_history=pydantic_history if pydantic_history else None,
|
||||
deps=tool_tracker,
|
||||
model_settings=ModelSettings(extra_body={"tool_choice": "required"})
|
||||
model_settings=get_tool_choice_settings(),
|
||||
)
|
||||
|
||||
# Extract tool calls and results from the agent's messages
|
||||
@@ -731,6 +738,23 @@ class TatlockAgent(AgentInterface):
|
||||
tool_output_count=len(tool_outputs),
|
||||
)
|
||||
|
||||
# Add tool-level spans from result messages
|
||||
if orchestrate_span:
|
||||
add_tool_spans_from_messages(result.new_messages(), orchestrate_span)
|
||||
|
||||
# End orchestration span with results
|
||||
end_span(
|
||||
orchestrate_span,
|
||||
metadata_update={
|
||||
"tools_called": tools_called,
|
||||
"expert_count": len(expert_results),
|
||||
"tool_output_count": len(tool_outputs),
|
||||
},
|
||||
details_update={
|
||||
"steward_note_preview": steward_note[:500] if steward_note else None,
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"tools_called": tools_called,
|
||||
"expert_results": expert_results,
|
||||
@@ -758,9 +782,8 @@ class TatlockAgent(AgentInterface):
|
||||
Returns:
|
||||
str: Butler-toned response synthesized from all results
|
||||
"""
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from pydantic_ai.messages import ModelRequest, ModelResponse, UserPromptPart, TextPart
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
logger.info(
|
||||
"tatlock_synthesize_from_results",
|
||||
@@ -769,6 +792,16 @@ class TatlockAgent(AgentInterface):
|
||||
tool_count=len(orchestration_results.get("tool_outputs", {})),
|
||||
)
|
||||
|
||||
# Start tracing span for synthesis phase
|
||||
synthesize_span = start_span(
|
||||
"tatlock_synthesize",
|
||||
SpanType.TATLOCK,
|
||||
metadata={
|
||||
"expert_count": len(orchestration_results.get("expert_results", {})),
|
||||
"tool_output_count": len(orchestration_results.get("tool_outputs", {})),
|
||||
},
|
||||
)
|
||||
|
||||
# Build synthesis prompt with all available information
|
||||
synthesis_parts = []
|
||||
synthesis_parts.append(f"The user asked: {user_message}")
|
||||
@@ -789,25 +822,19 @@ class TatlockAgent(AgentInterface):
|
||||
synthesis_parts.append("")
|
||||
|
||||
synthesis_parts.append(
|
||||
"Based on this information, provide a response to the user. "
|
||||
"Maintain your butler personality - address them as 'sir', "
|
||||
"use formal but personable language, and be helpful."
|
||||
"Synthesize a response for the user. Be direct and confident. "
|
||||
"Lead with the answer - no apologies, no caveats, no 'mix-ups'. "
|
||||
"Address them as 'sir', be concise, add dry wit if appropriate."
|
||||
)
|
||||
|
||||
synthesis_prompt = "\n".join(synthesis_parts)
|
||||
|
||||
# Create synthesis agent (no tools needed)
|
||||
clean_host = self.ollama_host.rstrip('/')
|
||||
base_url = f"{clean_host}/v1"
|
||||
|
||||
ollama_model = OpenAIChatModel(
|
||||
model_name=self.model_name,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
model = get_model()
|
||||
|
||||
# Synthesis agent uses butler prompt but no tools
|
||||
synthesis_agent = Agent(
|
||||
ollama_model,
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
# No tools for synthesis phase
|
||||
)
|
||||
@@ -841,6 +868,18 @@ class TatlockAgent(AgentInterface):
|
||||
response_preview=result.output[:100],
|
||||
)
|
||||
|
||||
# End synthesis span with result
|
||||
end_span(
|
||||
synthesize_span,
|
||||
metadata_update={
|
||||
"response_length": len(result.output),
|
||||
},
|
||||
details_update={
|
||||
"synthesis_prompt": synthesis_prompt[:1000],
|
||||
"response_preview": result.output[:500],
|
||||
},
|
||||
)
|
||||
|
||||
return result.output
|
||||
|
||||
async def get_capabilities(self) -> dict:
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
Anthropic/Claude integration module.
|
||||
|
||||
Provides model selection with Ollama as primary backend and Claude
|
||||
as the cloud fallback.
|
||||
"""
|
||||
|
||||
from src.anthropic.model_selector import (
|
||||
check_claude_health,
|
||||
check_ollama_health,
|
||||
get_model,
|
||||
get_tool_choice_settings,
|
||||
is_claude_available,
|
||||
is_ollama_available,
|
||||
resolve_backend,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"check_claude_health",
|
||||
"check_ollama_health",
|
||||
"get_model",
|
||||
"get_tool_choice_settings",
|
||||
"is_claude_available",
|
||||
"is_ollama_available",
|
||||
"resolve_backend",
|
||||
]
|
||||
@@ -0,0 +1,293 @@
|
||||
"""
|
||||
Model selector for Ollama/Claude backend switching.
|
||||
|
||||
Provides automatic model selection with Ollama as the primary local backend
|
||||
and Claude as the cloud fallback. Claude is used when PREFER_CLOUD_BACKEND
|
||||
is enabled, or automatically when Ollama is unavailable at startup.
|
||||
|
||||
The Anthropic SDK is imported lazily so a missing or broken `anthropic`
|
||||
package degrades to Ollama-only operation instead of crashing the app.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import httpx
|
||||
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Cached health check results (set once at startup)
|
||||
_claude_available: bool | None = None
|
||||
_ollama_available: bool | None = None
|
||||
|
||||
|
||||
async def check_ollama_health() -> bool:
|
||||
"""
|
||||
Check if the Ollama server is reachable and has the configured model.
|
||||
|
||||
This should be called once at application startup.
|
||||
The result is cached in `_ollama_available`.
|
||||
|
||||
Returns:
|
||||
True if Ollama is reachable and OLLAMA_DEFAULT_MODEL is pulled.
|
||||
"""
|
||||
global _ollama_available
|
||||
|
||||
host = str(config.OLLAMA_HOST).rstrip("/")
|
||||
model = config.OLLAMA_DEFAULT_MODEL
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
response = await client.get(f"{host}/api/tags")
|
||||
response.raise_for_status()
|
||||
names = [m.get("name", "") for m in response.json().get("models", [])]
|
||||
|
||||
if model in names or f"{model}:latest" in names:
|
||||
_ollama_available = True
|
||||
logger.info(
|
||||
"ollama_health_check_passed",
|
||||
host=host,
|
||||
model=model,
|
||||
)
|
||||
return True
|
||||
|
||||
_ollama_available = False
|
||||
logger.warning(
|
||||
"ollama_health_check_failed",
|
||||
reason="model_not_pulled",
|
||||
host=host,
|
||||
model=model,
|
||||
hint=f"run `ollama pull {model}`",
|
||||
)
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
_ollama_available = False
|
||||
logger.warning(
|
||||
"ollama_health_check_failed",
|
||||
reason="server_unreachable",
|
||||
host=host,
|
||||
error=str(e),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
async def check_claude_health() -> bool:
|
||||
"""
|
||||
Check if Claude API is reachable and working.
|
||||
|
||||
This should be called once at application startup.
|
||||
The result is cached in `_claude_available`.
|
||||
|
||||
Returns:
|
||||
True if Claude API is accessible, False otherwise.
|
||||
"""
|
||||
global _claude_available
|
||||
|
||||
# No API key configured - Claude not available
|
||||
if not config.ANTHROPIC_API_KEY:
|
||||
logger.info(
|
||||
"claude_health_check_skipped",
|
||||
reason="no_api_key",
|
||||
)
|
||||
_claude_available = False
|
||||
return False
|
||||
|
||||
try:
|
||||
from anthropic import AsyncAnthropic
|
||||
|
||||
client = AsyncAnthropic(api_key=config.ANTHROPIC_API_KEY)
|
||||
|
||||
# Minimal API call to verify connectivity
|
||||
# Using a tiny max_tokens to minimize cost
|
||||
await client.messages.create(
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
max_tokens=1,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
_claude_available = True
|
||||
logger.info(
|
||||
"claude_health_check_passed",
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
_claude_available = False
|
||||
logger.warning(
|
||||
"claude_health_check_failed",
|
||||
error=str(e),
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def is_claude_available() -> bool:
|
||||
"""
|
||||
Check if Claude is available (from cached health check result).
|
||||
|
||||
Returns:
|
||||
True if Claude API was reachable at startup, False otherwise.
|
||||
|
||||
Note:
|
||||
Returns False if health check hasn't been run yet.
|
||||
Call `check_claude_health()` at startup first.
|
||||
"""
|
||||
return _claude_available is True
|
||||
|
||||
|
||||
def is_ollama_available() -> bool:
|
||||
"""
|
||||
Check if Ollama is available (from cached health check result).
|
||||
|
||||
Returns:
|
||||
False only if the startup health check confirmed Ollama is down.
|
||||
Unknown (check not run yet) counts as available so that contexts
|
||||
without lifespan events keep the local-first behavior.
|
||||
"""
|
||||
return _ollama_available is not False
|
||||
|
||||
|
||||
def resolve_backend(prefer_cloud: bool | None = None) -> str:
|
||||
"""
|
||||
Resolve which backend should serve requests.
|
||||
|
||||
Ollama is the primary backend. Claude is used when explicitly
|
||||
preferred via PREFER_CLOUD_BACKEND, or as automatic fallback
|
||||
when the startup health check found Ollama down.
|
||||
|
||||
Args:
|
||||
prefer_cloud: Override config.PREFER_CLOUD_BACKEND for this call.
|
||||
|
||||
Returns:
|
||||
"claude" or "ollama".
|
||||
"""
|
||||
use_cloud = prefer_cloud if prefer_cloud is not None else config.PREFER_CLOUD_BACKEND
|
||||
|
||||
if use_cloud and is_claude_available():
|
||||
return "claude"
|
||||
|
||||
if not is_ollama_available() and is_claude_available():
|
||||
logger.warning(
|
||||
"backend_fallback_to_claude",
|
||||
reason="ollama_unavailable",
|
||||
)
|
||||
return "claude"
|
||||
|
||||
return "ollama"
|
||||
|
||||
|
||||
def get_model(prefer_cloud: bool | None = None) -> AnthropicModel | OpenAIChatModel:
|
||||
"""
|
||||
Get the best available model.
|
||||
|
||||
Returns Ollama unless Claude is preferred (or Ollama is down).
|
||||
|
||||
Args:
|
||||
prefer_cloud: Override config.PREFER_CLOUD_BACKEND for this call.
|
||||
If None, uses the config value.
|
||||
|
||||
Returns:
|
||||
PydanticAI model instance (OpenAIChatModel or AnthropicModel).
|
||||
|
||||
Example:
|
||||
>>> model = get_model()
|
||||
>>> agent = Agent(model, system_prompt="...")
|
||||
"""
|
||||
if resolve_backend(prefer_cloud) == "claude":
|
||||
try:
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
from pydantic_ai.providers.anthropic import AnthropicProvider
|
||||
|
||||
logger.debug(
|
||||
"model_selected",
|
||||
backend="claude",
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return AnthropicModel(
|
||||
model_name=config.ANTHROPIC_MODEL,
|
||||
provider=AnthropicProvider(api_key=config.ANTHROPIC_API_KEY),
|
||||
)
|
||||
except ImportError as e:
|
||||
logger.error(
|
||||
"claude_backend_import_failed",
|
||||
error=str(e),
|
||||
hint="anthropic package missing or incompatible; using Ollama",
|
||||
)
|
||||
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
logger.debug(
|
||||
"model_selected",
|
||||
backend="ollama",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
)
|
||||
return OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider(),
|
||||
)
|
||||
|
||||
|
||||
def get_tool_choice_settings() -> ModelSettings:
|
||||
"""
|
||||
Get model_settings for forcing tool calls on the first request.
|
||||
|
||||
For Claude: PydanticAI handles tool_choice natively, so no extra_body needed.
|
||||
For Ollama: Pass tool_choice="required" via extra_body to force tool calling.
|
||||
"""
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
if resolve_backend() == "claude":
|
||||
# PydanticAI's Anthropic model handles tool_choice internally
|
||||
return ModelSettings()
|
||||
else:
|
||||
# Ollama needs explicit tool_choice via extra_body
|
||||
return ModelSettings(extra_body={"tool_choice": "required"})
|
||||
|
||||
|
||||
def get_sampling_settings(temperature: float) -> ModelSettings:
|
||||
"""
|
||||
Get model_settings with a sampling temperature where the backend allows it.
|
||||
|
||||
Ollama accepts a temperature; Claude Sonnet 5+ rejects sampling
|
||||
parameters, so the Claude backend gets empty settings.
|
||||
"""
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
if resolve_backend() == "claude":
|
||||
return ModelSettings()
|
||||
return ModelSettings(temperature=temperature)
|
||||
|
||||
|
||||
def get_model_info() -> dict:
|
||||
"""
|
||||
Get information about the current model configuration.
|
||||
|
||||
Useful for health checks and debugging.
|
||||
|
||||
Returns:
|
||||
Dict with backend, model name, and availability info.
|
||||
"""
|
||||
backend = resolve_backend()
|
||||
|
||||
return {
|
||||
"backend": backend,
|
||||
"model": config.ANTHROPIC_MODEL if backend == "claude" else config.OLLAMA_DEFAULT_MODEL,
|
||||
"claude_available": is_claude_available(),
|
||||
"claude_configured": bool(config.ANTHROPIC_API_KEY),
|
||||
"ollama_available": is_ollama_available(),
|
||||
"ollama_model": config.OLLAMA_DEFAULT_MODEL,
|
||||
"prefer_cloud": config.PREFER_CLOUD_BACKEND,
|
||||
}
|
||||
+22
-18
@@ -7,7 +7,7 @@ import logging
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from fastapi import APIRouter
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from src.chat import service
|
||||
from src.chat.schemas import (
|
||||
@@ -22,47 +22,51 @@ router = APIRouter(prefix="/chat", tags=["chat"])
|
||||
|
||||
async def _stream_response(
|
||||
request: ChatCompletionRequest,
|
||||
) -> AsyncGenerator[dict, None]:
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Generate SSE stream for chat completion.
|
||||
|
||||
EventSourceResponse adds "data: " prefix automatically.
|
||||
We just yield the dict/string content.
|
||||
Yields raw SSE-formatted strings matching OpenAI's format exactly:
|
||||
data: {json}\n\n
|
||||
"""
|
||||
try:
|
||||
async for chunk in service.create_chat_completion_stream(request):
|
||||
# Yield dict - EventSourceResponse will format as SSE
|
||||
yield {"data": chunk.model_dump_json()}
|
||||
yield f"data: {chunk.model_dump_json(exclude_unset=True)}\n\n"
|
||||
|
||||
# Send [DONE] message
|
||||
yield {"data": "[DONE]"}
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in streaming response: {e}")
|
||||
error_data = {"error": {"message": str(e), "type": "internal_error"}}
|
||||
yield {"data": json.dumps(error_data)}
|
||||
error_data = json.dumps({"error": {"message": str(e), "type": "internal_error"}})
|
||||
yield f"data: {error_data}\n\n"
|
||||
|
||||
|
||||
@router.post("/completions", response_model=ChatCompletionResponse)
|
||||
async def create_chat_completion(
|
||||
request: ChatCompletionRequest,
|
||||
) -> ChatCompletionResponse | EventSourceResponse:
|
||||
) -> ChatCompletionResponse | StreamingResponse:
|
||||
"""
|
||||
Create chat completion (OpenAI-compatible).
|
||||
|
||||
|
||||
Supports both regular and streaming responses.
|
||||
Currently returns mock lorem ipsum responses.
|
||||
|
||||
|
||||
Args:
|
||||
request: Chat completion request
|
||||
|
||||
|
||||
Returns:
|
||||
Chat completion response or SSE stream
|
||||
"""
|
||||
logger.info(f"Chat completion request for model: {request.model}")
|
||||
|
||||
|
||||
if request.stream:
|
||||
logger.info("Streaming response requested")
|
||||
return EventSourceResponse(_stream_response(request))
|
||||
|
||||
return StreamingResponse(
|
||||
_stream_response(request),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-store",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
return await service.create_chat_completion(request)
|
||||
|
||||
@@ -1,345 +0,0 @@
|
||||
"""
|
||||
Performance benchmark storage using Redis.
|
||||
|
||||
Tracks operation timing, tool usage, and recommendation accuracy across sessions.
|
||||
Provides time-series data for performance analysis and optimization.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Literal, Optional
|
||||
|
||||
import redis.asyncio as redis
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .config import config
|
||||
from .logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class PerformanceBenchmark(BaseModel):
|
||||
"""
|
||||
Performance benchmark record.
|
||||
|
||||
Stores timing and metadata for operations like Steward analysis,
|
||||
tool calls, and agent execution.
|
||||
"""
|
||||
timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
operation: str # "steward_analysis", "tool_call", "tatlock_execution"
|
||||
duration_seconds: float
|
||||
success: bool
|
||||
|
||||
# Steward-specific fields
|
||||
recommendation_count: Optional[int] = None
|
||||
confidence: Optional[float] = None
|
||||
|
||||
# Tool-specific fields
|
||||
tool_name: Optional[str] = None
|
||||
was_recommended: Optional[bool] = None
|
||||
was_actually_used: Optional[bool] = None
|
||||
|
||||
# Context
|
||||
conversation_id: Optional[str] = None
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
def to_redis_dict(self) -> dict[str, Any]:
|
||||
"""Convert to dict suitable for Redis storage."""
|
||||
data = self.model_dump()
|
||||
data["timestamp"] = self.timestamp.isoformat()
|
||||
data["metadata"] = json.dumps(self.metadata)
|
||||
# Convert booleans to strings (Redis doesn't accept bool type)
|
||||
for key, value in data.items():
|
||||
if isinstance(value, bool):
|
||||
data[key] = str(value)
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
def from_redis_dict(cls, data: dict[str, Any]) -> "PerformanceBenchmark":
|
||||
"""Reconstruct from Redis dict."""
|
||||
data["timestamp"] = datetime.fromisoformat(data["timestamp"])
|
||||
data["metadata"] = json.loads(data.get("metadata", "{}"))
|
||||
# Convert string booleans back to bool
|
||||
for key in ["success", "was_recommended", "was_actually_used"]:
|
||||
if key in data and isinstance(data[key], str):
|
||||
data[key] = data[key] == "True"
|
||||
return cls(**data)
|
||||
|
||||
|
||||
class BenchmarkStore:
|
||||
"""
|
||||
Redis-backed benchmark storage with automatic expiry.
|
||||
|
||||
Stores performance metrics in time-series format with 30-day retention.
|
||||
Provides querying capabilities for analysis and reporting.
|
||||
"""
|
||||
|
||||
def __init__(self, redis_client: Optional[redis.Redis] = None):
|
||||
"""
|
||||
Initialize benchmark store.
|
||||
|
||||
Args:
|
||||
redis_client: Optional Redis client. If None, creates from config.
|
||||
"""
|
||||
self._client = redis_client
|
||||
self._ttl_days = 30 # 30-day retention
|
||||
|
||||
async def _get_client(self) -> redis.Redis:
|
||||
"""Get or create Redis client."""
|
||||
if self._client is None:
|
||||
self._client = redis.from_url(
|
||||
config.redis_url,
|
||||
encoding="utf-8",
|
||||
decode_responses=True,
|
||||
socket_timeout=config.REDIS_TIMEOUT,
|
||||
socket_connect_timeout=config.REDIS_TIMEOUT,
|
||||
)
|
||||
return self._client
|
||||
|
||||
async def record(self, benchmark: PerformanceBenchmark) -> None:
|
||||
"""
|
||||
Record a performance benchmark.
|
||||
|
||||
Args:
|
||||
benchmark: Performance benchmark to record
|
||||
|
||||
Example:
|
||||
>>> await store.record(PerformanceBenchmark(
|
||||
... operation="steward_analysis",
|
||||
... duration_seconds=1.23,
|
||||
... success=True,
|
||||
... recommendation_count=3,
|
||||
... ))
|
||||
"""
|
||||
if not config.ENABLE_BENCHMARKS:
|
||||
return
|
||||
|
||||
try:
|
||||
client = await self._get_client()
|
||||
|
||||
# Generate key: benchmark:{operation}:{timestamp_ms}
|
||||
timestamp_ms = int(benchmark.timestamp.timestamp() * 1000)
|
||||
key = f"benchmark:{benchmark.operation}:{timestamp_ms}"
|
||||
|
||||
# Store as hash
|
||||
await client.hset(key, mapping=benchmark.to_redis_dict())
|
||||
|
||||
# Set expiry
|
||||
await client.expire(key, self._ttl_days * 24 * 60 * 60)
|
||||
|
||||
# Add to sorted set for time-based queries
|
||||
index_key = f"benchmark_index:{benchmark.operation}"
|
||||
await client.zadd(index_key, {key: timestamp_ms})
|
||||
await client.expire(index_key, self._ttl_days * 24 * 60 * 60)
|
||||
|
||||
logger.debug(
|
||||
"benchmark_recorded",
|
||||
operation=benchmark.operation,
|
||||
duration=benchmark.duration_seconds,
|
||||
success=benchmark.success,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"benchmark_recording_failed",
|
||||
error=str(e),
|
||||
operation=benchmark.operation,
|
||||
)
|
||||
# Don't fail the request if benchmarking fails
|
||||
|
||||
async def query(
|
||||
self,
|
||||
operation: str,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
limit: int = 100,
|
||||
) -> list[PerformanceBenchmark]:
|
||||
"""
|
||||
Query benchmarks by operation and time range.
|
||||
|
||||
Args:
|
||||
operation: Operation name to filter by
|
||||
start_time: Start of time range (inclusive)
|
||||
end_time: End of time range (inclusive)
|
||||
limit: Maximum number of results
|
||||
|
||||
Returns:
|
||||
List of benchmarks matching the query
|
||||
|
||||
Example:
|
||||
>>> from datetime import timedelta
|
||||
>>> now = datetime.now(timezone.utc)
|
||||
>>> yesterday = now - timedelta(days=1)
|
||||
>>> benchmarks = await store.query(
|
||||
... "steward_analysis",
|
||||
... start_time=yesterday,
|
||||
... limit=50
|
||||
... )
|
||||
"""
|
||||
if not config.ENABLE_BENCHMARKS:
|
||||
return []
|
||||
|
||||
try:
|
||||
client = await self._get_client()
|
||||
index_key = f"benchmark_index:{operation}"
|
||||
|
||||
# Convert time range to timestamps
|
||||
min_score = (
|
||||
int(start_time.timestamp() * 1000)
|
||||
if start_time
|
||||
else "-inf"
|
||||
)
|
||||
max_score = (
|
||||
int(end_time.timestamp() * 1000)
|
||||
if end_time
|
||||
else "+inf"
|
||||
)
|
||||
|
||||
# Query sorted set
|
||||
keys = await client.zrevrangebyscore(
|
||||
index_key,
|
||||
max_score,
|
||||
min_score,
|
||||
start=0,
|
||||
num=limit,
|
||||
)
|
||||
|
||||
# Fetch benchmark data
|
||||
benchmarks = []
|
||||
for key in keys:
|
||||
data = await client.hgetall(key)
|
||||
if data:
|
||||
benchmarks.append(PerformanceBenchmark.from_redis_dict(data))
|
||||
|
||||
return benchmarks
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"benchmark_query_failed",
|
||||
error=str(e),
|
||||
operation=operation,
|
||||
)
|
||||
return []
|
||||
|
||||
async def get_statistics(
|
||||
self,
|
||||
operation: str,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Get aggregate statistics for an operation.
|
||||
|
||||
Args:
|
||||
operation: Operation name
|
||||
start_time: Start of time range
|
||||
end_time: End of time range
|
||||
|
||||
Returns:
|
||||
Dictionary with statistics (count, avg_duration, success_rate, etc.)
|
||||
|
||||
Example:
|
||||
>>> stats = await store.get_statistics("steward_analysis")
|
||||
>>> print(f"Average duration: {stats['avg_duration']}s")
|
||||
>>> print(f"Success rate: {stats['success_rate']}%")
|
||||
"""
|
||||
benchmarks = await self.query(operation, start_time, end_time, limit=1000)
|
||||
|
||||
if not benchmarks:
|
||||
return {
|
||||
"count": 0,
|
||||
"avg_duration": 0.0,
|
||||
"min_duration": 0.0,
|
||||
"max_duration": 0.0,
|
||||
"success_rate": 0.0,
|
||||
}
|
||||
|
||||
durations = [b.duration_seconds for b in benchmarks]
|
||||
successes = sum(1 for b in benchmarks if b.success)
|
||||
|
||||
return {
|
||||
"count": len(benchmarks),
|
||||
"avg_duration": sum(durations) / len(durations),
|
||||
"min_duration": min(durations),
|
||||
"max_duration": max(durations),
|
||||
"success_rate": (successes / len(benchmarks)) * 100,
|
||||
"total_successes": successes,
|
||||
"total_failures": len(benchmarks) - successes,
|
||||
}
|
||||
|
||||
async def get_tool_accuracy(
|
||||
self,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Analyze tool recommendation accuracy.
|
||||
|
||||
Compares recommended tools vs actually used tools to measure
|
||||
Steward's recommendation precision.
|
||||
|
||||
Args:
|
||||
start_time: Start of time range
|
||||
end_time: End of time range
|
||||
|
||||
Returns:
|
||||
Dictionary with accuracy metrics
|
||||
|
||||
Example:
|
||||
>>> accuracy = await store.get_tool_accuracy()
|
||||
>>> print(f"Precision: {accuracy['precision']}%")
|
||||
"""
|
||||
tool_calls = await self.query("tool_call", start_time, end_time, limit=1000)
|
||||
|
||||
if not tool_calls:
|
||||
return {
|
||||
"total_calls": 0,
|
||||
"recommended_and_used": 0,
|
||||
"recommended_not_used": 0,
|
||||
"not_recommended_but_used": 0,
|
||||
"precision": 0.0,
|
||||
}
|
||||
|
||||
recommended_and_used = sum(
|
||||
1 for b in tool_calls
|
||||
if b.was_recommended and b.was_actually_used
|
||||
)
|
||||
not_recommended_but_used = sum(
|
||||
1 for b in tool_calls
|
||||
if not b.was_recommended and b.was_actually_used
|
||||
)
|
||||
|
||||
total_used = sum(1 for b in tool_calls if b.was_actually_used)
|
||||
precision = (
|
||||
(recommended_and_used / total_used * 100) if total_used > 0 else 0.0
|
||||
)
|
||||
|
||||
return {
|
||||
"total_calls": len(tool_calls),
|
||||
"total_used": total_used,
|
||||
"recommended_and_used": recommended_and_used,
|
||||
"not_recommended_but_used": not_recommended_but_used,
|
||||
"precision": precision,
|
||||
}
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close Redis connection."""
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
|
||||
# Global benchmark store instance
|
||||
_benchmark_store: Optional[BenchmarkStore] = None
|
||||
|
||||
|
||||
def get_benchmark_store() -> BenchmarkStore:
|
||||
"""
|
||||
Get global benchmark store instance.
|
||||
|
||||
Returns:
|
||||
BenchmarkStore instance
|
||||
"""
|
||||
global _benchmark_store
|
||||
if _benchmark_store is None:
|
||||
_benchmark_store = BenchmarkStore()
|
||||
return _benchmark_store
|
||||
+21
-13
@@ -64,19 +64,37 @@ class Config(BaseSettings):
|
||||
API_PORT: int = Field(default=8000, description="API port")
|
||||
API_PREFIX: str = Field(default="/v1", description="API route prefix")
|
||||
|
||||
# Ollama Configuration
|
||||
# Anthropic Configuration (Claude - cloud fallback)
|
||||
ANTHROPIC_API_KEY: str | None = Field(
|
||||
default=None,
|
||||
description="Anthropic API key for the Claude fallback backend"
|
||||
)
|
||||
ANTHROPIC_MODEL: str = Field(
|
||||
default="claude-sonnet-5",
|
||||
description="Claude model for the fallback backend"
|
||||
)
|
||||
PREFER_CLOUD_BACKEND: bool = Field(
|
||||
default=False,
|
||||
description="Prefer Claude over Ollama (default: local-first)"
|
||||
)
|
||||
|
||||
# Ollama Configuration (local - primary backend)
|
||||
OLLAMA_HOST: HttpUrl = Field(
|
||||
default="http://localhost:11434",
|
||||
description="Ollama server URL"
|
||||
)
|
||||
OLLAMA_DEFAULT_MODEL: str = Field(
|
||||
default="mistral-nemo:latest",
|
||||
default="gemma4:e2b",
|
||||
description="Default Ollama model"
|
||||
)
|
||||
OLLAMA_TIMEOUT: int = Field(
|
||||
default=120,
|
||||
description="Ollama request timeout in seconds"
|
||||
)
|
||||
STEWARD_TIMEOUT: int = Field(
|
||||
default=60,
|
||||
description="Steward analysis timeout in seconds (gemma4 needs ~35s warm)"
|
||||
)
|
||||
STREAM_TIMEOUT: int = Field(
|
||||
default=20,
|
||||
description="Timeout for each streaming turn in seconds"
|
||||
@@ -101,10 +119,6 @@ class Config(BaseSettings):
|
||||
default=6379,
|
||||
description="Redis server port"
|
||||
)
|
||||
REDIS_BENCHMARK_DB: int = Field(
|
||||
default=6,
|
||||
description="Redis database number for benchmarks"
|
||||
)
|
||||
REDIS_TIMEOUT: int = Field(
|
||||
default=5,
|
||||
description="Redis connection timeout in seconds"
|
||||
@@ -158,7 +172,7 @@ class Config(BaseSettings):
|
||||
description="Ollama model for embeddings"
|
||||
)
|
||||
|
||||
# Redis Memory Database (separate from benchmarks)
|
||||
# Redis Memory Database
|
||||
REDIS_MEMORY_DB: int = Field(
|
||||
default=1,
|
||||
description="Redis database number for memory cache"
|
||||
@@ -173,7 +187,6 @@ class Config(BaseSettings):
|
||||
default=None,
|
||||
description="Logging level (auto-set based on environment if not specified)"
|
||||
)
|
||||
ENABLE_BENCHMARKS: bool = Field(default=True, description="Enable performance benchmarking")
|
||||
|
||||
# User Configuration
|
||||
DEFAULT_USER: str | None = Field(
|
||||
@@ -190,11 +203,6 @@ class Config(BaseSettings):
|
||||
CORS_ALLOW_METHODS: list[str] = ["*"]
|
||||
CORS_ALLOW_HEADERS: list[str] = ["*"]
|
||||
|
||||
@property
|
||||
def redis_url(self) -> str:
|
||||
"""Construct Redis connection URL for benchmarks."""
|
||||
return f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/{self.REDIS_BENCHMARK_DB}"
|
||||
|
||||
@property
|
||||
def redis_memory_url(self) -> str:
|
||||
"""Construct Redis connection URL for memory cache."""
|
||||
|
||||
@@ -6,7 +6,7 @@ Provides short-term memory storage with TTL:
|
||||
- Recent entities mentioned in conversation
|
||||
- User-scoped with conversation isolation
|
||||
|
||||
Uses Redis DB 2 (separate from benchmarks in DB 1).
|
||||
Uses Redis DB 1.
|
||||
"""
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
@@ -11,6 +11,7 @@ from src.agents.steward import analyze_request, format_steward_note
|
||||
from src.agents.steward.schemas import StewardRecommendation
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import trace_span, SpanType
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -93,12 +94,32 @@ async def preprocess_request(
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Call Steward with full conversation history
|
||||
recommendation = await analyze_request(
|
||||
enriched_request,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
# Call Steward with full conversation history (traced)
|
||||
async with trace_span(
|
||||
"steward_analysis",
|
||||
SpanType.STEWARD,
|
||||
metadata={
|
||||
"request_preview": user_request[:100],
|
||||
"history_length": len(conversation_history),
|
||||
},
|
||||
) as span:
|
||||
recommendation = await analyze_request(
|
||||
enriched_request,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Update span with results
|
||||
if span:
|
||||
span.metadata.update({
|
||||
"recommended_capabilities": recommendation.recommended_capabilities,
|
||||
"complexity": recommendation.estimated_complexity,
|
||||
"has_memory_context": bool(recommendation.memory_context),
|
||||
"has_conversation_context": recommendation.conversation_context.has_previous_context,
|
||||
})
|
||||
span.details["reasoning"] = recommendation.reasoning
|
||||
if recommendation.enriched_query:
|
||||
span.details["enriched_query"] = recommendation.enriched_query
|
||||
|
||||
# Format note for Tatlock (includes conversation context)
|
||||
steward_note = await format_steward_note(recommendation)
|
||||
|
||||
+21
-4
@@ -9,6 +9,11 @@ from src.agents.biographer import register_biographer
|
||||
from src.agents.housekeeper import register_housekeeper
|
||||
from src.agents.librarian import register_librarian
|
||||
from src.agents.tatlock_core import TATLOCK_CORE_CAPABILITY, tatlock_core_tools
|
||||
from src.anthropic.model_selector import (
|
||||
check_claude_health,
|
||||
check_ollama_health,
|
||||
get_model_info,
|
||||
)
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
@@ -81,19 +86,31 @@ def register_household_members():
|
||||
)
|
||||
|
||||
|
||||
def initialize_application():
|
||||
async def initialize_application():
|
||||
"""
|
||||
Initialize the application.
|
||||
|
||||
Performs all startup tasks:
|
||||
1. Register household members
|
||||
2. (Future) Initialize connections
|
||||
3. (Future) Load configuration
|
||||
1. Check Ollama (primary) and Claude (fallback) health for backend selection
|
||||
2. Register household members
|
||||
3. (Future) Initialize connections
|
||||
|
||||
This should be called once during application startup.
|
||||
"""
|
||||
logger.info("application_initialization_starting")
|
||||
|
||||
# Check backend health: Ollama is primary, Claude is the fallback
|
||||
await check_ollama_health()
|
||||
await check_claude_health()
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"model_backend_configured",
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
ollama_available=model_info["ollama_available"],
|
||||
claude_available=model_info["claude_available"],
|
||||
)
|
||||
|
||||
# Register household members
|
||||
register_household_members()
|
||||
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
"""
|
||||
Tool call tracking and benchmarking.
|
||||
Tool call tracking.
|
||||
|
||||
Tracks which tools are recommended by the Steward versus which tools
|
||||
are actually used by Tatlock, recording benchmarks for analysis.
|
||||
are actually used by Tatlock for debugging and analysis.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from src.core.benchmarks import PerformanceBenchmark, get_benchmark_store
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -15,7 +13,7 @@ logger = get_logger(__name__)
|
||||
|
||||
class ToolCallTracker:
|
||||
"""
|
||||
Tracks tool calls for benchmarking and accuracy analysis.
|
||||
Tracks tool calls for accuracy analysis.
|
||||
|
||||
Compares Steward's recommendations with Tatlock's actual tool usage
|
||||
to measure recommendation accuracy.
|
||||
@@ -53,6 +51,10 @@ class ToolCallTracker:
|
||||
return tool_name.replace("delegate_to_", "")
|
||||
return tool_name
|
||||
|
||||
def log_call(self, message: str):
|
||||
"""Log a tool call message (for UI display)."""
|
||||
logger.debug("tool_call_message", message=message)
|
||||
|
||||
async def track_call(self, tool_name: str, duration: float):
|
||||
"""
|
||||
Record a tool call with timing.
|
||||
@@ -78,23 +80,6 @@ class ToolCallTracker:
|
||||
recommended=list(self.recommended_capabilities),
|
||||
)
|
||||
|
||||
# Record benchmark to Redis
|
||||
benchmark = PerformanceBenchmark(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
operation="tool_call",
|
||||
duration_seconds=duration,
|
||||
success=True, # If we got here, the call succeeded
|
||||
tool_name=tool_name,
|
||||
was_recommended=was_recommended,
|
||||
was_actually_used=True,
|
||||
conversation_id=self.conversation_id,
|
||||
metadata={
|
||||
"recommended_capabilities": list(self.recommended_capabilities),
|
||||
},
|
||||
)
|
||||
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
logger.debug(
|
||||
"tool_call_tracked",
|
||||
tool_name=tool_name,
|
||||
@@ -124,24 +109,6 @@ class ToolCallTracker:
|
||||
conversation_id=self.conversation_id,
|
||||
)
|
||||
|
||||
# Record benchmarks for unused recommendations
|
||||
for tool_name in unused_tools:
|
||||
benchmark = PerformanceBenchmark(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
operation="tool_call",
|
||||
duration_seconds=0.0, # Not used
|
||||
success=True,
|
||||
tool_name=tool_name,
|
||||
was_recommended=True,
|
||||
was_actually_used=False,
|
||||
conversation_id=self.conversation_id,
|
||||
metadata={
|
||||
"recommended_capabilities": list(self.recommended_capabilities),
|
||||
"reason": "recommended_but_unused",
|
||||
},
|
||||
)
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
# Log summary
|
||||
total_calls = sum(len(durations) for durations in self.actual_calls.values())
|
||||
logger.info(
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
"""
|
||||
Lightweight request tracing for local development.
|
||||
|
||||
Captures the full request flow through Tatlock's multi-agent architecture
|
||||
as structured JSON traces for debugging and optimization.
|
||||
|
||||
Enable via DEBUG=true environment variable.
|
||||
|
||||
Traces are written to logs/traces/{trace_id}.json
|
||||
View with logs/traces/viewer.html
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import json
|
||||
import secrets
|
||||
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class SpanType(str, Enum):
|
||||
"""Types of traced operations."""
|
||||
ROUTER = "router"
|
||||
STEWARD = "steward"
|
||||
TATLOCK = "tatlock"
|
||||
EXPERT = "expert"
|
||||
TOOL = "tool"
|
||||
|
||||
|
||||
class SpanStatus(str, Enum):
|
||||
"""Span completion status."""
|
||||
OK = "ok"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Span:
|
||||
"""A single traced operation."""
|
||||
span_id: str
|
||||
name: str
|
||||
type: SpanType
|
||||
start_time: datetime
|
||||
parent_id: str | None = None
|
||||
end_time: datetime | None = None
|
||||
status: SpanStatus = SpanStatus.OK
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
children: list[str] = field(default_factory=list)
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def duration_ms(self) -> float | None:
|
||||
"""Calculate duration in milliseconds."""
|
||||
if self.end_time and self.start_time:
|
||||
return (self.end_time - self.start_time).total_seconds() * 1000
|
||||
return None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert span to dictionary for JSON serialization."""
|
||||
result = {
|
||||
"span_id": self.span_id,
|
||||
"parent_id": self.parent_id,
|
||||
"name": self.name,
|
||||
"type": self.type.value,
|
||||
"start_time": self.start_time.isoformat(),
|
||||
"end_time": self.end_time.isoformat() if self.end_time else None,
|
||||
"duration_ms": round(self.duration_ms, 2) if self.duration_ms else None,
|
||||
"status": self.status.value,
|
||||
"metadata": self.metadata if self.metadata else None,
|
||||
}
|
||||
# Only include non-empty optional fields
|
||||
if self.details:
|
||||
result["details"] = self.details
|
||||
if self.children:
|
||||
result["children"] = self.children
|
||||
if self.error:
|
||||
result["error"] = self.error
|
||||
return {k: v for k, v in result.items() if v is not None}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Trace:
|
||||
"""Complete trace of a request."""
|
||||
trace_id: str
|
||||
conversation_id: str | None
|
||||
user: str
|
||||
timestamp: datetime
|
||||
request: dict[str, Any]
|
||||
spans: list[Span] = field(default_factory=list)
|
||||
response: dict[str, Any] | None = None
|
||||
status: str = "in_progress"
|
||||
|
||||
@property
|
||||
def total_duration_ms(self) -> float | None:
|
||||
"""Calculate total trace duration from span timings."""
|
||||
if not self.spans:
|
||||
return None
|
||||
start = min(s.start_time for s in self.spans)
|
||||
ends = [s.end_time for s in self.spans if s.end_time]
|
||||
if not ends:
|
||||
return None
|
||||
end = max(ends)
|
||||
return (end - start).total_seconds() * 1000
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert trace to dictionary for JSON serialization."""
|
||||
return {
|
||||
"trace_id": self.trace_id,
|
||||
"conversation_id": self.conversation_id,
|
||||
"user": self.user,
|
||||
"timestamp": self.timestamp.isoformat(),
|
||||
"total_duration_ms": round(self.total_duration_ms, 2) if self.total_duration_ms else None,
|
||||
"status": self.status,
|
||||
"request": self.request,
|
||||
"response": self.response,
|
||||
"spans": [s.to_dict() for s in self.spans],
|
||||
}
|
||||
|
||||
|
||||
# ContextVar for async-safe trace propagation
|
||||
_current_trace: ContextVar[Trace | None] = ContextVar("current_trace", default=None)
|
||||
_current_span: ContextVar[Span | None] = ContextVar("current_span", default=None)
|
||||
|
||||
|
||||
def tracing_enabled() -> bool:
|
||||
"""Check if tracing is enabled (requires DEBUG=true)."""
|
||||
from src.core.config import config
|
||||
return config.DEBUG
|
||||
|
||||
|
||||
def _generate_id(prefix: str = "") -> str:
|
||||
"""Generate unique ID with optional prefix."""
|
||||
return f"{prefix}{secrets.token_hex(8)}"
|
||||
|
||||
|
||||
def start_trace(
|
||||
conversation_id: str | None,
|
||||
user: str,
|
||||
request: dict[str, Any],
|
||||
) -> Trace | None:
|
||||
"""
|
||||
Start a new trace for a request.
|
||||
|
||||
Args:
|
||||
conversation_id: Conversation identifier
|
||||
user: User identifier
|
||||
request: Request data (should include preview and full)
|
||||
|
||||
Returns:
|
||||
Trace object if tracing enabled, None otherwise
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
return None
|
||||
|
||||
trace = Trace(
|
||||
trace_id=_generate_id("trace_"),
|
||||
conversation_id=conversation_id,
|
||||
user=user,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
request=request,
|
||||
)
|
||||
_current_trace.set(trace)
|
||||
|
||||
logger.debug("trace_started", trace_id=trace.trace_id, user=user)
|
||||
return trace
|
||||
|
||||
|
||||
def get_current_trace() -> Trace | None:
|
||||
"""Get the current trace from context."""
|
||||
return _current_trace.get()
|
||||
|
||||
|
||||
def get_current_span() -> Span | None:
|
||||
"""Get the current span from context."""
|
||||
return _current_span.get()
|
||||
|
||||
|
||||
def start_span(
|
||||
name: str,
|
||||
span_type: SpanType,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> Span | None:
|
||||
"""
|
||||
Start a new span within the current trace.
|
||||
|
||||
Args:
|
||||
name: Span name (e.g., "steward_analysis")
|
||||
span_type: Type of operation
|
||||
metadata: Quick-access metadata (shown in timeline)
|
||||
details: Expandable details (prompts, full responses)
|
||||
|
||||
Returns:
|
||||
Span object if tracing enabled, None otherwise
|
||||
"""
|
||||
trace = get_current_trace()
|
||||
if not trace:
|
||||
return None
|
||||
|
||||
parent = get_current_span()
|
||||
span = Span(
|
||||
span_id=_generate_id("span_"),
|
||||
name=name,
|
||||
type=span_type,
|
||||
start_time=datetime.now(timezone.utc),
|
||||
parent_id=parent.span_id if parent else None,
|
||||
metadata=metadata or {},
|
||||
details=details or {},
|
||||
)
|
||||
|
||||
# Add to parent's children list
|
||||
if parent:
|
||||
parent.children.append(span.span_id)
|
||||
|
||||
trace.spans.append(span)
|
||||
_current_span.set(span)
|
||||
|
||||
logger.debug(
|
||||
"span_started",
|
||||
span_id=span.span_id,
|
||||
name=name,
|
||||
type=span_type.value,
|
||||
parent_id=span.parent_id,
|
||||
)
|
||||
return span
|
||||
|
||||
|
||||
def end_span(
|
||||
span: Span | None = None,
|
||||
status: SpanStatus = SpanStatus.OK,
|
||||
metadata_update: dict[str, Any] | None = None,
|
||||
details_update: dict[str, Any] | None = None,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
End a span and restore parent as current.
|
||||
|
||||
Args:
|
||||
span: Span to end (defaults to current span)
|
||||
status: Completion status
|
||||
metadata_update: Additional metadata to merge
|
||||
details_update: Additional details to merge
|
||||
error: Error message if failed
|
||||
"""
|
||||
if span is None:
|
||||
span = get_current_span()
|
||||
if not span:
|
||||
return
|
||||
|
||||
span.end_time = datetime.now(timezone.utc)
|
||||
span.status = status
|
||||
if error:
|
||||
span.error = error
|
||||
span.status = SpanStatus.ERROR
|
||||
if metadata_update:
|
||||
span.metadata.update(metadata_update)
|
||||
if details_update:
|
||||
span.details.update(details_update)
|
||||
|
||||
# Restore parent span as current
|
||||
trace = get_current_trace()
|
||||
if trace and span.parent_id:
|
||||
parent = next((s for s in trace.spans if s.span_id == span.parent_id), None)
|
||||
_current_span.set(parent)
|
||||
else:
|
||||
_current_span.set(None)
|
||||
|
||||
logger.debug(
|
||||
"span_ended",
|
||||
span_id=span.span_id,
|
||||
duration_ms=span.duration_ms,
|
||||
status=status.value,
|
||||
)
|
||||
|
||||
|
||||
def end_trace(
|
||||
response: dict[str, Any] | None = None,
|
||||
status: str = "completed",
|
||||
) -> str | None:
|
||||
"""
|
||||
End the current trace and write to file.
|
||||
|
||||
Args:
|
||||
response: Response data to include
|
||||
status: Final trace status ("completed" or "error")
|
||||
|
||||
Returns:
|
||||
Path to trace file if written, None otherwise
|
||||
"""
|
||||
trace = get_current_trace()
|
||||
if not trace:
|
||||
return None
|
||||
|
||||
trace.response = response
|
||||
trace.status = status
|
||||
|
||||
# Write trace to file
|
||||
trace_path = _write_trace(trace)
|
||||
|
||||
# Clear context
|
||||
_current_trace.set(None)
|
||||
_current_span.set(None)
|
||||
|
||||
logger.info(
|
||||
"trace_completed",
|
||||
trace_id=trace.trace_id,
|
||||
total_duration_ms=round(trace.total_duration_ms, 2) if trace.total_duration_ms else None,
|
||||
span_count=len(trace.spans),
|
||||
path=str(trace_path) if trace_path else None,
|
||||
)
|
||||
|
||||
return str(trace_path) if trace_path else None
|
||||
|
||||
|
||||
def _write_trace(trace: Trace) -> Path | None:
|
||||
"""Write trace to JSON file."""
|
||||
try:
|
||||
# Ensure traces directory exists
|
||||
traces_dir = Path("logs/traces")
|
||||
traces_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write trace file
|
||||
trace_path = traces_dir / f"{trace.trace_id}.json"
|
||||
with open(trace_path, "w") as f:
|
||||
json.dump(trace.to_dict(), f, indent=2, default=str)
|
||||
|
||||
return trace_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error("trace_write_failed", error=str(e), trace_id=trace.trace_id)
|
||||
return None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def trace_span(
|
||||
name: str,
|
||||
span_type: SpanType,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
):
|
||||
"""
|
||||
Async context manager for tracing a span.
|
||||
|
||||
Automatically handles start/end timing and error capture.
|
||||
|
||||
Usage:
|
||||
async with trace_span("steward_analysis", SpanType.STEWARD) as span:
|
||||
result = await analyze_request(...)
|
||||
if span:
|
||||
span.metadata["result_count"] = len(result)
|
||||
|
||||
Args:
|
||||
name: Span name
|
||||
span_type: Type of operation
|
||||
metadata: Initial metadata
|
||||
details: Initial details (expandable in viewer)
|
||||
|
||||
Yields:
|
||||
Span object or None if tracing disabled
|
||||
"""
|
||||
span = start_span(name, span_type, metadata, details)
|
||||
try:
|
||||
yield span
|
||||
except Exception as e:
|
||||
end_span(span, SpanStatus.ERROR, error=str(e))
|
||||
raise
|
||||
else:
|
||||
end_span(span, SpanStatus.OK)
|
||||
|
||||
|
||||
def add_tool_spans_from_messages(messages: list[Any], parent_span: Span | None = None) -> None:
|
||||
"""
|
||||
Extract tool calls from PydanticAI result messages and add as child spans.
|
||||
|
||||
Call this after an agent.run() to capture tool-level timing retroactively.
|
||||
Note: Since we don't have actual timing, we estimate based on sequence.
|
||||
|
||||
Args:
|
||||
messages: List from result.new_messages()
|
||||
parent_span: Parent span to attach tool spans to
|
||||
"""
|
||||
trace = get_current_trace()
|
||||
if not trace or not parent_span:
|
||||
return
|
||||
|
||||
# Import PydanticAI message types
|
||||
try:
|
||||
from pydantic_ai.messages import ModelRequest, ModelResponse, ToolCallPart, ToolReturnPart
|
||||
except ImportError:
|
||||
return
|
||||
|
||||
# Track tool calls and their returns
|
||||
tool_calls: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for msg in messages:
|
||||
if isinstance(msg, ModelResponse):
|
||||
for part in msg.parts:
|
||||
if isinstance(part, ToolCallPart):
|
||||
tool_calls[part.tool_call_id] = {
|
||||
"name": part.tool_name,
|
||||
"args": part.args if hasattr(part, 'args') else {},
|
||||
}
|
||||
elif isinstance(msg, ModelRequest):
|
||||
for part in msg.parts:
|
||||
if isinstance(part, ToolReturnPart):
|
||||
if part.tool_call_id in tool_calls:
|
||||
tool_info = tool_calls[part.tool_call_id]
|
||||
# Create a span for this tool call
|
||||
span = Span(
|
||||
span_id=_generate_id("span_"),
|
||||
name=tool_info["name"],
|
||||
type=SpanType.TOOL,
|
||||
start_time=parent_span.start_time, # Approximate
|
||||
end_time=parent_span.end_time or datetime.now(timezone.utc),
|
||||
parent_id=parent_span.span_id,
|
||||
status=SpanStatus.OK,
|
||||
metadata={
|
||||
"tool_name": tool_info["name"],
|
||||
"args_preview": str(tool_info.get("args", {}))[:100],
|
||||
},
|
||||
details={
|
||||
"args": tool_info.get("args", {}),
|
||||
"result": part.content[:2000] if isinstance(part.content, str) else str(part.content)[:2000],
|
||||
},
|
||||
)
|
||||
parent_span.children.append(span.span_id)
|
||||
trace.spans.append(span)
|
||||
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
Trace viewer router.
|
||||
|
||||
Serves the trace viewer UI and trace files when tracing is enabled.
|
||||
Only available when DEBUG=true.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/traces", tags=["traces"])
|
||||
|
||||
TRACES_DIR = Path("logs/traces")
|
||||
VIEWER_PATH = TRACES_DIR / "viewer.html"
|
||||
|
||||
|
||||
def tracing_enabled() -> bool:
|
||||
"""Check if tracing is enabled."""
|
||||
return config.DEBUG
|
||||
|
||||
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def get_trace_viewer():
|
||||
"""
|
||||
Serve the trace viewer UI.
|
||||
|
||||
Returns the standalone HTML viewer for browsing traces.
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
raise HTTPException(status_code=404, detail="Tracing not enabled")
|
||||
|
||||
if not VIEWER_PATH.exists():
|
||||
raise HTTPException(status_code=404, detail="Viewer not found")
|
||||
|
||||
return HTMLResponse(content=VIEWER_PATH.read_text())
|
||||
|
||||
|
||||
@router.get("/list")
|
||||
async def list_traces(
|
||||
limit: int = 50,
|
||||
since_minutes: int | None = None,
|
||||
status: str | None = None,
|
||||
search: str | None = None,
|
||||
):
|
||||
"""
|
||||
List available trace files.
|
||||
|
||||
Returns most recent traces first, with basic metadata.
|
||||
|
||||
Args:
|
||||
limit: Maximum number of traces to return (default 50)
|
||||
since_minutes: Only return traces from the last N minutes
|
||||
status: Filter by status (completed, error, streaming)
|
||||
search: Search in request preview text
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
raise HTTPException(status_code=404, detail="Tracing not enabled")
|
||||
|
||||
if not TRACES_DIR.exists():
|
||||
return {"traces": [], "total": 0}
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
# Calculate cutoff time if filtering by time
|
||||
cutoff_time = None
|
||||
if since_minutes:
|
||||
cutoff_time = datetime.now(timezone.utc) - timedelta(minutes=since_minutes)
|
||||
|
||||
# Get all trace files, sorted by modification time (newest first)
|
||||
trace_files = sorted(
|
||||
TRACES_DIR.glob("trace_*.json"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
traces = []
|
||||
for path in trace_files:
|
||||
if len(traces) >= limit:
|
||||
break
|
||||
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Parse timestamp for filtering
|
||||
trace_timestamp = data.get("timestamp")
|
||||
if cutoff_time and trace_timestamp:
|
||||
try:
|
||||
ts = datetime.fromisoformat(trace_timestamp.replace('Z', '+00:00'))
|
||||
if ts < cutoff_time:
|
||||
continue
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Filter by status
|
||||
trace_status = data.get("status", "")
|
||||
if status and trace_status != status:
|
||||
continue
|
||||
|
||||
# Filter by search text
|
||||
request_preview = data.get("request", {}).get("input_preview", "")
|
||||
if search and search.lower() not in request_preview.lower():
|
||||
continue
|
||||
|
||||
traces.append({
|
||||
"trace_id": data.get("trace_id"),
|
||||
"timestamp": trace_timestamp,
|
||||
"user": data.get("user"),
|
||||
"status": trace_status,
|
||||
"total_duration_ms": data.get("total_duration_ms"),
|
||||
"span_count": len(data.get("spans", [])),
|
||||
"request_preview": request_preview[:100],
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning("trace_list_parse_error", path=str(path), error=str(e))
|
||||
|
||||
return {"traces": traces, "total": len(traces)}
|
||||
|
||||
|
||||
@router.get("/{trace_id}")
|
||||
async def get_trace(trace_id: str):
|
||||
"""
|
||||
Get a specific trace by ID.
|
||||
|
||||
Returns the full trace JSON.
|
||||
"""
|
||||
if not tracing_enabled():
|
||||
raise HTTPException(status_code=404, detail="Tracing not enabled")
|
||||
|
||||
# Sanitize trace_id to prevent path traversal
|
||||
if not trace_id.startswith("trace_") or "/" in trace_id or "\\" in trace_id:
|
||||
raise HTTPException(status_code=400, detail="Invalid trace ID")
|
||||
|
||||
trace_path = TRACES_DIR / f"{trace_id}.json"
|
||||
|
||||
if not trace_path.exists():
|
||||
raise HTTPException(status_code=404, detail="Trace not found")
|
||||
|
||||
try:
|
||||
import json
|
||||
with open(trace_path) as f:
|
||||
data = json.load(f)
|
||||
return JSONResponse(content=data)
|
||||
except Exception as e:
|
||||
logger.error("trace_read_error", trace_id=trace_id, error=str(e))
|
||||
raise HTTPException(status_code=500, detail="Failed to read trace")
|
||||
+12
-4
@@ -23,6 +23,7 @@ from src.core.exceptions import AppException
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.router import router as core_router
|
||||
from src.core.startup import initialize_application
|
||||
from src.core.tracing_router import router as tracing_router
|
||||
from src.models.router import router as models_router
|
||||
from src.responses.router import router as responses_router
|
||||
|
||||
@@ -43,14 +44,16 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app_name=config.APP_NAME,
|
||||
version=config.APP_VERSION,
|
||||
environment=config.ENVIRONMENT.value,
|
||||
prefer_cloud=config.PREFER_CLOUD_BACKEND,
|
||||
anthropic_model=config.ANTHROPIC_MODEL,
|
||||
ollama_host=str(config.OLLAMA_HOST),
|
||||
ollama_model=config.OLLAMA_DEFAULT_MODEL,
|
||||
redis_url=config.redis_url,
|
||||
redis_url=config.redis_memory_url,
|
||||
log_format=config.log_format,
|
||||
)
|
||||
|
||||
# Initialize application (register household members, etc.)
|
||||
initialize_application()
|
||||
# Initialize application (check Claude health, register household members, etc.)
|
||||
await initialize_application()
|
||||
|
||||
yield
|
||||
|
||||
@@ -90,7 +93,12 @@ def create_application() -> FastAPI:
|
||||
application.include_router(chat_router, prefix=config.API_PREFIX)
|
||||
application.include_router(models_router, prefix=config.API_PREFIX)
|
||||
application.include_router(responses_router, prefix=config.API_PREFIX) # Responses API
|
||||
|
||||
|
||||
# Conditionally include tracing router (only in debug mode)
|
||||
if config.DEBUG:
|
||||
application.include_router(tracing_router)
|
||||
logger.info("tracing_router_enabled")
|
||||
|
||||
return application
|
||||
|
||||
|
||||
|
||||
+5
-73
@@ -10,7 +10,6 @@ from sse_starlette.sse import EventSourceResponse
|
||||
from src.responses import service
|
||||
from src.responses.schemas import ResponseRequest, Response
|
||||
from src.core.exceptions import ModelNotFoundError, AppException
|
||||
from src.core.context import current_user, current_conversation, get_default_user
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -37,77 +36,16 @@ async def create_response(
|
||||
|
||||
Returns:
|
||||
Response object or SSE stream
|
||||
|
||||
Example non-streaming request:
|
||||
POST /v1/responses
|
||||
{
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Hello"}],
|
||||
"reasoning": {"effort": "medium", "summary": "auto"},
|
||||
"stream": false
|
||||
}
|
||||
|
||||
Example streaming request:
|
||||
POST /v1/responses
|
||||
{
|
||||
"model": "lorem-tester",
|
||||
"input": [{"role": "user", "content": "Hello"}],
|
||||
"stream": true
|
||||
}
|
||||
|
||||
Response format (non-streaming):
|
||||
{
|
||||
"id": "resp_...",
|
||||
"object": "response",
|
||||
"created_at": 1733529600,
|
||||
"model": "lorem-tester",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_...",
|
||||
"summary": ["Analyzing...", "Considering..."]
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_...",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Lorem ipsum..."}]
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 50,
|
||||
"reasoning_tokens": 20,
|
||||
"total_tokens": 80
|
||||
}
|
||||
}
|
||||
|
||||
Streaming format (SSE):
|
||||
event: response.reasoning_summary_text.delta
|
||||
data: {"delta": "Analyzing..."}
|
||||
|
||||
event: response.output_text.delta
|
||||
data: {"delta": "Lorem"}
|
||||
|
||||
event: response.done
|
||||
data: {"response": {...}}
|
||||
"""
|
||||
# Set request context (propagates through all async calls)
|
||||
effective_user = request.user or get_default_user()
|
||||
user_token = current_user.set(effective_user)
|
||||
conv_id = request.metadata.get("conversation_id") if request.metadata else None
|
||||
conv_token = current_conversation.set(conv_id)
|
||||
|
||||
logger.info(
|
||||
"response_request_received",
|
||||
model=request.model,
|
||||
user=effective_user,
|
||||
conversation_id=conv_id,
|
||||
user=request.user,
|
||||
streaming=request.stream,
|
||||
)
|
||||
|
||||
try:
|
||||
# Check if this is a Tatlock request - use Steward preprocessing (Phase 2)
|
||||
# Check if this is a Tatlock request - use Steward preprocessing
|
||||
model_id = request.model
|
||||
if "." in model_id:
|
||||
model_id = model_id.split(".", 1)[1]
|
||||
@@ -116,21 +54,20 @@ async def create_response(
|
||||
|
||||
if request.stream:
|
||||
logger.info("Streaming response requested")
|
||||
|
||||
if use_steward:
|
||||
logger.info("Streaming with Steward preprocessing for Tatlock request")
|
||||
# Use Steward + Tatlock streaming (Milestone 3.5)
|
||||
from src.responses.streaming import StreamingCoordinator
|
||||
coordinator = StreamingCoordinator()
|
||||
return EventSourceResponse(
|
||||
coordinator.stream_response_with_steward(request)
|
||||
)
|
||||
else:
|
||||
# Regular streaming for non-Tatlock models
|
||||
return EventSourceResponse(
|
||||
service.create_response_stream(request)
|
||||
)
|
||||
|
||||
# Use appropriate service method
|
||||
# Non-streaming response
|
||||
if use_steward:
|
||||
logger.info("Using Steward preprocessing for Tatlock request")
|
||||
return await service.create_response_with_steward(request)
|
||||
@@ -148,8 +85,3 @@ async def create_response(
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
finally:
|
||||
# Reset context (important for connection reuse)
|
||||
current_user.reset(user_token)
|
||||
current_conversation.reset(conv_token)
|
||||
|
||||
+249
-140
@@ -26,6 +26,8 @@ from src.responses.context import ContextWindow
|
||||
from src.core.preprocessing import preprocess_request
|
||||
from src.core.tool_tracking import ToolCallTracker
|
||||
from src.core.logging_config import get_logger
|
||||
from src.core.tracing import start_trace, end_trace, start_span, SpanType
|
||||
from src.core.context import current_user, current_conversation, get_default_user
|
||||
from src.agents.steward.schemas import StewardRecommendation
|
||||
|
||||
import re
|
||||
@@ -34,6 +36,29 @@ import asyncio
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def _extract_user_input(input_data) -> str:
|
||||
"""Extract user input text from request input for tracing."""
|
||||
if isinstance(input_data, str):
|
||||
return input_data
|
||||
elif isinstance(input_data, list) and input_data:
|
||||
last_msg = input_data[-1]
|
||||
if isinstance(last_msg, dict):
|
||||
return last_msg.get("content", str(last_msg))
|
||||
return str(last_msg)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_response_preview(response: Response) -> str:
|
||||
"""Extract response preview text for tracing."""
|
||||
if response.output:
|
||||
for item in response.output:
|
||||
if hasattr(item, 'content'):
|
||||
for content in item.content:
|
||||
if hasattr(content, 'text'):
|
||||
return content.text[:200]
|
||||
return ""
|
||||
|
||||
|
||||
async def _execute_single_delegation(
|
||||
agent_name: str,
|
||||
task: str,
|
||||
@@ -405,45 +430,88 @@ async def create_response(request: ResponseRequest) -> Response:
|
||||
# Get or generate conversation ID
|
||||
conversation_id = await _conversation_history.get_conversation_id(request)
|
||||
|
||||
# Strip pipeline prefix if present (e.g., "pipeline.model" -> "model")
|
||||
model_id = request.model
|
||||
if "." in model_id:
|
||||
model_id = model_id.split(".", 1)[1]
|
||||
# Set context for tracing
|
||||
effective_user = request.user or get_default_user()
|
||||
current_user.set(effective_user)
|
||||
current_conversation.set(conversation_id)
|
||||
|
||||
# Get agent for model
|
||||
agent = ModelRegistry.get_agent(model_id)
|
||||
# Extract user input for tracing
|
||||
user_input = _extract_user_input(request.input)
|
||||
|
||||
# Collect all output items from agent
|
||||
output_items = []
|
||||
async for item in agent.generate_response(
|
||||
messages=request.input,
|
||||
reasoning=request.reasoning,
|
||||
tools=request.tools,
|
||||
temperature=request.temperature,
|
||||
max_tokens=request.max_output_tokens,
|
||||
stop=request.stop,
|
||||
):
|
||||
output_items.append(item)
|
||||
|
||||
# Convert agent OutputItems to schema OutputItems
|
||||
converted_items = _convert_output_items(output_items)
|
||||
|
||||
# Calculate token usage
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=converted_items,
|
||||
usage=usage
|
||||
# Start trace
|
||||
trace = start_trace(
|
||||
conversation_id=conversation_id,
|
||||
user=effective_user,
|
||||
request={
|
||||
"model": request.model,
|
||||
"input_preview": user_input[:200] if user_input else "",
|
||||
"full_input": request.input,
|
||||
"streaming": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Track conversation history (for analytics and future vector memory)
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
# Start service span
|
||||
service_span = start_span(
|
||||
"create_response",
|
||||
SpanType.ROUTER,
|
||||
metadata={"model": request.model, "user": effective_user},
|
||||
)
|
||||
|
||||
return response
|
||||
try:
|
||||
# Strip pipeline prefix if present (e.g., "pipeline.model" -> "model")
|
||||
model_id = request.model
|
||||
if "." in model_id:
|
||||
model_id = model_id.split(".", 1)[1]
|
||||
|
||||
# Get agent for model
|
||||
agent = ModelRegistry.get_agent(model_id)
|
||||
|
||||
# Collect all output items from agent
|
||||
output_items = []
|
||||
async for item in agent.generate_response(
|
||||
messages=request.input,
|
||||
reasoning=request.reasoning,
|
||||
tools=request.tools,
|
||||
temperature=request.temperature,
|
||||
max_tokens=request.max_output_tokens,
|
||||
stop=request.stop,
|
||||
):
|
||||
output_items.append(item)
|
||||
|
||||
# Convert agent OutputItems to schema OutputItems
|
||||
converted_items = _convert_output_items(output_items)
|
||||
|
||||
# Calculate token usage
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=converted_items,
|
||||
usage=usage
|
||||
)
|
||||
|
||||
# Track conversation history (for analytics and future vector memory)
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
|
||||
# End trace with response info
|
||||
response_preview = _extract_response_preview(response)
|
||||
end_trace(
|
||||
response={
|
||||
"output_preview": response_preview,
|
||||
"output_count": len(response.output) if response.output else 0,
|
||||
"status": response.status,
|
||||
},
|
||||
status="completed",
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
end_trace(status="error")
|
||||
raise
|
||||
|
||||
|
||||
async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
@@ -454,7 +522,7 @@ async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
1. Steward analyzes the request and recommends capabilities
|
||||
2. Phase 1: Tatlock orchestrates tool calls and expert delegations
|
||||
3. Phase 2: Tatlock synthesizes butler-toned response from results
|
||||
4. Tool usage is tracked for benchmarking
|
||||
4. Tool usage is tracked for analysis
|
||||
|
||||
Args:
|
||||
request: Response request
|
||||
@@ -473,133 +541,174 @@ async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
# Get or generate conversation ID
|
||||
conversation_id = await _conversation_history.get_conversation_id(request)
|
||||
|
||||
# Extract user message and conversation history
|
||||
user_message = ""
|
||||
for msg in reversed(request.input):
|
||||
if msg.get("role") == "user":
|
||||
user_message = msg.get("content", "")
|
||||
break
|
||||
# Set context for tracing
|
||||
effective_user = request.user or get_default_user()
|
||||
current_user.set(effective_user)
|
||||
current_conversation.set(conversation_id)
|
||||
|
||||
# Conversation history is all messages except the current one
|
||||
conversation_history = request.input[:-1] if len(request.input) > 1 else []
|
||||
# Extract user input for tracing
|
||||
user_input = _extract_user_input(request.input)
|
||||
|
||||
logger.info(
|
||||
"creating_response_with_steward",
|
||||
user_message_preview=user_message[:100],
|
||||
history_length=len(conversation_history),
|
||||
# Start trace
|
||||
trace = start_trace(
|
||||
conversation_id=conversation_id,
|
||||
user=effective_user,
|
||||
request={
|
||||
"model": request.model,
|
||||
"input_preview": user_input[:200] if user_input else "",
|
||||
"full_input": request.input,
|
||||
"streaming": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Steward preprocessing
|
||||
enriched = await preprocess_request(
|
||||
user_message,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
# Start service span
|
||||
service_span = start_span(
|
||||
"create_response_with_steward",
|
||||
SpanType.ROUTER,
|
||||
metadata={"model": request.model, "user": effective_user},
|
||||
)
|
||||
|
||||
# Initialize tool tracker
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
try:
|
||||
# Extract user message and conversation history
|
||||
user_message = ""
|
||||
for msg in reversed(request.input):
|
||||
if msg.get("role") == "user":
|
||||
user_message = msg.get("content", "")
|
||||
break
|
||||
|
||||
# Check if direct delegation is recommended
|
||||
# If Steward recommends ONLY delegation agents (biographer/librarian/housekeeper),
|
||||
# we still use two-phase but delegate directly in Phase 1
|
||||
delegation_agents = {"biographer", "librarian", "housekeeper"}
|
||||
delegation_only = all(
|
||||
cap in delegation_agents
|
||||
for cap in enriched.recommendation.recommended_capabilities
|
||||
) and enriched.recommendation.recommended_capabilities
|
||||
# Conversation history is all messages except the current one
|
||||
conversation_history = request.input[:-1] if len(request.input) > 1 else []
|
||||
|
||||
from src.agents.tatlock import TatlockAgent
|
||||
tatlock = TatlockAgent()
|
||||
|
||||
# Use enriched query (with location/timezone context) if available
|
||||
effective_query = enriched.recommendation.enriched_query or user_message
|
||||
|
||||
if delegation_only:
|
||||
# Direct delegation path - collect results then synthesize
|
||||
orchestration_results = await _direct_delegation_with_results(
|
||||
effective_query, enriched.recommendation, tracker, conversation_id
|
||||
)
|
||||
else:
|
||||
# Phase 1: Orchestrate tool calls
|
||||
orchestration_results = await tatlock.orchestrate_tool_calls(
|
||||
user_message=effective_query,
|
||||
steward_note=enriched.steward_note,
|
||||
scoped_tools=enriched.scoped_tools,
|
||||
message_history=conversation_history,
|
||||
tool_tracker=tracker,
|
||||
logger.info(
|
||||
"creating_response_with_steward",
|
||||
user_message_preview=user_message[:100],
|
||||
history_length=len(conversation_history),
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Handle text-based delegation fallback if present
|
||||
if "[DELEGATE:" in orchestration_results.get("raw_output", ""):
|
||||
text_delegation_results = await _handle_text_delegation(
|
||||
orchestration_results["raw_output"], tracker, conversation_id
|
||||
# Steward preprocessing
|
||||
enriched = await preprocess_request(
|
||||
user_message,
|
||||
conversation_history=conversation_history,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Initialize tool tracker
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Check if direct delegation is recommended
|
||||
# If Steward recommends ONLY delegation agents (biographer/librarian/housekeeper),
|
||||
# we still use two-phase but delegate directly in Phase 1
|
||||
delegation_agents = {"biographer", "librarian", "housekeeper"}
|
||||
delegation_only = all(
|
||||
cap in delegation_agents
|
||||
for cap in enriched.recommendation.recommended_capabilities
|
||||
) and enriched.recommendation.recommended_capabilities
|
||||
|
||||
from src.agents.tatlock import TatlockAgent
|
||||
tatlock = TatlockAgent()
|
||||
|
||||
# Use enriched query (with location/timezone context) if available
|
||||
effective_query = enriched.recommendation.enriched_query or user_message
|
||||
|
||||
if delegation_only:
|
||||
# Direct delegation path - collect results then synthesize
|
||||
orchestration_results = await _direct_delegation_with_results(
|
||||
effective_query, enriched.recommendation, tracker, conversation_id
|
||||
)
|
||||
else:
|
||||
# Phase 1: Orchestrate tool calls
|
||||
orchestration_results = await tatlock.orchestrate_tool_calls(
|
||||
user_message=effective_query,
|
||||
steward_note=enriched.steward_note,
|
||||
scoped_tools=enriched.scoped_tools,
|
||||
message_history=conversation_history,
|
||||
tool_tracker=tracker,
|
||||
)
|
||||
# Add text delegation results to expert_results
|
||||
if text_delegation_results != orchestration_results["raw_output"]:
|
||||
orchestration_results["expert_results"]["text_delegation"] = text_delegation_results
|
||||
|
||||
# Phase 2: Synthesize butler-toned response from all results
|
||||
tatlock_response = await tatlock.synthesize_from_results(
|
||||
user_message=user_message,
|
||||
orchestration_results=orchestration_results,
|
||||
message_history=conversation_history,
|
||||
)
|
||||
# Handle text-based delegation fallback if present
|
||||
if "[DELEGATE:" in orchestration_results.get("raw_output", ""):
|
||||
text_delegation_results = await _handle_text_delegation(
|
||||
orchestration_results["raw_output"], tracker, conversation_id
|
||||
)
|
||||
# Add text delegation results to expert_results
|
||||
if text_delegation_results != orchestration_results["raw_output"]:
|
||||
orchestration_results["expert_results"]["text_delegation"] = text_delegation_results
|
||||
|
||||
# Finalize tool tracking
|
||||
await tracker.finalize()
|
||||
# Phase 2: Synthesize butler-toned response from all results
|
||||
tatlock_response = await tatlock.synthesize_from_results(
|
||||
user_message=user_message,
|
||||
orchestration_results=orchestration_results,
|
||||
message_history=conversation_history,
|
||||
)
|
||||
|
||||
# Build response output items
|
||||
output_items = []
|
||||
# Finalize tool tracking
|
||||
await tracker.finalize()
|
||||
|
||||
# Add Steward reasoning as a reasoning output item
|
||||
output_items.append(ReasoningOutputItem(
|
||||
id=f"reasoning_{generate_id()}",
|
||||
summary=[
|
||||
"🎩 Steward's Analysis:",
|
||||
enriched.steward_reasoning,
|
||||
],
|
||||
status="completed"
|
||||
))
|
||||
# Build response output items
|
||||
output_items = []
|
||||
|
||||
# Add Tatlock's message
|
||||
output_items.append(MessageOutputItem(
|
||||
id=f"msg_{generate_id()}",
|
||||
role="assistant",
|
||||
content=[OutputTextContent(
|
||||
type="output_text",
|
||||
text=tatlock_response,
|
||||
annotations=[]
|
||||
)],
|
||||
status="completed"
|
||||
))
|
||||
# Add Steward reasoning as reasoning output
|
||||
if enriched.steward_reasoning:
|
||||
output_items.append(ReasoningOutputItem(
|
||||
id=f"rs_{generate_id()}",
|
||||
summary=[enriched.steward_reasoning],
|
||||
status="completed"
|
||||
))
|
||||
|
||||
# Calculate usage (approximate)
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
# Add Tatlock's message
|
||||
output_items.append(MessageOutputItem(
|
||||
id=f"msg_{generate_id()}",
|
||||
role="assistant",
|
||||
content=[OutputTextContent(
|
||||
type="output_text",
|
||||
text=tatlock_response,
|
||||
annotations=[]
|
||||
)],
|
||||
status="completed"
|
||||
))
|
||||
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=output_items,
|
||||
usage=usage
|
||||
)
|
||||
# Calculate usage (approximate)
|
||||
usage = _calculate_usage(request.input, output_items)
|
||||
|
||||
# Track conversation history
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
response = Response(
|
||||
id=f"resp_{generate_id()}",
|
||||
created_at=int(time.time()),
|
||||
model=request.model,
|
||||
status="completed",
|
||||
output=output_items,
|
||||
usage=usage
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"response_with_steward_complete",
|
||||
response_id=response.id,
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
tool_summary=tracker.get_summary(),
|
||||
)
|
||||
# Track conversation history
|
||||
await _conversation_history.add_response(conversation_id, response)
|
||||
|
||||
return response
|
||||
logger.info(
|
||||
"response_with_steward_complete",
|
||||
response_id=response.id,
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
tool_summary=tracker.get_summary(),
|
||||
)
|
||||
|
||||
# End trace with response info
|
||||
response_preview = _extract_response_preview(response)
|
||||
end_trace(
|
||||
response={
|
||||
"output_preview": response_preview,
|
||||
"output_count": len(response.output) if response.output else 0,
|
||||
"status": response.status,
|
||||
},
|
||||
status="completed",
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
end_trace(status="error")
|
||||
raise
|
||||
|
||||
|
||||
async def create_response_stream(
|
||||
|
||||
@@ -166,26 +166,6 @@ class StreamingCoordinator:
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Stream Steward's analysis as reasoning summary
|
||||
steward_lines = enriched.steward_reasoning.split('\n')
|
||||
for line in steward_lines:
|
||||
if line.strip():
|
||||
yield ReasoningSummaryDelta(delta=line + "\n")
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
yield ReasoningSummaryDone()
|
||||
|
||||
# Add Steward reasoning to output items
|
||||
reasoning_item = ReasoningOutputItem(
|
||||
id=f"reasoning_{generate_id()}",
|
||||
summary=[
|
||||
"🎩 Steward's Analysis:",
|
||||
enriched.steward_reasoning,
|
||||
],
|
||||
status="completed"
|
||||
)
|
||||
output_items.append(reasoning_item)
|
||||
|
||||
# Initialize tool tracker
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
|
||||
@@ -9,13 +9,13 @@ import pytest
|
||||
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
from src.agents.steward.service import analyze_request, format_steward_note, _build_enriched_query
|
||||
from src.core.startup import initialize_application
|
||||
from src.core.startup import register_household_members
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def setup_household_registry():
|
||||
"""Initialize household registry before running tests."""
|
||||
initialize_application()
|
||||
register_household_members()
|
||||
|
||||
|
||||
class TestAnalyzeRequest:
|
||||
@@ -29,17 +29,14 @@ class TestAnalyzeRequest:
|
||||
mock_agent.analyze = AsyncMock(return_value="Simple greeting requires no tools. This is a simple request.")
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"Hello!",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"Hello!",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert result.recommended_capabilities == []
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert mock_agent.analyze.called
|
||||
assert result.recommended_capabilities == []
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert mock_agent.analyze.called
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_math_request(self):
|
||||
@@ -50,16 +47,13 @@ class TestAnalyzeRequest:
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"What's sqrt(144)?",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"What's sqrt(144)?",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert "tatlock_core" in result.recommended_capabilities
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert "tatlock_core" in result.recommended_capabilities
|
||||
assert result.estimated_complexity == "simple"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_conversation_history(self):
|
||||
@@ -75,21 +69,18 @@ class TestAnalyzeRequest:
|
||||
]
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"And what's that times 5?",
|
||||
conversation_history=conversation_history,
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"And what's that times 5?",
|
||||
conversation_history=conversation_history,
|
||||
)
|
||||
assert result.conversation_context.has_previous_context is True
|
||||
assert 0 in result.conversation_context.relevant_turns
|
||||
|
||||
assert result.conversation_context.has_previous_context is True
|
||||
assert 0 in result.conversation_context.relevant_turns
|
||||
|
||||
# Verify conversation history was passed
|
||||
call_kwargs = mock_agent.analyze.call_args.kwargs
|
||||
assert "conversation_history" in call_kwargs
|
||||
assert len(call_kwargs["conversation_history"]) == 2
|
||||
# Verify conversation history was passed
|
||||
call_kwargs = mock_agent.analyze.call_args.kwargs
|
||||
assert "conversation_history" in call_kwargs
|
||||
assert len(call_kwargs["conversation_history"]) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_missing_capabilities(self):
|
||||
@@ -100,16 +91,13 @@ class TestAnalyzeRequest:
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"Generate an image of a sunset",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"Generate an image of a sunset",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert result.missing_capabilities is not None
|
||||
assert "not available" in result.missing_capabilities
|
||||
assert result.missing_capabilities is not None
|
||||
assert "not available" in result.missing_capabilities
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_conversation_id(self):
|
||||
@@ -120,18 +108,15 @@ class TestAnalyzeRequest:
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"Test request",
|
||||
conversation_history=[],
|
||||
conversation_id="test_conv_123",
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"Test request",
|
||||
conversation_history=[],
|
||||
conversation_id="test_conv_123",
|
||||
)
|
||||
|
||||
# Verify analysis completed successfully
|
||||
assert result.recommended_capabilities == ["tatlock_core"]
|
||||
assert result.estimated_complexity == "simple"
|
||||
# Verify analysis completed successfully
|
||||
assert result.recommended_capabilities == ["tatlock_core"]
|
||||
assert result.estimated_complexity == "simple"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_handles_errors(self):
|
||||
|
||||
@@ -34,7 +34,7 @@ async def test_tatlock_conversation_history_memory(async_client: AsyncClient):
|
||||
response_1 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data_1,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_1.status_code == 200
|
||||
@@ -56,7 +56,7 @@ async def test_tatlock_conversation_history_memory(async_client: AsyncClient):
|
||||
response_2 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data_2,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_2.status_code == 200
|
||||
@@ -95,7 +95,7 @@ async def test_tatlock_multi_turn_context(async_client: AsyncClient):
|
||||
response_1 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_1,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_1.status_code == 200
|
||||
@@ -117,7 +117,7 @@ async def test_tatlock_multi_turn_context(async_client: AsyncClient):
|
||||
response_2 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_2,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_2.status_code == 200
|
||||
@@ -150,7 +150,7 @@ async def test_tatlock_tool_call_logging_search(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=60.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -192,7 +192,7 @@ async def test_tatlock_tool_call_logging_calculator(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -243,7 +243,7 @@ async def test_tatlock_tool_call_logging_datetime(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -293,7 +293,7 @@ async def test_tatlock_no_tool_calls_no_logging(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -336,7 +336,7 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
|
||||
response_1 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_1,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_1.status_code == 200
|
||||
@@ -362,7 +362,7 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
|
||||
response_2 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_2,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_2.status_code == 200
|
||||
@@ -378,3 +378,57 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
|
||||
)
|
||||
if not has_calculation:
|
||||
pytest.xfail(f"LLM did not remember calculation (non-deterministic): {second_response[:200]}")
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_tatlock_ollama_fallback(async_client: AsyncClient):
|
||||
"""
|
||||
Test that Tatlock falls back to Ollama when Claude is unavailable.
|
||||
|
||||
Patches _claude_available to False to force the Ollama path,
|
||||
then verifies the system still produces a valid response.
|
||||
"""
|
||||
import src.anthropic.model_selector as model_selector
|
||||
|
||||
# Save original value
|
||||
original = model_selector._claude_available
|
||||
|
||||
try:
|
||||
# Force Ollama fallback
|
||||
model_selector._claude_available = False
|
||||
|
||||
# Verify we're actually using Ollama
|
||||
info = model_selector.get_model_info()
|
||||
assert info["backend"] == "ollama", f"Expected ollama backend, got {info['backend']}"
|
||||
|
||||
request_data = {
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Say hello to me."}
|
||||
],
|
||||
"stream": False
|
||||
}
|
||||
|
||||
# 300s: this test forbids the Claude rescue, and the full local
|
||||
# Steward -> orchestrate -> synthesize flow on gemma4 exceeds 120s
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=300.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify response structure is valid
|
||||
assert "choices" in data
|
||||
assert len(data["choices"]) == 1
|
||||
full_response = data["choices"][0]["message"]["content"]
|
||||
assert len(full_response) > 0, "Ollama should produce a non-empty response"
|
||||
|
||||
print(f"\nOllama fallback response: {full_response[:200]}")
|
||||
|
||||
finally:
|
||||
# Restore original value
|
||||
model_selector._claude_available = original
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Unit tests for backend selection (Ollama primary, Claude fallback).
|
||||
|
||||
These tests set the cached health-check globals directly so they are
|
||||
deterministic regardless of which services are reachable.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from src.anthropic import model_selector
|
||||
from src.core.config import config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_first(monkeypatch):
|
||||
"""Baseline: local-first config, both backends healthy."""
|
||||
monkeypatch.setattr(config, "PREFER_CLOUD_BACKEND", False)
|
||||
monkeypatch.setattr(config, "ANTHROPIC_API_KEY", "sk-test-fake")
|
||||
monkeypatch.setattr(model_selector, "_claude_available", True)
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", True)
|
||||
|
||||
|
||||
class TestResolveBackend:
|
||||
def test_default_is_ollama(self, local_first):
|
||||
assert model_selector.resolve_backend() == "ollama"
|
||||
|
||||
def test_prefer_cloud_config_selects_claude(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(config, "PREFER_CLOUD_BACKEND", True)
|
||||
assert model_selector.resolve_backend() == "claude"
|
||||
|
||||
def test_prefer_cloud_override_selects_claude(self, local_first):
|
||||
assert model_selector.resolve_backend(prefer_cloud=True) == "claude"
|
||||
|
||||
def test_prefer_cloud_without_claude_falls_back_to_ollama(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(config, "PREFER_CLOUD_BACKEND", True)
|
||||
monkeypatch.setattr(model_selector, "_claude_available", False)
|
||||
assert model_selector.resolve_backend() == "ollama"
|
||||
|
||||
def test_ollama_down_falls_back_to_claude(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", False)
|
||||
assert model_selector.resolve_backend() == "claude"
|
||||
|
||||
def test_ollama_down_without_claude_stays_ollama(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", False)
|
||||
monkeypatch.setattr(model_selector, "_claude_available", False)
|
||||
assert model_selector.resolve_backend() == "ollama"
|
||||
|
||||
def test_unknown_ollama_state_counts_as_available(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", None)
|
||||
assert model_selector.resolve_backend() == "ollama"
|
||||
|
||||
|
||||
class TestGetModel:
|
||||
def test_ollama_backend_returns_openai_chat_model(self, local_first):
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
|
||||
model = model_selector.get_model()
|
||||
assert isinstance(model, OpenAIChatModel)
|
||||
assert model.model_name == config.OLLAMA_DEFAULT_MODEL
|
||||
|
||||
def test_claude_backend_returns_anthropic_model(self, local_first):
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
|
||||
model = model_selector.get_model(prefer_cloud=True)
|
||||
assert isinstance(model, AnthropicModel)
|
||||
assert model.model_name == config.ANTHROPIC_MODEL
|
||||
|
||||
|
||||
class TestToolChoiceSettings:
|
||||
def test_ollama_forces_tool_choice(self, local_first):
|
||||
settings = model_selector.get_tool_choice_settings()
|
||||
assert settings.get("extra_body") == {"tool_choice": "required"}
|
||||
|
||||
def test_claude_uses_native_tool_choice(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(config, "PREFER_CLOUD_BACKEND", True)
|
||||
settings = model_selector.get_tool_choice_settings()
|
||||
assert not settings.get("extra_body")
|
||||
|
||||
|
||||
class TestGetModelInfo:
|
||||
def test_reports_ollama_primary(self, local_first):
|
||||
info = model_selector.get_model_info()
|
||||
assert info["backend"] == "ollama"
|
||||
assert info["model"] == config.OLLAMA_DEFAULT_MODEL
|
||||
assert info["ollama_available"] is True
|
||||
assert info["claude_available"] is True
|
||||
assert info["prefer_cloud"] is False
|
||||
|
||||
def test_reports_claude_when_ollama_down(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", False)
|
||||
info = model_selector.get_model_info()
|
||||
assert info["backend"] == "claude"
|
||||
assert info["model"] == config.ANTHROPIC_MODEL
|
||||
+16
-3
@@ -2,6 +2,8 @@
|
||||
Shared test fixtures for all tests.
|
||||
Following FastAPI testing best practices.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
@@ -9,11 +11,22 @@ from httpx import AsyncClient, ASGITransport
|
||||
from src.main import app
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _initialize_app():
|
||||
"""
|
||||
Run application lifespan (Claude health check, household registration, etc.)
|
||||
once per test session. ASGITransport doesn't trigger lifespan events,
|
||||
so we call it explicitly.
|
||||
"""
|
||||
from src.core.startup import initialize_application
|
||||
asyncio.run(initialize_application())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
"""
|
||||
Synchronous test client for FastAPI.
|
||||
|
||||
|
||||
Use for simple tests that don't require async.
|
||||
"""
|
||||
return TestClient(app)
|
||||
@@ -23,7 +36,7 @@ def client() -> TestClient:
|
||||
async def async_client() -> AsyncClient:
|
||||
"""
|
||||
Async test client for FastAPI.
|
||||
|
||||
|
||||
Use for testing async endpoints and streaming.
|
||||
"""
|
||||
async with AsyncClient(
|
||||
@@ -37,7 +50,7 @@ async def async_client() -> AsyncClient:
|
||||
def mock_chat_request() -> dict:
|
||||
"""Standard chat completion request fixture."""
|
||||
return {
|
||||
"model": "Tatlock",
|
||||
"model": "lorem-tester",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, world!"}
|
||||
],
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
"""
|
||||
Wire-level contract tests for external service boundaries.
|
||||
|
||||
Each test sends the raw request the application code sends (no client
|
||||
wrappers, no mocks) and asserts on the response shape, so boundary
|
||||
breakage is caught directly instead of surfacing as agent misbehavior.
|
||||
|
||||
Semantics:
|
||||
- Service unreachable -> skip (an outage is not a contract violation)
|
||||
- Service reachable but wrong response shape -> fail
|
||||
|
||||
Run with: make test-contracts
|
||||
"""
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from src.core.config import config
|
||||
|
||||
OLLAMA = str(config.OLLAMA_HOST).rstrip("/")
|
||||
QDRANT = f"http://{config.QDRANT_HOST}:{config.QDRANT_PORT}"
|
||||
SEARXNG = str(config.SEARXNG_HOST).rstrip("/")
|
||||
|
||||
CALCULATOR_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calculator",
|
||||
"description": "Evaluate a math expression",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"expression": {"type": "string"}},
|
||||
"required": ["expression"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def _get_or_skip(url: str, service: str, timeout: float = 5.0) -> httpx.Response:
|
||||
"""GET a URL, skipping the test if the service is unreachable."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
return await client.get(url)
|
||||
except httpx.TransportError as e:
|
||||
pytest.skip(f"{service} unreachable at {url}: {e}")
|
||||
|
||||
|
||||
async def _post_or_skip(
|
||||
url: str, service: str, payload: dict, timeout: float, headers: dict | None = None
|
||||
) -> httpx.Response:
|
||||
"""POST a payload, skipping the test if the service is unreachable."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
return await client.post(url, json=payload, headers=headers)
|
||||
except httpx.TransportError as e:
|
||||
pytest.skip(f"{service} unreachable at {url}: {e}")
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestOllamaContract:
|
||||
"""Boundary: Ollama native API and its OpenAI-compat layer."""
|
||||
|
||||
async def test_tags_lists_configured_model(self):
|
||||
# Mirrors check_ollama_health()
|
||||
response = await _get_or_skip(f"{OLLAMA}/api/tags", "ollama")
|
||||
assert response.status_code == 200
|
||||
names = [m["name"] for m in response.json()["models"]]
|
||||
model = config.OLLAMA_DEFAULT_MODEL
|
||||
assert model in names or f"{model}:latest" in names, (
|
||||
f"{model} not pulled; available: {names}"
|
||||
)
|
||||
|
||||
async def test_generate_returns_plain_text(self):
|
||||
# Mirrors StewardAgent._call_ollama()
|
||||
response = await _post_or_skip(
|
||||
f"{OLLAMA}/api/generate",
|
||||
"ollama",
|
||||
{
|
||||
"model": config.OLLAMA_DEFAULT_MODEL,
|
||||
"prompt": "Reply with the single word: pong",
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.3, "top_p": 0.9},
|
||||
},
|
||||
timeout=config.OLLAMA_TIMEOUT,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["response"].strip()
|
||||
|
||||
async def test_openai_compat_tool_calling(self):
|
||||
# Mirrors the request PydanticAI's OpenAIChatModel sends for the
|
||||
# orchestration phase, including the extra_body tool_choice.
|
||||
response = await _post_or_skip(
|
||||
f"{OLLAMA}/v1/chat/completions",
|
||||
"ollama",
|
||||
{
|
||||
"model": config.OLLAMA_DEFAULT_MODEL,
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 6 * 7? Use the calculator."}
|
||||
],
|
||||
"tools": [CALCULATOR_TOOL],
|
||||
"tool_choice": "required",
|
||||
"stream": False,
|
||||
},
|
||||
timeout=config.OLLAMA_TIMEOUT,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
message = response.json()["choices"][0]["message"]
|
||||
tool_calls = message.get("tool_calls")
|
||||
assert tool_calls, f"model answered in text instead of calling the tool: {message}"
|
||||
assert tool_calls[0]["function"]["name"] == "calculator"
|
||||
arguments = json.loads(tool_calls[0]["function"]["arguments"])
|
||||
assert "expression" in arguments
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestAnthropicContract:
|
||||
"""Boundary: Anthropic Messages API (the Claude fallback backend)."""
|
||||
|
||||
HEADERS_KEY = "anthropic-version"
|
||||
|
||||
def _headers(self) -> dict:
|
||||
if not config.ANTHROPIC_API_KEY:
|
||||
pytest.skip("ANTHROPIC_API_KEY not configured")
|
||||
return {
|
||||
"x-api-key": config.ANTHROPIC_API_KEY,
|
||||
"anthropic-version": "2023-06-01",
|
||||
}
|
||||
|
||||
async def test_minimal_message_accepted(self):
|
||||
# Mirrors check_claude_health(): tiny request, no sampling params
|
||||
response = await _post_or_skip(
|
||||
"https://api.anthropic.com/v1/messages",
|
||||
"anthropic",
|
||||
{
|
||||
"model": config.ANTHROPIC_MODEL,
|
||||
"max_tokens": 1,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
},
|
||||
timeout=30.0,
|
||||
headers=self._headers(),
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
async def test_temperature_rejected(self):
|
||||
# Pins the Claude Sonnet 5+ contract that broke the Steward:
|
||||
# sampling parameters are rejected with a 400 (and not billed).
|
||||
response = await _post_or_skip(
|
||||
"https://api.anthropic.com/v1/messages",
|
||||
"anthropic",
|
||||
{
|
||||
"model": config.ANTHROPIC_MODEL,
|
||||
"max_tokens": 1,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"temperature": 0.3,
|
||||
},
|
||||
timeout=30.0,
|
||||
headers=self._headers(),
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "temperature" in response.text
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestQdrantContract:
|
||||
"""Boundary: Qdrant REST API (Biographer's vector memory)."""
|
||||
|
||||
async def test_collections_endpoint(self):
|
||||
response = await _get_or_skip(f"{QDRANT}/collections", "qdrant")
|
||||
assert response.status_code == 200
|
||||
assert "collections" in response.json()["result"]
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestSearxngContract:
|
||||
"""Boundary: SearXNG JSON search API (web search tool)."""
|
||||
|
||||
async def test_json_search(self):
|
||||
response = await _get_or_skip(
|
||||
f"{SEARXNG}/search?q=test&format=json", "searxng", timeout=config.SEARXNG_TIMEOUT
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert "results" in response.json()
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestLibraryDeskContract:
|
||||
"""Boundary: library-desk research API (the Librarian's backend)."""
|
||||
|
||||
async def test_health(self):
|
||||
host = getattr(config, "LIBRARY_DESK_HOST", None)
|
||||
if not host:
|
||||
pytest.skip("LIBRARY_DESK_HOST not configured")
|
||||
response = await _get_or_skip(f"{str(host).rstrip('/')}/health", "library-desk")
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestRedisContract:
|
||||
"""Boundary: Redis on the configured memory DB."""
|
||||
|
||||
async def test_roundtrip(self):
|
||||
import redis.asyncio as redis
|
||||
|
||||
client = redis.Redis(
|
||||
host=config.REDIS_HOST,
|
||||
port=config.REDIS_PORT,
|
||||
db=config.REDIS_MEMORY_DB,
|
||||
socket_connect_timeout=3,
|
||||
)
|
||||
try:
|
||||
await client.ping()
|
||||
except Exception as e:
|
||||
pytest.skip(f"redis unreachable: {e}")
|
||||
try:
|
||||
await client.set("contract-test-key", "ok", ex=30)
|
||||
assert await client.get("contract-test-key") == b"ok"
|
||||
await client.delete("contract-test-key")
|
||||
finally:
|
||||
await client.aclose()
|
||||
@@ -1,352 +0,0 @@
|
||||
"""
|
||||
Tests for benchmark storage.
|
||||
|
||||
Tests performance tracking, Redis storage, and analytics features.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.benchmarks import (
|
||||
BenchmarkStore,
|
||||
PerformanceBenchmark,
|
||||
get_benchmark_store,
|
||||
)
|
||||
|
||||
|
||||
class TestPerformanceBenchmark:
|
||||
"""Test PerformanceBenchmark model."""
|
||||
|
||||
def test_benchmark_creation(self):
|
||||
"""Test creating a performance benchmark."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="steward_analysis",
|
||||
duration_seconds=1.23,
|
||||
success=True,
|
||||
recommendation_count=3,
|
||||
)
|
||||
|
||||
assert benchmark.operation == "steward_analysis"
|
||||
assert benchmark.duration_seconds == 1.23
|
||||
assert benchmark.success is True
|
||||
assert benchmark.recommendation_count == 3
|
||||
assert isinstance(benchmark.timestamp, datetime)
|
||||
|
||||
def test_benchmark_with_tool_fields(self):
|
||||
"""Test benchmark with tool-specific fields."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="tool_call",
|
||||
duration_seconds=0.5,
|
||||
success=True,
|
||||
tool_name="calculate",
|
||||
was_recommended=True,
|
||||
was_actually_used=True,
|
||||
)
|
||||
|
||||
assert benchmark.tool_name == "calculate"
|
||||
assert benchmark.was_recommended is True
|
||||
assert benchmark.was_actually_used is True
|
||||
|
||||
def test_benchmark_to_redis_dict(self):
|
||||
"""Test conversion to Redis dict."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
metadata={"key": "value"},
|
||||
)
|
||||
|
||||
redis_dict = benchmark.to_redis_dict()
|
||||
assert redis_dict["operation"] == "test_op"
|
||||
assert redis_dict["duration_seconds"] == 1.0
|
||||
assert redis_dict["success"] == "True" # Booleans stored as strings in Redis
|
||||
assert isinstance(redis_dict["timestamp"], str)
|
||||
assert isinstance(redis_dict["metadata"], str)
|
||||
|
||||
def test_benchmark_from_redis_dict(self):
|
||||
"""Test reconstruction from Redis dict."""
|
||||
now = datetime.now(timezone.utc)
|
||||
redis_dict = {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": 1.5,
|
||||
"success": "True", # Booleans stored as strings in Redis
|
||||
"metadata": json.dumps({"test": "data"}),
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
benchmark = PerformanceBenchmark.from_redis_dict(redis_dict)
|
||||
assert benchmark.operation == "test_op"
|
||||
assert benchmark.duration_seconds == 1.5
|
||||
assert benchmark.success is True # Converted back to bool
|
||||
assert benchmark.metadata == {"test": "data"}
|
||||
|
||||
|
||||
class TestBenchmarkStore:
|
||||
"""Test BenchmarkStore functionality."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_redis(self):
|
||||
"""Create mock Redis client."""
|
||||
mock = AsyncMock()
|
||||
mock.hset = AsyncMock()
|
||||
mock.expire = AsyncMock()
|
||||
mock.zadd = AsyncMock()
|
||||
mock.zrevrangebyscore = AsyncMock(return_value=[])
|
||||
mock.hgetall = AsyncMock(return_value={})
|
||||
mock.aclose = AsyncMock()
|
||||
return mock
|
||||
|
||||
@pytest.fixture
|
||||
def store(self, mock_redis):
|
||||
"""Create benchmark store with mock Redis."""
|
||||
return BenchmarkStore(redis_client=mock_redis)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark(self, store, mock_redis):
|
||||
"""Test recording a benchmark."""
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
await store.record(benchmark)
|
||||
|
||||
# Verify Redis calls
|
||||
mock_redis.hset.assert_called_once()
|
||||
mock_redis.expire.assert_called()
|
||||
mock_redis.zadd.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark_disabled(self, mock_redis):
|
||||
"""Test recording when benchmarks are disabled."""
|
||||
with patch("src.core.benchmarks.config.ENABLE_BENCHMARKS", False):
|
||||
store = BenchmarkStore(redis_client=mock_redis)
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
await store.record(benchmark)
|
||||
|
||||
# Should not call Redis
|
||||
mock_redis.hset.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_record_benchmark_handles_errors(self, store, mock_redis):
|
||||
"""Test recording handles Redis errors gracefully."""
|
||||
mock_redis.hset.side_effect = Exception("Redis error")
|
||||
|
||||
benchmark = PerformanceBenchmark(
|
||||
operation="test_op",
|
||||
duration_seconds=1.0,
|
||||
success=True,
|
||||
)
|
||||
|
||||
# Should not raise exception
|
||||
await store.record(benchmark)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_benchmarks(self, store, mock_redis):
|
||||
"""Test querying benchmarks."""
|
||||
# Setup mock data
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_key = f"benchmark:test_op:{int(now.timestamp() * 1000)}"
|
||||
mock_redis.zrevrangebyscore.return_value = [mock_key]
|
||||
|
||||
# Mock hgetall to return proper data (booleans as strings, like Redis)
|
||||
mock_redis.hgetall.return_value = {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": 1.5, # Numeric, not string
|
||||
"success": "True", # Booleans stored as strings in Redis
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
results = await store.query("test_op", limit=10)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].operation == "test_op"
|
||||
mock_redis.zrevrangebyscore.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_with_time_range(self, store, mock_redis):
|
||||
"""Test querying with time range."""
|
||||
now = datetime.now(timezone.utc)
|
||||
start_time = now - timedelta(hours=1)
|
||||
end_time = now
|
||||
|
||||
await store.query("test_op", start_time=start_time, end_time=end_time)
|
||||
|
||||
# Verify time range was converted to timestamps
|
||||
call_args = mock_redis.zrevrangebyscore.call_args
|
||||
assert call_args is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_disabled_benchmarks(self, mock_redis):
|
||||
"""Test querying when benchmarks are disabled."""
|
||||
with patch("src.core.benchmarks.config.ENABLE_BENCHMARKS", False):
|
||||
store = BenchmarkStore(redis_client=mock_redis)
|
||||
results = await store.query("test_op")
|
||||
assert results == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_handles_errors(self, store, mock_redis):
|
||||
"""Test query handles errors gracefully."""
|
||||
mock_redis.zrevrangebyscore.side_effect = Exception("Redis error")
|
||||
|
||||
results = await store.query("test_op")
|
||||
assert results == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_statistics(self, store, mock_redis):
|
||||
"""Test getting statistics."""
|
||||
# Setup mock data with multiple benchmarks
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_keys = [
|
||||
f"benchmark:test_op:{int((now - timedelta(seconds=i)).timestamp() * 1000)}"
|
||||
for i in range(3)
|
||||
]
|
||||
mock_redis.zrevrangebyscore.return_value = mock_keys
|
||||
|
||||
# Return different durations and success values
|
||||
benchmarks_data = [
|
||||
{"duration_seconds": "1.0", "success": "True"},
|
||||
{"duration_seconds": "2.0", "success": "True"},
|
||||
{"duration_seconds": "3.0", "success": "False"},
|
||||
]
|
||||
|
||||
async def mock_hgetall(key):
|
||||
idx = mock_keys.index(key)
|
||||
data = benchmarks_data[idx]
|
||||
return {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "test_op",
|
||||
"duration_seconds": float(data["duration_seconds"]),
|
||||
"success": data["success"], # Pass string through, from_redis_dict converts
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": None,
|
||||
"was_recommended": None,
|
||||
"was_actually_used": None,
|
||||
"conversation_id": None,
|
||||
}
|
||||
|
||||
mock_redis.hgetall.side_effect = mock_hgetall
|
||||
|
||||
stats = await store.get_statistics("test_op")
|
||||
|
||||
assert stats["count"] == 3
|
||||
assert stats["avg_duration"] == 2.0 # (1 + 2 + 3) / 3
|
||||
assert stats["min_duration"] == 1.0
|
||||
assert stats["max_duration"] == 3.0
|
||||
assert stats["success_rate"] == pytest.approx(66.67, rel=0.01)
|
||||
assert stats["total_successes"] == 2
|
||||
assert stats["total_failures"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_statistics_empty(self, store, mock_redis):
|
||||
"""Test statistics with no data."""
|
||||
mock_redis.zrevrangebyscore.return_value = []
|
||||
|
||||
stats = await store.get_statistics("test_op")
|
||||
|
||||
assert stats["count"] == 0
|
||||
assert stats["avg_duration"] == 0.0
|
||||
assert stats["success_rate"] == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_tool_accuracy(self, store, mock_redis):
|
||||
"""Test tool accuracy calculation."""
|
||||
# Setup mock data
|
||||
now = datetime.now(timezone.utc)
|
||||
mock_keys = [
|
||||
f"benchmark:tool_call:{int((now - timedelta(seconds=i)).timestamp() * 1000)}"
|
||||
for i in range(4)
|
||||
]
|
||||
mock_redis.zrevrangebyscore.return_value = mock_keys
|
||||
|
||||
# Different combinations of recommended/used
|
||||
tool_data = [
|
||||
{"was_recommended": "True", "was_actually_used": "True"}, # Good
|
||||
{"was_recommended": "True", "was_actually_used": "True"}, # Good
|
||||
{"was_recommended": "False", "was_actually_used": "True"}, # Missed
|
||||
{"was_recommended": "True", "was_actually_used": "False"}, # Not used
|
||||
]
|
||||
|
||||
async def mock_hgetall(key):
|
||||
idx = mock_keys.index(key)
|
||||
data = tool_data[idx]
|
||||
return {
|
||||
"timestamp": now.isoformat(),
|
||||
"operation": "tool_call",
|
||||
"duration_seconds": 1.0,
|
||||
"success": "True", # Booleans stored as strings in Redis
|
||||
"metadata": "{}",
|
||||
"recommendation_count": None,
|
||||
"confidence": None,
|
||||
"tool_name": "test_tool",
|
||||
"conversation_id": None,
|
||||
"was_recommended": data["was_recommended"], # Already strings
|
||||
"was_actually_used": data["was_actually_used"], # Already strings
|
||||
}
|
||||
|
||||
mock_redis.hgetall.side_effect = mock_hgetall
|
||||
|
||||
accuracy = await store.get_tool_accuracy()
|
||||
|
||||
assert accuracy["total_calls"] == 4
|
||||
assert accuracy["total_used"] == 3
|
||||
assert accuracy["recommended_and_used"] == 2
|
||||
assert accuracy["not_recommended_but_used"] == 1
|
||||
assert accuracy["precision"] == pytest.approx(66.67, rel=0.01)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_tool_accuracy_empty(self, store, mock_redis):
|
||||
"""Test tool accuracy with no data."""
|
||||
mock_redis.zrevrangebyscore.return_value = []
|
||||
|
||||
accuracy = await store.get_tool_accuracy()
|
||||
|
||||
assert accuracy["total_calls"] == 0
|
||||
assert accuracy["precision"] == 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close(self, store, mock_redis):
|
||||
"""Test closing the store."""
|
||||
await store.close()
|
||||
mock_redis.aclose.assert_called_once()
|
||||
|
||||
# Client should be None after close
|
||||
assert store._client is None
|
||||
|
||||
|
||||
class TestGlobalBenchmarkStore:
|
||||
"""Test global benchmark store instance."""
|
||||
|
||||
def test_get_benchmark_store(self):
|
||||
"""Test getting global store instance."""
|
||||
store = get_benchmark_store()
|
||||
assert isinstance(store, BenchmarkStore)
|
||||
|
||||
def test_get_benchmark_store_singleton(self):
|
||||
"""Test store is singleton."""
|
||||
store1 = get_benchmark_store()
|
||||
store2 = get_benchmark_store()
|
||||
assert store1 is store2
|
||||
@@ -3,8 +3,6 @@ Tests for tool call tracking.
|
||||
|
||||
Tests capability extraction and recommendation matching.
|
||||
"""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.tool_tracking import ToolCallTracker
|
||||
@@ -35,15 +33,11 @@ class TestToolCallTracker:
|
||||
recommended_capabilities=["librarian", "biographer"]
|
||||
)
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
await tracker.track_call("delegate_to_librarian", 1.0)
|
||||
|
||||
await tracker.track_call("delegate_to_librarian", 1.0)
|
||||
|
||||
# Should NOT log warning since librarian was recommended
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.was_recommended is True
|
||||
# Should record the call
|
||||
assert "delegate_to_librarian" in tracker.actual_calls
|
||||
assert tracker.actual_calls["delegate_to_librarian"] == [1.0]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_track_call_detects_not_recommended(self):
|
||||
@@ -52,14 +46,12 @@ class TestToolCallTracker:
|
||||
recommended_capabilities=["librarian"]
|
||||
)
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
await tracker.track_call("delegate_to_housekeeper", 1.0)
|
||||
|
||||
await tracker.track_call("delegate_to_housekeeper", 1.0)
|
||||
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.was_recommended is False
|
||||
# Should record the call even though not recommended
|
||||
assert "delegate_to_housekeeper" in tracker.actual_calls
|
||||
summary = tracker.get_summary()
|
||||
assert summary["accuracy"]["not_recommended_but_used"] == 1
|
||||
|
||||
def test_get_summary_with_delegation_tools(self):
|
||||
"""Test summary correctly maps delegation tools to capabilities."""
|
||||
@@ -87,15 +79,9 @@ class TestToolCallTracker:
|
||||
"delegate_to_librarian": [1.0],
|
||||
}
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
await tracker.finalize()
|
||||
|
||||
await tracker.finalize()
|
||||
|
||||
# Should record benchmark for unused biographer
|
||||
assert mock_store.return_value.record.called
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.tool_name == "biographer"
|
||||
assert benchmark.was_recommended is True
|
||||
assert benchmark.was_actually_used is False
|
||||
# Summary should show biographer as recommended but unused
|
||||
summary = tracker.get_summary()
|
||||
assert summary["accuracy"]["recommended_and_used"] == 1 # librarian
|
||||
assert summary["accuracy"]["recommended_but_unused"] == 1 # biographer
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
|
||||
|
||||
#!/bin/bash
|
||||
# Tatlock Server Startup Script
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${GREEN}Starting Tatlock server...${NC}"
|
||||
|
||||
# Check if port 8777 is already in use
|
||||
if lsof -Pi :8777 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
|
||||
echo -e "${RED}Error: Port 8777 is already in use${NC}"
|
||||
echo "Run: lsof -i :8777 to see what's using it"
|
||||
echo "Or run: kill \$(lsof -t -i:8777) to stop it"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Activate virtual environment if not already activated
|
||||
if [ -z "$VIRTUAL_ENV" ]; then
|
||||
if [ -d ".venv" ]; then
|
||||
echo -e "${YELLOW}Activating virtual environment...${NC}"
|
||||
source .venv/bin/activate
|
||||
else
|
||||
echo -e "${RED}Error: Virtual environment not found${NC}"
|
||||
echo "Run: python -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
LOGS_DIR="logs"
|
||||
mkdir -p "$LOGS_DIR"
|
||||
|
||||
# Clear/create log file
|
||||
LOG_FILE="$LOGS_DIR/server.log"
|
||||
> "$LOG_FILE"
|
||||
echo -e "${YELLOW}Logs will be written to: ${LOG_FILE}${NC}"
|
||||
|
||||
# Start the server
|
||||
echo -e "${GREEN}Starting uvicorn server on http://localhost:8777${NC}"
|
||||
echo -e "${YELLOW}Press Ctrl+C to stop the server${NC}"
|
||||
echo ""
|
||||
|
||||
uvicorn src.main:app --reload --host localhost --port 8777 2>&1 | tee "$LOG_FILE"
|
||||
Reference in New Issue
Block a user