Compare commits
+11
-9
@@ -8,17 +8,19 @@ API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
API_PREFIX=/v1
|
||||
|
||||
# Anthropic Configuration (Claude - preferred backend)
|
||||
# Set ANTHROPIC_API_KEY to enable Claude as the default backend
|
||||
# Without an API key, Tatlock uses Ollama exclusively
|
||||
# ANTHROPIC_API_KEY=sk-ant-api03-your-key-here
|
||||
ANTHROPIC_MODEL=claude-sonnet-4-20250514
|
||||
PREFER_CLOUD_BACKEND=true
|
||||
|
||||
# Ollama Configuration (local fallback when Claude unavailable)
|
||||
# Ollama Configuration (local - primary backend)
|
||||
OLLAMA_HOST=http://localhost:11434
|
||||
OLLAMA_DEFAULT_MODEL=mistral-nemo:latest
|
||||
OLLAMA_DEFAULT_MODEL=gemma4:e2b
|
||||
OLLAMA_TIMEOUT=120
|
||||
STEWARD_TIMEOUT=60
|
||||
|
||||
# Anthropic Configuration (Claude - cloud fallback)
|
||||
# Set ANTHROPIC_API_KEY to keep the Claude fallback available: it is used
|
||||
# automatically when Ollama is down, or exclusively when PREFER_CLOUD_BACKEND=true
|
||||
# Without an API key, Tatlock uses Ollama only
|
||||
# ANTHROPIC_API_KEY=sk-ant-api03-your-key-here
|
||||
ANTHROPIC_MODEL=claude-sonnet-5
|
||||
PREFER_CLOUD_BACKEND=false
|
||||
|
||||
# SearXNG Configuration
|
||||
SEARXNG_HOST=http://localhost:8087
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
name: Build and Push
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
push:
|
||||
tags:
|
||||
- 'v[0-9]*'
|
||||
|
||||
jobs:
|
||||
release:
|
||||
@@ -15,7 +16,7 @@ jobs:
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"tag_name": "${{ github.ref_name }}", "name": "Release ${{ github.ref_name }}", "body": "Automated release for ${{ github.ref_name }}"}' \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases"
|
||||
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
+11
-6
@@ -46,27 +46,32 @@ ENV/
|
||||
.ipynb_checkpoints/
|
||||
*.ipynb
|
||||
|
||||
# Testing & Coverage
|
||||
# Caches (pytest, mypy, ruff)
|
||||
.cache/
|
||||
|
||||
# Build output (coverage, logs)
|
||||
build/
|
||||
|
||||
# Legacy cache/output locations (in case tools fall back)
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
.coverage.*
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
|
||||
# Testing
|
||||
.tox/
|
||||
.nox/
|
||||
*.cover
|
||||
.hypothesis/
|
||||
|
||||
# Type checking
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
.pyre/
|
||||
.pytype/
|
||||
|
||||
# Linting
|
||||
.ruff_cache/
|
||||
|
||||
# Logs
|
||||
logs/*
|
||||
!logs/traces/
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
This document contains instructions and documentation references for AI assistants working with this codebase.
|
||||
|
||||
> **📖 Important**: Before working on this project, read [PHILOSOPHY.md](PHILOSOPHY.md) to understand the system vision, architectural patterns, and design goals. All development should work towards realizing those patterns.
|
||||
> **📖 Important**: Before working on this project, read [docs/philosophy.md](docs/philosophy.md) to understand the system vision, architectural patterns, and design goals. All development should work towards realizing those patterns.
|
||||
# AGENTS.md
|
||||
|
||||
> **Start every session by reading this file.**
|
||||
|
||||
+71
-1
@@ -7,6 +7,74 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.3.0] - 2026-07-13
|
||||
|
||||
### Changed
|
||||
|
||||
- **Local-first backend (claudification rollback)** - Ollama/gemma4 is now the primary backend; Claude remains as fallback. `PREFER_CLOUD_BACKEND` defaults to `false`, Claude is used automatically when the Ollama startup health check fails, and the Steward retries mid-request failures on the other backend in both directions
|
||||
- **Default Claude model `claude-sonnet-5`** - `claude-sonnet-4-20250514` was retired by Anthropic on 2026-06-15 and would 404, leaving the fallback dead
|
||||
- **Dedicated orchestration prompt** - `orchestrate_tool_calls()` now uses a terse tool-execution prompt (`TATLOCK_ORCHESTRATION_PROMPT`); the butler persona prompt suppressed gemma4 tool calling (the model reasoned about the calculator, then answered from memory with wrong arithmetic). Synthesis keeps the persona prompt, so user-visible voice is unchanged
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Startup crash with broken anthropic package** - Anthropic SDK imports in the model selector are now lazy, so an incompatible `anthropic` install degrades to Ollama-only operation instead of crashing the app at import time (root cause of the production outage since April)
|
||||
- **Claude Sonnet 5 rejects sampling parameters** - removed `temperature` from the Steward's direct Claude call and made the Housekeeper's temperature setting backend-conditional via `get_sampling_settings()`
|
||||
- **Pin `anthropic>=0.77,<1.0`** - the April image resolved an anthropic version incompatible with pydantic-ai 1.27
|
||||
- **Steward timeout configurable** - new `STEWARD_TIMEOUT` (default 60s) replaces the hardcoded 30s, which gemma4 chronically exceeded (~35s warm analysis), causing every request to fail or fall back
|
||||
|
||||
### Added
|
||||
|
||||
- **Ollama startup health check** - verifies the server is reachable and `OLLAMA_DEFAULT_MODEL` is pulled; feeds backend resolution and `get_model_info()`
|
||||
- **Contract tests** (`tests/contracts/`, `make test-contracts`) - wire-level tests that send the raw requests the code sends to Ollama (native + OpenAI-compat tool calling), Anthropic (including the pinned temperature-rejection contract), Qdrant, SearXNG, library-desk, and Redis; unreachable services skip, wrong response shapes fail
|
||||
- **Backend resolution unit tests** (`tests/anthropic/`)
|
||||
|
||||
## [2.2.0] - 2026-04-04
|
||||
|
||||
### Changed
|
||||
|
||||
- **Switch default Ollama model to gemma4:e2b** - Replaces mistral-nemo as the local LLM backend; gemma4:e2b has native function calling support, faster tool calling (2-4s vs 15-20s), better parameter accuracy on word problems, and uses less VRAM (8GB vs 9.2GB)
|
||||
|
||||
### Added
|
||||
|
||||
- **Tool calling benchmark script** (`scripts/benchmark_tool_calling.py`) - Compares tool calling accuracy and latency across Ollama models via the Tatlock API
|
||||
|
||||
## [2.1.0] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Streaming SSE compatibility with Open WebUI** - Switch from `exclude_none=True` to `exclude_unset=True` for SSE chunk serialization; `exclude_none` was too aggressive — it stripped `finish_reason: null` from intermediate chunks (which OpenAI includes), while `exclude_unset` correctly omits only fields never passed to the constructor (like `reasoning_content` on content-only chunks) while preserving explicitly-set `finish_reason: null`
|
||||
|
||||
### Changed
|
||||
|
||||
- **Project structure consolidation** - Moved documentation to `docs/`, consolidated all config into `pyproject.toml`, replaced `wakeup.sh`/`pytest.ini`/`requirements*.txt` with `Makefile` + `pyproject.toml`
|
||||
- **CI test gate** - Unit tests now gate release and build jobs in Gitea Actions workflow
|
||||
- **Build output organization** - Tool caches in `.cache/`, generated output (coverage, logs) in `build/`
|
||||
|
||||
## [2.0.5] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Streaming JSON compatibility** - Exclude null fields from streaming chunks using `exclude_none=True`; OpenAI's API omits null fields entirely, and including them (e.g., `content: null`, `reasoning_content: null`) caused parsing issues in Open WebUI
|
||||
|
||||
## [2.0.4] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Open WebUI streaming compatibility** - Replaced `sse_starlette` `EventSourceResponse` with plain `StreamingResponse` for chat completions; `sse_starlette` added `\r\n` line endings and extra SSE fields that Open WebUI couldn't parse
|
||||
|
||||
## [2.0.3] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Steward analysis leaking into responses** - Removed internal routing analysis (`DELEGATE: tatlock_core...`) from user-visible reasoning in both streaming and non-streaming paths
|
||||
|
||||
## [2.0.2] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
|
||||
- **tool_choice format incompatibility** - Removed `extra_body` tool_choice hack for Claude backend; PydanticAI handles tool_choice natively for Anthropic, preventing infinite tool call loops
|
||||
- **CI trigger** - Changed workflow trigger from `release:published` to `push:tags:v[0-9]*`
|
||||
|
||||
## [2.0.1] - 2026-02-05
|
||||
|
||||
### Fixed
|
||||
@@ -896,7 +964,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- CORS middleware
|
||||
- Exception handlers (OpenAI-compatible error format)
|
||||
|
||||
[Unreleased]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.0.0...main
|
||||
[Unreleased]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.1.0...main
|
||||
[2.1.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.0.5...v2.1.0
|
||||
[2.0.5]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v2.0.0...v2.0.5
|
||||
[2.0.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.11.0...v2.0.0
|
||||
[1.11.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.10.0...v1.11.0
|
||||
[1.10.0]: https://git.schweitz.net/jpmschweitzer/tatlock/compare/v1.9.0...v1.10.0
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# CLAUDE.md
|
||||
|
||||
Claude Code-specific notes for this project. For general development instructions, architecture, coding standards, and deployment — see [AGENTS.md](AGENTS.md).
|
||||
|
||||
## Setup & Commands
|
||||
|
||||
```bash
|
||||
make setup # Create venv and install all dependencies
|
||||
make test # Unit tests (no external services)
|
||||
make test-integration # Integration tests (needs Claude/Ollama)
|
||||
make test-contracts # Wire-level contract tests against live service boundaries
|
||||
make run # Start dev server on port 8777
|
||||
make lint # Ruff linter + formatter check
|
||||
make typecheck # Mypy
|
||||
make clean # Remove caches and build artifacts
|
||||
```
|
||||
|
||||
Dependencies are in `pyproject.toml` (`[project.dependencies]` and `[project.optional-dependencies.dev]`).
|
||||
|
||||
## Critical Gotchas
|
||||
|
||||
**ASGITransport does NOT trigger FastAPI lifespan events.** The session-scoped `_initialize_app` fixture in `tests/conftest.py` calls `initialize_application()` explicitly via `asyncio.run()`. Without this, the Ollama/Claude health checks never run: `_ollama_available` stays `None` (treated as available, so requests go to Ollama) and `_claude_available` stays `None` (treated as unavailable, so the Claude fallback never engages).
|
||||
|
||||
**AsyncIO scope mismatch.** `asyncio_default_fixture_loop_scope = function` is set in `pyproject.toml`. Session-scoped async fixtures cause `ScopeMismatch` errors. The fix is to use a sync fixture with `asyncio.run()` for session-scoped initialization.
|
||||
|
||||
**The butler persona prompt suppresses local-model tool calling.** With `TATLOCK_SYSTEM_PROMPT` attached, gemma4 reasons about calling the calculator, then answers from memory with wrong arithmetic (a different wrong product each run). `orchestrate_tool_calls()` therefore uses the terse `TATLOCK_ORCHESTRATION_PROMPT`; the persona is applied in `synthesize_from_results()`. Do not reattach the persona prompt to a tool-phase agent. `tool_choice: "required"` via extra_body does NOT force Ollama to call tools — it is advisory at best.
|
||||
|
||||
**Claude Sonnet 5+ rejects sampling parameters.** `temperature`/`top_p`/`top_k` return a 400. Use `get_sampling_settings()` from the model selector instead of passing `ModelSettings(temperature=...)` directly to agents that can run on the Claude fallback. The contract test suite pins this (`make test-contracts`).
|
||||
|
||||
**Integration test timeouts.** Set to 120s to match `OLLAMA_TIMEOUT` config (300s for the pure-Ollama fallback test, which cannot be rescued by Claude). The full local Steward → orchestrate → synthesize flow takes ~2 minutes on gemma4. Steward analysis alone needs ~35s warm — `STEWARD_TIMEOUT` defaults to 60s.
|
||||
|
||||
**`get_benchmark_store` does not exist.** The benchmarking module (`src/core/benchmarks.py`) was never implemented. `scripts/benchmark_analysis.py` also references it and is broken. Do not add mocks for it in tests.
|
||||
|
||||
**Steward tests need household registry.** Use `register_household_members()` (sync) in fixtures, not `initialize_application()` (async). The steward extracts capabilities from the registry.
|
||||
+2
-2
@@ -5,8 +5,8 @@ WORKDIR /app
|
||||
RUN apt-get update && apt-get install -y curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt pyproject.toml ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY pyproject.toml ./
|
||||
RUN pip install --no-cache-dir .
|
||||
|
||||
COPY src/ ./src/
|
||||
|
||||
|
||||
@@ -1,920 +0,0 @@
|
||||
# Tatlock Implementation Roadmap
|
||||
|
||||
> **Reference**: See [PHILOSOPHY.md](PHILOSOPHY.md) for the target architecture and vision
|
||||
|
||||
This document outlines the phased implementation plan to transform the current OpenAI-compatible API into the full Tatlock household butler system.
|
||||
|
||||
## Current State (v1.2.0 - Phase F Complete)
|
||||
|
||||
**What we have**:
|
||||
- ✅ **The Orchestrator** - FastAPI infrastructure layer
|
||||
- OpenAI-compatible API endpoints (Responses API + Chat Completions)
|
||||
- Streaming coordination and conversation management
|
||||
- Response format with reasoning support
|
||||
- Test infrastructure (~400 tests)
|
||||
- ✅ **Two-Tier Architecture**
|
||||
- The Steward analyzes requests and recommends capabilities
|
||||
- Tatlock coordinates execution with scoped tools
|
||||
- Real-time streaming of analysis and reasoning
|
||||
- ✅ **Household Staff**
|
||||
- **Tatlock** (Butler): Primary interface with witty personality
|
||||
- **The Steward**: Request analysis and capability recommendation
|
||||
- **The Librarian**: Research via library-desk HybridRAG + wiki
|
||||
- **The Biographer**: User memory, profiles, preferences, semantic recall
|
||||
- ✅ **Core Tools**
|
||||
- Calculator, Date/Time toolkit, Web search (SearXNG)
|
||||
- ✅ **Memory System**
|
||||
- Direct access layer (memory_service) for fast lookups
|
||||
- Vector storage (Qdrant) for semantic recall
|
||||
- Session cache (Redis) with 24h TTL
|
||||
- Multi-tenancy via ContextVar
|
||||
- ✅ Mock agent (lorem-tester for testing)
|
||||
|
||||
**What we need**:
|
||||
- More household staff (Developer, Secretary, Handyman, Housekeeper)
|
||||
- MCP (Model Context Protocol) integration
|
||||
- Dynamic model switching for specialized tasks
|
||||
- Full multi-tenant database (PostgreSQL)
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Real LLM Integration - PydanticAI + Tools
|
||||
|
||||
**Goal**: Connect to actual language models and establish the base plumbing
|
||||
|
||||
**Note**: Ollama is an external service dependency (already running separately)
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **PydanticAI Integration** ✅
|
||||
- PydanticAI → Ollama connection ✅
|
||||
- Agent creation patterns ✅
|
||||
- Streaming response handling ✅
|
||||
- Error handling and retries ✅
|
||||
|
||||
2. **Convert Tatlock Agent** ✅
|
||||
- Convert Tatlock agent from mock to PydanticAI ✅
|
||||
- British butler personality prompt ✅
|
||||
- Research-oriented mindset ✅
|
||||
- Streaming to reasoning output ✅
|
||||
- Tool calling framework setup ✅
|
||||
|
||||
3. **Permanent Tools** ✅
|
||||
- Calculator: Safe mathematical expression evaluation ✅
|
||||
- Date/Time toolkit: Current time, relative dates, time differences ✅
|
||||
- Web search: SearXNG integration (external service) ✅
|
||||
- Tool registration with PydanticAI ✅
|
||||
|
||||
4. **Testing Infrastructure** ✅
|
||||
- Integration tests with real LLM ✅
|
||||
- Tool functionality tests ✅
|
||||
- Response quality validation ✅
|
||||
- 131 tests, 81.78% coverage ✅
|
||||
|
||||
### Success Criteria
|
||||
- [x] **PydanticAI agents can call Ollama** (mistral-nemo:latest)
|
||||
- [x] **Streaming works end-to-end**
|
||||
- [x] **Tool calling framework functional**
|
||||
- [x] **Permanent tools working** (calculator, date/time, search)
|
||||
- [x] **Tests pass with real LLM**
|
||||
- [ ] Can switch models dynamically (e.g., Codestral for code)
|
||||
|
||||
### Status
|
||||
**✅ MOSTLY COMPLETE** - Tatlock agent functional with permanent tools
|
||||
|
||||
### Remaining Work
|
||||
- Dynamic model switching for specialized tasks (e.g., Codestral for coding)
|
||||
|
||||
### Why First?
|
||||
Without real LLM integration, we can't meaningfully implement the Steward/Butler pattern. Everything else depends on having actual AI agents working.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Orchestration Layer - The Steward
|
||||
|
||||
**Goal**: Implement the first-tier LLM call for tool/agent selection
|
||||
|
||||
**Purpose**: The Steward performs crucial preparatory work before Tatlock engages with a request. By analyzing incoming requests and determining which tools, services, and household staff members will be needed, the Steward creates a curated recommendation that streamlines Tatlock's work and prevents cognitive overload.
|
||||
|
||||
### Core Architecture
|
||||
|
||||
The Steward operates as the first tier in the two-tier request flow:
|
||||
|
||||
```
|
||||
User Request → Orchestrator → Steward Analysis → Recommendations → Tatlock (with scoped tools/agents)
|
||||
```
|
||||
|
||||
**Key Principle**: The Steward narrows the scope to only relevant capabilities, making Tatlock's decision-making cleaner and more focused.
|
||||
|
||||
### Deliverables
|
||||
|
||||
#### 1. Tool & Agent Registry System
|
||||
|
||||
**Purpose**: Centralized catalog of all available capabilities for the Steward to recommend
|
||||
|
||||
**Implementation Details**:
|
||||
- **Registry Module** (`src/core/registry.py`)
|
||||
- Tool registration decorator pattern
|
||||
- Agent registration with capability metadata
|
||||
- Category-based organization (computation, information, automation, communication)
|
||||
- Dynamic tool/agent discovery and loading
|
||||
|
||||
- **Tool Metadata Schema**
|
||||
```python
|
||||
{
|
||||
"name": "calculator",
|
||||
"category": "computation",
|
||||
"description": "Safe mathematical expression evaluation",
|
||||
"capabilities": ["arithmetic", "algebra", "trigonometry"],
|
||||
"cost": "low", # computational cost indicator
|
||||
"requires_network": false
|
||||
}
|
||||
```
|
||||
|
||||
- **Agent Metadata Schema**
|
||||
```python
|
||||
{
|
||||
"name": "developer",
|
||||
"role": "The Developer",
|
||||
"category": "technical",
|
||||
"description": "Software development assistance",
|
||||
"domains": ["code_generation", "debugging", "architecture"],
|
||||
"specialized_model": "codestral", # optional
|
||||
"cost": "high"
|
||||
}
|
||||
```
|
||||
|
||||
- **Registry API**
|
||||
- `get_all_tools()` - List all available tools
|
||||
- `get_all_agents()` - List all expert agents
|
||||
- `get_by_category(category)` - Filter by category
|
||||
- `search_by_capability(query)` - Semantic search (future: vector search)
|
||||
|
||||
**Testing**:
|
||||
- Unit tests for registration and retrieval
|
||||
- Test dynamic loading of new tools/agents
|
||||
- Validate metadata schemas
|
||||
|
||||
#### 2. Steward PydanticAI Agent
|
||||
|
||||
**Purpose**: First-tier LLM that analyzes requests and recommends relevant tools/agents
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Agent Module** (`src/agents/steward.py`)
|
||||
```python
|
||||
from pydantic_ai import Agent, RunContext
|
||||
from pydantic import BaseModel
|
||||
|
||||
class StewardRecommendation(BaseModel):
|
||||
"""Structured output from Steward analysis"""
|
||||
recommended_tools: list[str]
|
||||
recommended_agents: list[str]
|
||||
reasoning: str
|
||||
estimated_complexity: str # "simple", "moderate", "complex"
|
||||
requires_multi_step: bool
|
||||
|
||||
steward = Agent(
|
||||
'ollama:mistral-nemo', # Same base model as Tatlock
|
||||
result_type=StewardRecommendation,
|
||||
system_prompt="""..."""
|
||||
)
|
||||
```
|
||||
|
||||
- **System Prompt Engineering**
|
||||
- Role: Estate steward responsible for efficient household coordination
|
||||
- Task: Analyze requests to determine needed resources
|
||||
- Output: Structured recommendations with reasoning
|
||||
- Constraints: Be conservative (recommend only truly relevant capabilities)
|
||||
- Context: Full registry of available tools and agents
|
||||
|
||||
- **Steward Tools**
|
||||
```python
|
||||
@steward.tool
|
||||
def get_available_capabilities(ctx: RunContext) -> dict:
|
||||
"""Get catalog of all available tools and agents."""
|
||||
return {
|
||||
"tools": registry.get_all_tools(),
|
||||
"agents": registry.get_all_agents()
|
||||
}
|
||||
```
|
||||
|
||||
- **Request Analysis Flow**
|
||||
1. Receive user request
|
||||
2. Query capability registry via tool
|
||||
3. Analyze request for required capabilities
|
||||
4. Generate structured recommendation
|
||||
5. Format as note to Tatlock
|
||||
|
||||
**Testing**:
|
||||
- Test various request types (simple, complex, multi-domain)
|
||||
- Verify recommendations are relevant and not over-inclusive
|
||||
- Test structured output parsing
|
||||
- Validate reasoning quality
|
||||
|
||||
#### 3. Request Preprocessing Pipeline
|
||||
|
||||
**Purpose**: Integration layer that routes requests through Steward before Tatlock
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Preprocessing Module** (`src/core/preprocessing.py`)
|
||||
```python
|
||||
async def preprocess_request(user_request: str) -> EnrichedRequest:
|
||||
"""
|
||||
1. Call Steward for analysis
|
||||
2. Get recommendations
|
||||
3. Enrich original request
|
||||
4. Return scoped context for Tatlock
|
||||
"""
|
||||
# Get Steward analysis
|
||||
steward_result = await steward.run(user_request)
|
||||
recommendations = steward_result.data
|
||||
|
||||
# Create note to Tatlock
|
||||
steward_note = format_steward_note(recommendations)
|
||||
|
||||
# Build scoped tool/agent list
|
||||
scoped_tools = get_scoped_tools(recommendations.recommended_tools)
|
||||
scoped_agents = get_scoped_agents(recommendations.recommended_agents)
|
||||
|
||||
return EnrichedRequest(
|
||||
original_request=user_request,
|
||||
steward_note=steward_note,
|
||||
available_tools=scoped_tools,
|
||||
available_agents=scoped_agents,
|
||||
metadata=recommendations
|
||||
)
|
||||
```
|
||||
|
||||
- **Note Formatting**
|
||||
```
|
||||
=== Internal Note from the Steward ===
|
||||
|
||||
Request Analysis:
|
||||
{steward reasoning}
|
||||
|
||||
Recommended Tools:
|
||||
- calculator: For mathematical computations
|
||||
- web_search: To find current information
|
||||
|
||||
Recommended Household Staff:
|
||||
- The Developer: For code generation assistance
|
||||
|
||||
Estimated Complexity: moderate
|
||||
===================================
|
||||
|
||||
[Original User Request]
|
||||
```
|
||||
|
||||
- **Orchestrator Integration**
|
||||
- Modify `src/responses/service.py` to call preprocessing
|
||||
- Prepend Steward note to request before sending to Tatlock
|
||||
- Limit Tatlock's tool access to recommended tools only
|
||||
- Stream Steward's reasoning to output
|
||||
|
||||
**Testing**:
|
||||
- Integration tests for full preprocessing flow
|
||||
- Test request enrichment format
|
||||
- Verify tool scoping works correctly
|
||||
- Test streaming of Steward reasoning
|
||||
|
||||
#### 4. Real-Time Transparency
|
||||
|
||||
**Purpose**: Stream Steward's analysis to user's reasoning output
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Streaming Integration** (`src/responses/streaming.py`)
|
||||
- Add Steward analysis phase to stream
|
||||
- Format as reasoning item
|
||||
- Include recommendation summary
|
||||
|
||||
- **Example Output to User**:
|
||||
```
|
||||
[Reasoning]
|
||||
Consulting the Steward for resource planning...
|
||||
|
||||
The Steward's Analysis:
|
||||
- Request requires mathematical computation
|
||||
- Need to verify current information via web search
|
||||
- May benefit from Developer's code expertise
|
||||
|
||||
Recommended: calculator, web_search, The Developer
|
||||
|
||||
Proceeding with scoped resources...
|
||||
```
|
||||
|
||||
**Testing**:
|
||||
- Test streaming of Steward analysis
|
||||
- Verify formatting in Open WebUI
|
||||
- Test error handling if Steward fails
|
||||
|
||||
#### 5. Model Efficiency Optimization
|
||||
|
||||
**Purpose**: Ensure the base model stays loaded in VRAM
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
- **Shared Model Configuration**
|
||||
- Both Steward and Tatlock use `ollama:mistral-nemo` by default
|
||||
- Sequential calls (Steward → Tatlock) keep model hot
|
||||
- No reload delays between tiers
|
||||
|
||||
- **Performance Monitoring**
|
||||
- Log response times for Steward calls
|
||||
- Track total request latency (Steward + Tatlock)
|
||||
- Identify optimization opportunities
|
||||
|
||||
**Testing**:
|
||||
- Benchmark Steward → Tatlock call latency
|
||||
- Verify model stays loaded between calls
|
||||
- Test performance under load
|
||||
|
||||
### Implementation Strategy
|
||||
|
||||
#### Week 1-2: Foundation
|
||||
- [ ] Design and implement registry system
|
||||
- [ ] Create tool/agent metadata schemas
|
||||
- [ ] Build registry API with tests
|
||||
- [ ] Migrate existing tools to registry
|
||||
|
||||
#### Week 3-4: Steward Agent
|
||||
- [ ] Create Steward PydanticAI agent
|
||||
- [ ] Engineer system prompt for analysis
|
||||
- [ ] Implement structured recommendation output
|
||||
- [ ] Add registry query tool
|
||||
- [ ] Test with various request types
|
||||
|
||||
#### Week 5-6: Integration
|
||||
- [ ] Build request preprocessing pipeline
|
||||
- [ ] Implement note formatting
|
||||
- [ ] Integrate with Orchestrator
|
||||
- [ ] Add streaming transparency
|
||||
- [ ] Tool scoping for Tatlock
|
||||
|
||||
#### Week 7: Testing & Refinement
|
||||
- [ ] End-to-end integration tests
|
||||
- [ ] Performance optimization
|
||||
- [ ] Prompt refinement based on results
|
||||
- [ ] Documentation and examples
|
||||
|
||||
### Success Criteria
|
||||
|
||||
- [x] **Steward analyzes incoming requests** using PydanticAI agent
|
||||
- [x] **Produces structured recommendations** (tools, agents, reasoning)
|
||||
- [x] **Recommendations formatted as prepended note** to Tatlock
|
||||
- [x] **Tool registry is queryable and extensible** via clean API
|
||||
- [x] **Steward output visible in reasoning stream** for transparency
|
||||
- [x] **Only recommended tools available** to Tatlock (scoped context)
|
||||
- [x] **Base model stays loaded** between Steward and Tatlock calls
|
||||
- [x] **Recommendations are accurate** (not over/under-inclusive)
|
||||
- [x] **Integration tests pass** for full Steward → Tatlock flow
|
||||
|
||||
### Status
|
||||
**✅ COMPLETE** (v0.2.5)
|
||||
|
||||
### Performance Targets
|
||||
|
||||
- **Steward Analysis Time**: < 2 seconds for typical requests
|
||||
- **Total Added Latency**: < 3 seconds including streaming
|
||||
- **Recommendation Accuracy**: > 90% relevance (manual evaluation)
|
||||
- **Model Reload Delay**: 0 seconds (model stays hot)
|
||||
|
||||
### Risk Mitigation
|
||||
|
||||
**Risk**: Steward recommendations too broad (defeats purpose)
|
||||
- Mitigation: Conservative prompt engineering, test with diverse requests, iterate
|
||||
|
||||
**Risk**: Added latency unacceptable to users
|
||||
- Mitigation: Stream Steward reasoning for transparency, optimize prompt, parallel processing where possible
|
||||
|
||||
**Risk**: Tool registry becomes unwieldy
|
||||
- Mitigation: Good categorization, semantic search (future), regular pruning
|
||||
|
||||
**Risk**: Steward and Tatlock models compete for VRAM
|
||||
- Mitigation: Use same base model, sequential calls, monitor memory
|
||||
|
||||
### Future Enhancements (Post-Phase 2)
|
||||
|
||||
- **Semantic Search**: Vector-based capability search instead of metadata lookup
|
||||
- **Learning from Usage**: Track which recommendations work well, adjust over time
|
||||
- **Confidence Scores**: Steward provides confidence for each recommendation
|
||||
- **Request Classification**: Cache classifications for similar requests
|
||||
- **Multi-Model Support**: Allow Steward to recommend specialized models for specific tasks
|
||||
|
||||
### Estimated Effort
|
||||
|
||||
**7-8 weeks** - Core intelligence routing with comprehensive implementation
|
||||
|
||||
### Why Second?
|
||||
|
||||
The Steward is the foundation of the household architecture. Without it, we'd need to expose all tools/agents to Tatlock, creating cognitive overload and poor decision-making. The Steward enables the focused expertise pattern that makes the whole system work.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: The Butler - Tatlock Agent
|
||||
|
||||
**Goal**: Implement the second-tier coordinator with personality within the existing Orchestrator infrastructure
|
||||
|
||||
**Context**: The Orchestrator (FastAPI infrastructure) already exists. This phase implements the real Tatlock PydanticAI agent to replace the current mock agent.
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Butler Agent (Tatlock)**
|
||||
- PydanticAI agent implementation within Orchestrator
|
||||
- Personality prompt engineering (witty British butler)
|
||||
- Tool calling framework
|
||||
- Multi-agent coordination logic
|
||||
|
||||
2. **Scoped Tool Access**
|
||||
- Filter tools based on Steward recommendations
|
||||
- Dynamic tool loading for Butler context
|
||||
- Tool execution framework
|
||||
- Result aggregation
|
||||
|
||||
3. **Real-Time Reasoning Output**
|
||||
- Stream all Butler activities to reasoning output
|
||||
- Tool call progress indicators
|
||||
- Expert agent consultation messages
|
||||
- Wait time transparency
|
||||
|
||||
### Success Criteria
|
||||
- [x] Tatlock receives enriched requests (user + Steward notes)
|
||||
- [x] Only recommended tools are available
|
||||
- [x] Tatlock coordinates multiple tool calls
|
||||
- [x] All actions streamed to reasoning output
|
||||
- [x] Responses have consistent personality
|
||||
- [x] Synthesizes multi-source results coherently
|
||||
|
||||
### Status
|
||||
**✅ COMPLETE** (v1.1.0)
|
||||
|
||||
### Estimated Effort
|
||||
**4-5 weeks** - Complex coordination logic
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Expert Household Staff - Core Agents
|
||||
|
||||
**Goal**: Implement the initial set of domain-specific expert agents
|
||||
|
||||
### Priority Expert Agents
|
||||
|
||||
1. **The Librarian** (Research & Knowledge Management) ✅ **COMPLETE** (v1.1.0)
|
||||
- Research assistance via library-desk HybridRAG
|
||||
- Wiki page management (search, create, update)
|
||||
- Semantic vector search
|
||||
- Knowledge graph queries
|
||||
- Dossier browsing
|
||||
|
||||
2. **The Biographer** (User Memory) ✅ **COMPLETE** (v1.2.0)
|
||||
- User profile management (name, location, timezone)
|
||||
- Preference storage (units, theme)
|
||||
- Semantic memory recall ("What car do I drive?")
|
||||
- Fact storage from conversations
|
||||
- Session context caching
|
||||
|
||||
3. **The Developer** (Software Development) 🔜 **Planned**
|
||||
- Code generation assistance
|
||||
- Debugging support
|
||||
- Documentation generation
|
||||
- Architecture guidance
|
||||
- *Rationale: Directly supports building the system itself*
|
||||
|
||||
4. **The Handyman** (System Maintenance) 🔜 **Planned**
|
||||
- System status queries
|
||||
- Log analysis
|
||||
- Basic troubleshooting
|
||||
- Infrastructure monitoring
|
||||
|
||||
5. **The Secretary** (Scheduling & Organization) 🔜 **Planned**
|
||||
- Calendar integration
|
||||
- Task management
|
||||
- Reminder system
|
||||
- Schedule conflict detection
|
||||
|
||||
6. **The Housekeeper** (Home Automation) 🔜 **Planned**
|
||||
- Home Assistant integration
|
||||
- Device control interface
|
||||
- Status queries
|
||||
- Automation triggers
|
||||
|
||||
### Each Agent Includes
|
||||
- Specialized prompt and personality
|
||||
- Domain-specific tools
|
||||
- MCP integration points (where applicable)
|
||||
- Integration with Butler orchestration
|
||||
|
||||
### Success Criteria
|
||||
- [x] Each agent implemented as separate module
|
||||
- [x] Agents callable via tool framework
|
||||
- [x] Agents use specialized prompts
|
||||
- [x] Results integrate cleanly with Butler
|
||||
- [ ] Can invoke specialized models (e.g., Codestral for Developer)
|
||||
|
||||
### Status
|
||||
**🔶 PARTIAL** - Librarian and Biographer complete, others planned
|
||||
|
||||
### Estimated Effort
|
||||
**6-8 weeks** - Parallel development possible
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: Persistence Layer - Database & Multi-Tenancy
|
||||
|
||||
**Goal**: Add persistent storage and multi-user support when needed
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **PostgreSQL Integration**
|
||||
- Docker compose configuration for PostgreSQL
|
||||
- Database schema design with tenant isolation
|
||||
- Alembic migrations setup
|
||||
- SQLAlchemy models
|
||||
|
||||
2. **Multi-Tenant Architecture**
|
||||
- Tenant identification middleware
|
||||
- Tenant-scoped database sessions
|
||||
- User authentication system (basic)
|
||||
- Per-tenant data isolation
|
||||
|
||||
3. **Core Data Models**
|
||||
- Users and tenants
|
||||
- Conversations and messages (migrate from in-memory)
|
||||
- Agent interactions log
|
||||
- System configuration and preferences
|
||||
|
||||
4. **Migration Strategy**
|
||||
- Gradual migration from in-memory to database
|
||||
- Backward compatibility during transition
|
||||
- Data export/import utilities
|
||||
|
||||
### Success Criteria
|
||||
- [ ] PostgreSQL container running
|
||||
- [ ] Multiple users can authenticate separately
|
||||
- [ ] Each user sees only their own data
|
||||
- [ ] Conversations persist across restarts
|
||||
- [ ] Database migrations work correctly
|
||||
- [ ] Tests verify tenant isolation
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Data layer foundation
|
||||
|
||||
### Why Later?
|
||||
The core orchestration (Steward → Butler → Experts) can work entirely with in-memory state. We only need database persistence when we want conversations to survive restarts and multiple users to have isolated experiences.
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Extended Services Integration
|
||||
|
||||
**Goal**: Connect to additional supporting services
|
||||
|
||||
### Services to Integrate
|
||||
|
||||
1. **Redis (Memory & Caching)** ✅ **COMPLETE** (v1.2.0)
|
||||
- Benchmark storage (db=1)
|
||||
- Memory cache for sessions (db=2)
|
||||
- 24h TTL for session context
|
||||
- Recent entities tracking
|
||||
|
||||
2. **Qdrant (Vector Storage)** ✅ **COMPLETE** (v1.2.0)
|
||||
- Per-user memory collections
|
||||
- 768-dim nomic-embed-text vectors
|
||||
- Semantic search for recall
|
||||
- Type-based filtering
|
||||
|
||||
3. **SearxNG (Web Search)** ✅ **COMPLETE** (v0.2.0)
|
||||
- Search tool integration
|
||||
- Result processing
|
||||
- Privacy-preserving queries
|
||||
|
||||
4. **library-desk (Research API)** ✅ **COMPLETE** (v1.1.0)
|
||||
- HybridRAG search
|
||||
- Wiki management
|
||||
- Knowledge graph queries
|
||||
|
||||
### Success Criteria
|
||||
- [x] Services communicate correctly
|
||||
- [x] Tatlock can invoke web search
|
||||
- [x] Redis used for session data
|
||||
- [x] Qdrant stores user memories
|
||||
- [x] Ollama serves the base model
|
||||
|
||||
### Status
|
||||
**✅ COMPLETE** - All core services integrated
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Infrastructure setup
|
||||
|
||||
---
|
||||
|
||||
## Phase 7: MCP (Model Context Protocol) Integration
|
||||
|
||||
**Goal**: Enable rich tool integrations via MCP
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **MCP Server Framework**
|
||||
- MCP server implementation
|
||||
- Tool registration via MCP
|
||||
- Schema validation
|
||||
- Error handling
|
||||
|
||||
2. **MCP Client in Agents**
|
||||
- PydanticAI MCP integration
|
||||
- Tool discovery from MCP servers
|
||||
- Dynamic tool loading
|
||||
- Result processing
|
||||
|
||||
3. **Initial MCP Tools**
|
||||
- File system operations
|
||||
- Database queries
|
||||
- API integrations
|
||||
- System commands
|
||||
|
||||
### Success Criteria
|
||||
- [ ] MCP server running
|
||||
- [ ] Tools exposed via MCP protocol
|
||||
- [ ] Agents can discover and use MCP tools
|
||||
- [ ] New tools addable without code changes
|
||||
- [ ] MCP tools visible in Steward recommendations
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Standards-based integration
|
||||
|
||||
---
|
||||
|
||||
## Phase 8: Advanced Memory & Context
|
||||
|
||||
**Goal**: Implement sophisticated memory and context management
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Long-Term Memory** ✅ **COMPLETE** (v1.2.0 - Phase F)
|
||||
- Memory service for direct key-based access
|
||||
- Qdrant vector storage for semantic recall
|
||||
- Embedding via nomic-embed-text
|
||||
- The Biographer agent for memory management
|
||||
|
||||
2. **Session Memory** ✅ **COMPLETE** (v1.2.0)
|
||||
- Redis session cache with 24h TTL
|
||||
- Recent entities tracking
|
||||
- Conversation context preservation
|
||||
- Multi-tenancy via ContextVar
|
||||
|
||||
3. **Steward Integration** ✅ **COMPLETE** (v1.2.0)
|
||||
- Memory pre-fetch during request analysis
|
||||
- Profile/preferences included in context
|
||||
- Keyword-based context determination
|
||||
|
||||
4. **Context Management** 🔜 **Future**
|
||||
- Smart context window trimming
|
||||
- Conversation branching
|
||||
- Topic tracking
|
||||
- Memory retrieval integration
|
||||
|
||||
5. **Personalization** 🔜 **Future**
|
||||
- User preference learning
|
||||
- Interaction pattern analysis
|
||||
- Adaptive responses
|
||||
- Custom agent personalities per user
|
||||
|
||||
### Success Criteria
|
||||
- [x] User facts stored in Qdrant with semantic search
|
||||
- [x] Profile and preferences accessible via memory_service
|
||||
- [x] Session context cached in Redis
|
||||
- [x] User preferences affect responses (via Steward pre-fetch)
|
||||
- [ ] Conversations automatically embedded to Qdrant
|
||||
- [ ] Memory improves over time (learning from interactions)
|
||||
|
||||
### Status
|
||||
**🔶 PARTIAL** - Core memory system complete, advanced features planned
|
||||
|
||||
### Estimated Effort
|
||||
**4-5 weeks** - AI/ML heavy (remaining work)
|
||||
|
||||
---
|
||||
|
||||
## Phase 9: Extended Household Staff
|
||||
|
||||
**Goal**: Add specialized agents for additional domains
|
||||
|
||||
### Future Agents
|
||||
|
||||
1. **The Librarian** (Knowledge Management)
|
||||
- Personal documentation indexing
|
||||
- Research assistance
|
||||
- Knowledge base queries
|
||||
- Reference management
|
||||
|
||||
2. **The Accountant** (Financial Tracking)
|
||||
- Expense tracking
|
||||
- Budget monitoring
|
||||
- Financial reports
|
||||
- Transaction categorization
|
||||
|
||||
3. **The Chef** (Meal Planning)
|
||||
- Recipe management
|
||||
- Meal planning
|
||||
- Nutrition tracking
|
||||
- Grocery lists
|
||||
|
||||
4. **Others as Needed**
|
||||
- Domain-specific as requirements emerge
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Each new agent follows household pattern
|
||||
- [ ] Integrates with Steward/Butler flow
|
||||
- [ ] Has appropriate specialized tools
|
||||
- [ ] Documented in PHILOSOPHY.md updates
|
||||
|
||||
### Estimated Effort
|
||||
**Ongoing** - Add as needed
|
||||
|
||||
---
|
||||
|
||||
## Phase 10: User Experience Refinement
|
||||
|
||||
**Goal**: Polish the interaction experience
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Personality Tuning**
|
||||
- Refine Tatlock's wit and tone
|
||||
- Consistent household character
|
||||
- Cultural references appropriate
|
||||
- Humor that doesn't annoy
|
||||
|
||||
2. **Transparency Improvements**
|
||||
- Better progress indicators
|
||||
- Clearer reasoning explanations
|
||||
- Informative wait messages
|
||||
- Error message clarity
|
||||
|
||||
3. **Performance Optimization**
|
||||
- Response time improvements
|
||||
- Model loading optimization
|
||||
- Caching strategies
|
||||
- Streaming smoothness
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Users find Tatlock engaging
|
||||
- [ ] Wait times feel reasonable
|
||||
- [ ] Errors are understandable
|
||||
- [ ] System feels responsive
|
||||
|
||||
### Estimated Effort
|
||||
**Ongoing** - Continuous improvement
|
||||
|
||||
---
|
||||
|
||||
## Phase 11: Production Hardening
|
||||
|
||||
**Goal**: Make the system production-ready for homelab deployment
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **Deployment**
|
||||
- Complete docker-compose stack
|
||||
- Environment configuration
|
||||
- Backup strategies
|
||||
- Update procedures
|
||||
|
||||
2. **Monitoring**
|
||||
- Health checks
|
||||
- Performance metrics
|
||||
- Error tracking
|
||||
- Usage analytics
|
||||
|
||||
3. **Security**
|
||||
- Authentication hardening
|
||||
- Rate limiting
|
||||
- Input validation
|
||||
- Audit logging
|
||||
|
||||
4. **Documentation**
|
||||
- Installation guide
|
||||
- Configuration reference
|
||||
- Troubleshooting guide
|
||||
- Architecture documentation
|
||||
|
||||
### Success Criteria
|
||||
- [ ] One-command deployment
|
||||
- [ ] System health is monitorable
|
||||
- [ ] Secure for homelab use
|
||||
- [ ] Well documented
|
||||
|
||||
### Estimated Effort
|
||||
**3-4 weeks** - Production polish
|
||||
|
||||
---
|
||||
|
||||
## Dependencies Between Phases
|
||||
|
||||
```
|
||||
Phase 1 (Ollama + PydanticAI) ← Foundation for all AI
|
||||
↓
|
||||
Phase 2 (Steward)
|
||||
↓
|
||||
Phase 3 (Butler/Tatlock)
|
||||
↓
|
||||
Phase 4 (Expert Agents) ← Phase 7 (MCP) can enhance
|
||||
↓
|
||||
Phase 5 (Database/Multi-Tenancy) ← Can be deferred
|
||||
↓
|
||||
Phase 6 (Extended Services) → Phase 8 (Advanced Memory)
|
||||
↓
|
||||
Phase 9 (Extended Staff) → Phase 10 (UX) → Phase 11 (Production)
|
||||
```
|
||||
|
||||
**Critical Path**: Phases 1 → 2 → 3 → 4 must be sequential
|
||||
**Can Be Deferred**: Phase 5 (Database) until you need persistence
|
||||
**Parallel Opportunities**: Phase 6 and 7 can overlap; Phase 9 and 10 ongoing
|
||||
|
||||
---
|
||||
|
||||
## Overall Timeline Estimate
|
||||
|
||||
**Minimum Viable Household** (Phases 1-4): **15-20 weeks**
|
||||
- Working Steward → Butler → Expert Agents with real LLM
|
||||
- In-memory state (no persistence needed yet)
|
||||
- Core household functional
|
||||
|
||||
**With Persistence** (Phases 1-5): **18-24 weeks**
|
||||
- Add database and multi-tenancy
|
||||
- Conversations survive restarts
|
||||
- Multiple users supported
|
||||
|
||||
**Full-Featured System** (Phases 1-9): **35-45 weeks**
|
||||
- All services integrated
|
||||
- Advanced memory and context
|
||||
- Extended household staff
|
||||
|
||||
**Production-Ready** (All phases): **40-50 weeks**
|
||||
- Polished UX
|
||||
- Hardened for homelab deployment
|
||||
- Fully documented
|
||||
|
||||
*Note: Timeline assumes consistent part-time development effort*
|
||||
|
||||
---
|
||||
|
||||
## Success Metrics
|
||||
|
||||
### Technical
|
||||
- System implements PHILOSOPHY.md patterns
|
||||
- All household roles functional
|
||||
- Multi-tenant isolation verified
|
||||
- Real-time reasoning transparency working
|
||||
- MCP integration complete
|
||||
|
||||
### User Experience
|
||||
- Tatlock feels like interacting with a butler
|
||||
- Wait times are transparent and acceptable
|
||||
- Expert agents provide value in their domains
|
||||
- System is reliable and trustworthy
|
||||
|
||||
### Architecture
|
||||
- Clean separation between household roles
|
||||
- Easy to add new agents/tools
|
||||
- Model efficiency (base model stays loaded)
|
||||
- Scales to household + friends usage
|
||||
|
||||
---
|
||||
|
||||
## Risk Management
|
||||
|
||||
### High Risk Items
|
||||
1. **PydanticAI + Ollama integration complexity**
|
||||
- Mitigation: Prototype early, iterate on connection layer
|
||||
|
||||
2. **Multi-agent coordination complexity**
|
||||
- Mitigation: Start simple, add coordination gradually
|
||||
|
||||
3. **Model performance on homelab hardware**
|
||||
- Mitigation: Model selection, quantization, optimization
|
||||
|
||||
4. **Prompt engineering for personality consistency**
|
||||
- Mitigation: Extensive testing, user feedback, iteration
|
||||
|
||||
### Medium Risk Items
|
||||
- MCP protocol adoption and tooling maturity
|
||||
- Vector embedding quality for memory
|
||||
- Home automation integration variability
|
||||
- User authentication security
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Priority**: Implement The Developer agent for code assistance
|
||||
2. **Integration**: Add Home Assistant integration for The Housekeeper
|
||||
3. **Calendar**: Integrate scheduling service for The Secretary
|
||||
4. **Ongoing**: Add more household staff as needed
|
||||
|
||||
---
|
||||
|
||||
**Document Status**: Active planning document
|
||||
**Created**: 2025-12-06
|
||||
**Last Updated**: 2025-12-13
|
||||
@@ -0,0 +1,51 @@
|
||||
.PHONY: help setup run test test-unit test-integration test-contracts lint typecheck clean
|
||||
|
||||
VENV := .venv
|
||||
PYTHON := $(VENV)/bin/python
|
||||
PIP := $(VENV)/bin/pip
|
||||
PYTEST := $(VENV)/bin/pytest
|
||||
RUFF := $(VENV)/bin/ruff
|
||||
MYPY := $(VENV)/bin/mypy
|
||||
UVICORN := $(VENV)/bin/uvicorn
|
||||
|
||||
HOST := 0.0.0.0
|
||||
PORT := 8777
|
||||
|
||||
help: ## Show this help
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
setup: ## Create venv and install all dependencies
|
||||
python3 -m venv $(VENV)
|
||||
$(PIP) install --upgrade pip
|
||||
$(PIP) install -e ".[dev]"
|
||||
|
||||
run: ## Start the development server on port 8777
|
||||
@mkdir -p build/logs
|
||||
@if lsof -Pi :$(PORT) -sTCP:LISTEN -t >/dev/null 2>&1; then \
|
||||
echo "Error: Port $(PORT) is already in use"; \
|
||||
echo "Run: lsof -i :$(PORT) to see what's using it"; \
|
||||
exit 1; \
|
||||
fi
|
||||
$(UVICORN) src.main:app --reload --host $(HOST) --port $(PORT) 2>&1 | tee build/logs/server.log
|
||||
|
||||
test: ## Run unit tests (no external services needed)
|
||||
$(PYTEST) --ignore=tests/e2e --ignore=tests/integration --ignore=tests/contracts
|
||||
|
||||
test-unit: test ## Alias for test
|
||||
|
||||
test-integration: ## Run integration tests (needs Claude/Ollama)
|
||||
$(PYTEST) tests/agents/test_tatlock_agent.py -v
|
||||
|
||||
test-contracts: ## Wire-level contract tests against live service boundaries
|
||||
$(PYTEST) tests/contracts -v --no-cov
|
||||
|
||||
lint: ## Run ruff linter and formatter check
|
||||
$(RUFF) check src tests
|
||||
$(RUFF) format --check src tests
|
||||
|
||||
typecheck: ## Run mypy type checking
|
||||
$(MYPY) src
|
||||
|
||||
clean: ## Remove build artifacts, caches, and coverage reports
|
||||
rm -rf .cache build
|
||||
find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
||||
@@ -1,401 +0,0 @@
|
||||
# Tatlock Enhancement Plan: Bidirectional Claude Integration
|
||||
|
||||
## Executive Summary
|
||||
|
||||
Implement a **bidirectional architecture** that:
|
||||
1. **Superpowers Tatlock** by swapping Ollama→Claude backend (200k context, better reasoning, same butler personality)
|
||||
2. **Exposes Tatlock as MCP server** for Claude instances on any device (phone, browser, desktop)
|
||||
|
||||
This gives you the flexibility to use whichever AI is best/most accessible at any moment.
|
||||
|
||||
## Key Insight: Blanket Backend Swap (Simpler Than Sidecar)
|
||||
|
||||
Instead of adding a Claude "Analyst" sidecar agent, **swap the underlying model for ALL agents**:
|
||||
|
||||
```
|
||||
CURRENT: TatlockAgent → OpenAIChatModel → OllamaProvider → Ollama (mistral-nemo)
|
||||
PROPOSED: TatlockAgent → AnthropicModel → AnthropicProvider → Claude API
|
||||
↘ (fallback when offline) → OllamaProvider → Ollama
|
||||
```
|
||||
|
||||
**Why this works:**
|
||||
- PydanticAI natively supports Anthropic via `AnthropicModel` + `AnthropicProvider`
|
||||
- The same `TATLOCK_SYSTEM_PROMPT` is passed to Claude - butler personality preserved
|
||||
- Claude is **better** at following system prompts than mistral-nemo
|
||||
- 200k context for ALL queries, not just "complex" ones
|
||||
- Simpler architecture: no routing logic, no sidecar delegation
|
||||
|
||||
---
|
||||
|
||||
## Research Findings
|
||||
|
||||
### Industry Best Practices (2025-2026)
|
||||
|
||||
**MCP Protocol Updates** ([MCP Spec Updates June 2025](https://auth0.com/blog/mcp-specs-update-all-about-auth/)):
|
||||
- Streamable HTTP replaced SSE (March 2025) - better for cloud deployment
|
||||
- OAuth 2.0 required for remote servers - MCP servers are OAuth Resource Servers
|
||||
- Tool Output Schemas now available - better structured data handling
|
||||
- MCP Registry launched (Sept 2025) - community server discovery
|
||||
|
||||
**Community Patterns** ([Claude Code Router](https://github.com/musistudio/claude-code-router)):
|
||||
- Task-based routing is becoming standard: route simple→local, complex→cloud
|
||||
- Translation proxies bridge Anthropic Messages API ↔ OpenAI format
|
||||
- Cost savings of up to 98% reported with smart routing
|
||||
|
||||
**Home Automation MCP** ([ha-mcp](https://github.com/homeassistant-ai/ha-mcp)):
|
||||
- Production-ready MCP servers exist for Home Assistant
|
||||
- Support Claude Code, Gemini CLI, Open WebUI, VSCode, Cursor
|
||||
- Pattern: expose local tools securely to remote AI clients
|
||||
|
||||
**Remote MCP Access** ([mcp-remote](https://www.npmjs.com/package/mcp-remote)):
|
||||
- Bridge local MCP servers to Claude Desktop/Browser via proxy
|
||||
- Supports authentication headers for security
|
||||
- Works with ngrok/Cloudflare Tunnel for HTTPS
|
||||
|
||||
---
|
||||
|
||||
## Recommended Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ BIDIRECTIONAL TATLOCK-CLAUDE ARCHITECTURE │
|
||||
├─────────────────────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ╔═══════════════════════════════════════════════════════════════════════════════╗ │
|
||||
│ ║ SCENARIO A: Using Tatlock (Open WebUI, local apps) ║ │
|
||||
│ ║ ───────────────────────────────────────────────── ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Request → Steward → Tatlock → Tools + Expert Delegation ║ │
|
||||
│ ║ │ ║ │
|
||||
│ ║ ├─→ Librarian (Claude) → research, wiki, RAG ║ │
|
||||
│ ║ ├─→ Biographer (Claude) → memory, preferences ║ │
|
||||
│ ║ ├─→ Housekeeper (Claude) → home automation ║ │
|
||||
│ ║ └─→ All powered by Claude with Ollama fallback ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Butler personality preserved, 200k context for all queries ║ │
|
||||
│ ╚═══════════════════════════════════════════════════════════════════════════════╝ │
|
||||
│ │
|
||||
│ ╔═══════════════════════════════════════════════════════════════════════════════╗ │
|
||||
│ ║ SCENARIO B: Using Claude.ai / Claude Desktop / Phone ║ │
|
||||
│ ║ ──────────────────────────────────────────────────── ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Claude ──[MCP over HTTPS]──► Tatlock MCP Server → Household Tools ║ │
|
||||
│ ║ │ ║ │
|
||||
│ ║ ├─→ calculator, datetime ║ │
|
||||
│ ║ ├─→ web_search, wiki_search ║ │
|
||||
│ ║ ├─→ hybrid_search (RAG) ║ │
|
||||
│ ║ ├─→ memory_recall, store_insight ║ │
|
||||
│ ║ └─→ home_control (lights, climate) ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Full 200k context, your local tools accessible from anywhere ║ │
|
||||
│ ╚═══════════════════════════════════════════════════════════════════════════════╝ │
|
||||
│ │
|
||||
│ ╔═══════════════════════════════════════════════════════════════════════════════╗ │
|
||||
│ ║ SCENARIO C: Offline (internet down) ║ │
|
||||
│ ║ ────────────────────────────────── ║ │
|
||||
│ ║ ║ │
|
||||
│ ║ Tatlock operates fully locally with Ollama ║ │
|
||||
│ ║ • All tools work (except web search) ║ │
|
||||
│ ║ • Graceful degradation with same butler personality ║ │
|
||||
│ ╚═══════════════════════════════════════════════════════════════════════════════╝ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Plan
|
||||
|
||||
### Phase 1: Blanket Backend Swap (Claude for All Agents)
|
||||
|
||||
Replace Ollama with Claude as the default backend for all PydanticAI agents, with automatic offline fallback.
|
||||
|
||||
**New Files:**
|
||||
```
|
||||
src/anthropic/
|
||||
├── __init__.py
|
||||
├── provider.py # Claude provider with health check
|
||||
└── model_selector.py # Chooses Claude or Ollama based on availability
|
||||
```
|
||||
|
||||
**Key Implementation (`src/anthropic/provider.py`):**
|
||||
```python
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
from pydantic_ai.providers.anthropic import AnthropicProvider
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
from src.core.config import config
|
||||
|
||||
_anthropic_available: bool | None = None
|
||||
|
||||
async def check_anthropic_health() -> bool:
|
||||
"""Check if Anthropic API is reachable."""
|
||||
global _anthropic_available
|
||||
try:
|
||||
from anthropic import AsyncAnthropic
|
||||
client = AsyncAnthropic(api_key=config.ANTHROPIC_API_KEY)
|
||||
await client.messages.create(
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
max_tokens=1,
|
||||
messages=[{"role": "user", "content": "hi"}]
|
||||
)
|
||||
_anthropic_available = True
|
||||
except Exception:
|
||||
_anthropic_available = False
|
||||
return _anthropic_available
|
||||
|
||||
def get_model(prefer_cloud: bool = True):
|
||||
"""Get the best available model. Returns Claude if available, otherwise Ollama."""
|
||||
if prefer_cloud and config.ANTHROPIC_API_KEY and _anthropic_available:
|
||||
provider = AnthropicProvider(api_key=config.ANTHROPIC_API_KEY)
|
||||
return AnthropicModel(
|
||||
model_name=config.ANTHROPIC_MODEL,
|
||||
provider=provider,
|
||||
)
|
||||
else:
|
||||
return OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
provider=get_ollama_provider()
|
||||
)
|
||||
```
|
||||
|
||||
**Modify TatlockAgent (`src/agents/tatlock.py`):**
|
||||
```python
|
||||
def _ensure_agent(self):
|
||||
if self._agent is not None:
|
||||
return
|
||||
|
||||
from src.anthropic.model_selector import get_model
|
||||
|
||||
model = get_model(prefer_cloud=True)
|
||||
|
||||
self._agent = Agent(
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT, # Same butler personality!
|
||||
)
|
||||
self._register_tools()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: MCP Server (Expose Tools to Claude)
|
||||
|
||||
Create an MCP server that exposes Tatlock's household tools to external Claude instances.
|
||||
|
||||
**New Files:**
|
||||
```
|
||||
src/mcp/
|
||||
├── __init__.py
|
||||
├── server.py # MCP server using mcp Python SDK
|
||||
├── tool_adapters.py # Convert PydanticAI tools → MCP schemas
|
||||
├── auth.py # API key authentication
|
||||
└── transport.py # Streamable HTTP transport
|
||||
```
|
||||
|
||||
**Docker Stack Addition (`stacks/agents.yml`):**
|
||||
```yaml
|
||||
tatlock-mcp:
|
||||
image: git.schweitz.internal/jpmschweitzer/tatlock:latest
|
||||
command: ["python", "-m", "src.mcp.server"]
|
||||
ports:
|
||||
- "8002:8002"
|
||||
environment:
|
||||
- MCP_AUTH_TOKEN=${MCP_AUTH_TOKEN}
|
||||
networks:
|
||||
- docker-dataplane
|
||||
```
|
||||
|
||||
**Claude Desktop Configuration:**
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"tatlock": {
|
||||
"command": "npx",
|
||||
"args": ["mcp-remote", "https://mcp.schweitz.net/sse", "--header", "Authorization: Bearer ${MCP_AUTH_TOKEN}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Files to Modify
|
||||
|
||||
### Phase 1 - Backend Swap
|
||||
|
||||
**New Files:**
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `src/anthropic/__init__.py` | Package init |
|
||||
| `src/anthropic/provider.py` | Claude provider with health check |
|
||||
| `src/anthropic/model_selector.py` | Choose Claude or Ollama based on availability |
|
||||
|
||||
**Modified Files:**
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `src/core/config.py` | Add `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `PREFER_CLOUD_BACKEND` |
|
||||
| `src/agents/tatlock.py` | Use `get_model()` instead of hardcoded Ollama |
|
||||
| `src/agents/librarian/agent.py` | Use `get_model()` instead of hardcoded Ollama |
|
||||
| `src/agents/biographer/agent.py` | Use `get_model()` instead of hardcoded Ollama |
|
||||
| `src/agents/steward/agent.py` | Convert to PydanticAI or add Anthropic API support |
|
||||
| `src/core/startup.py` | Add Anthropic health check on startup |
|
||||
| `requirements.txt` | Add `anthropic>=0.40.0` |
|
||||
| `.env.example` | Document new environment variables |
|
||||
|
||||
### Phase 2 - MCP Server
|
||||
|
||||
**New Files:**
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `src/mcp/__init__.py` | Package init |
|
||||
| `src/mcp/server.py` | MCP server implementation |
|
||||
| `src/mcp/tool_adapters.py` | PydanticAI → MCP schema conversion |
|
||||
| `src/mcp/auth.py` | Token-based authentication |
|
||||
|
||||
---
|
||||
|
||||
## Cost Analysis
|
||||
|
||||
- **Claude API**: $5-30/month (10-50 calls/day, ~2k input + 1k output tokens/call)
|
||||
- **MCP via Claude Pro**: Included in subscription
|
||||
- **Total**: ~$10-80/month for full bidirectional integration
|
||||
|
||||
---
|
||||
|
||||
## Verification Plan
|
||||
|
||||
### Phase 1 Testing
|
||||
```bash
|
||||
# 1. Run with Claude backend
|
||||
ANTHROPIC_API_KEY=your-key docker-compose up -d tatlock
|
||||
|
||||
# 2. Verify Claude is being used
|
||||
docker logs tatlock 2>&1 | grep -i "anthropic\|claude"
|
||||
|
||||
# 3. Test butler personality
|
||||
curl -X POST http://tatlock.schweitz.internal:8000/v1/responses \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "Tatlock", "input": "Hello, who are you?"}'
|
||||
|
||||
# 4. Test offline fallback
|
||||
ANTHROPIC_API_KEY="" docker-compose up -d tatlock
|
||||
docker logs tatlock 2>&1 | grep -i "ollama\|fallback"
|
||||
```
|
||||
|
||||
### Phase 2 Testing
|
||||
```bash
|
||||
# 1. Start MCP server
|
||||
docker-compose up -d tatlock-mcp
|
||||
|
||||
# 2. Test MCP endpoint
|
||||
curl -X POST https://mcp.schweitz.net/tools/list \
|
||||
-H "Authorization: Bearer $MCP_AUTH_TOKEN"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Priority
|
||||
|
||||
1. **Phase 1: Backend Swap** (~1 week)
|
||||
- Immediate value: 200k context for ALL queries
|
||||
- Low risk: provider abstraction, graceful offline fallback
|
||||
|
||||
2. **Phase 2: MCP Server** (~2-3 weeks)
|
||||
- Enables cross-device access
|
||||
- Bidirectional: Tatlock superpowered by Claude AND accessible to Claude
|
||||
|
||||
---
|
||||
|
||||
## Future Phases (Optional)
|
||||
|
||||
- **Phase 3: LiteLLM Gateway** - Unified endpoint for all models, config-driven routing
|
||||
- **Phase 4: Multi-Provider** - Add OpenAI, Vertex AI, etc.
|
||||
- **Phase 5: Smart Routing** - Context-aware model selection, cost ceiling enforcement
|
||||
|
||||
---
|
||||
|
||||
## Offline Behavior
|
||||
|
||||
| Scenario | Behavior |
|
||||
|----------|----------|
|
||||
| No API key | Use Ollama exclusively |
|
||||
| API unreachable | Use Ollama, log warning |
|
||||
| API rate limited | Fallback to Ollama |
|
||||
|
||||
| Aspect | Claude | Ollama |
|
||||
|--------|--------|--------|
|
||||
| Context | 200k tokens | ~8k tokens |
|
||||
| Latency | 1-3s (network) | 0.5-1s (local) |
|
||||
| Personality | Preserved | Preserved |
|
||||
| Tools | All work | All work |
|
||||
| Cost | API charges | Free |
|
||||
|
||||
---
|
||||
|
||||
## Implementation Status
|
||||
|
||||
### Phase 1: Backend Swap - CODE COMPLETE (awaiting API access)
|
||||
|
||||
- [x] Add Anthropic config settings to `src/core/config.py`
|
||||
- [x] Add `pydantic-ai-slim[openai,anthropic]` to requirements.txt
|
||||
- [x] Create `src/anthropic/` module (model_selector.py)
|
||||
- [x] Add Claude health check to startup.py
|
||||
- [x] Refactor all PydanticAI agents to use `get_model()`
|
||||
- [x] Librarian
|
||||
- [x] Biographer
|
||||
- [x] Housekeeper
|
||||
- [x] Tatlock (6 locations)
|
||||
- [x] Add Claude API path to Steward agent (direct API calls)
|
||||
- [x] Update `.env.example` with new variables
|
||||
- [x] Test Ollama fallback (working)
|
||||
- [ ] Test with Claude API key (blocked: no API access currently)
|
||||
|
||||
**Note:** Implementation complete. Currently runs in Ollama-only mode. Will automatically use Claude when `ANTHROPIC_API_KEY` is configured.
|
||||
|
||||
### Phase 2: MCP Server - NOT STARTED
|
||||
|
||||
- [ ] Create `src/mcp/` module
|
||||
- [ ] Tool adapters (PydanticAI → MCP schema)
|
||||
- [ ] Authentication middleware
|
||||
- [ ] Streamable HTTP transport
|
||||
- [ ] Docker stack configuration
|
||||
|
||||
---
|
||||
|
||||
## Related Repository Handovers
|
||||
|
||||
Handover documents created in each repo: `PROJECT_CLAUDIFICATION_HANDOVER.md`
|
||||
|
||||
### library-desk - HANDOVER CREATED
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Review HybridRAG response size limits
|
||||
- [ ] Review smart_create endpoint for Claude optimization
|
||||
- [ ] Evaluate response formats for LLM consumption
|
||||
|
||||
### core-api - HANDOVER CREATED
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Review list_devices response format
|
||||
- [ ] Review error messages for LLM consumption
|
||||
- [ ] Evaluate rate limiting for faster Claude processing
|
||||
|
||||
### portainer-core - HANDOVER CREATED (blocking for production)
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Update stack with new environment variables
|
||||
- [ ] Configure secrets management for API key
|
||||
- [ ] Update CONTAINERS.md documentation
|
||||
|
||||
### webber - HANDOVER CREATED
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Review content truncation limits
|
||||
- [ ] Evaluate extraction quality for LLM consumption
|
||||
|
||||
### tatlock-ui - HANDOVER CREATED
|
||||
|
||||
- [x] Write handover document
|
||||
- [ ] Test streaming responses with Claude backend
|
||||
- [ ] Test conversation history with larger context
|
||||
- [ ] Verify tool call display and reasoning rendering
|
||||
@@ -1,6 +1,6 @@
|
||||
# Tatlock - Your Homelab Butler
|
||||
|
||||
> **📖 For the complete system vision and architectural philosophy, see [PHILOSOPHY.md](PHILOSOPHY.md)**
|
||||
> **📖 For the complete system vision and architectural philosophy, see [docs/philosophy.md](docs/philosophy.md)**
|
||||
|
||||
A privacy-first, offline-capable personal assistant system that coordinates specialized AI agents to help with research, development, home automation, and daily organization.
|
||||
|
||||
@@ -58,7 +58,7 @@ A privacy-first, offline-capable personal assistant system that coordinates spec
|
||||
- Error triggers for testing (rate_limit, context_overflow)
|
||||
|
||||
- **Tatlock**: Real PydanticAI agent with butler personality
|
||||
- **LLM Backend**: Ollama (mistral-nemo:latest by default)
|
||||
- **LLM Backend**: Ollama (gemma4:e2b by default, local-first) with optional Claude fallback
|
||||
- **Personality**: Witty British butler, research-oriented
|
||||
- **Core Tools**:
|
||||
- **Calculator**: Safe mathematical expression evaluation
|
||||
@@ -74,7 +74,7 @@ A privacy-first, offline-capable personal assistant system that coordinates spec
|
||||
|
||||
- Python 3.12+ (Python 3.12.11 recommended)
|
||||
- **External Services** (must be running separately):
|
||||
- **Ollama**: LLM inference (mistral-nemo:latest, nomic-embed-text)
|
||||
- **Ollama**: LLM inference (gemma4:e2b, nomic-embed-text)
|
||||
- **Redis**: Caching and session memory
|
||||
- **Qdrant**: Vector storage for The Biographer's memory
|
||||
- **SearXNG**: Web search (optional)
|
||||
@@ -89,12 +89,8 @@ A privacy-first, offline-capable personal assistant system that coordinates spec
|
||||
git clone https://git.schweitz.net/jpmschweitzer/tatlock.git
|
||||
cd tatlock
|
||||
|
||||
# Create virtual environment
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
||||
|
||||
# Install dependencies
|
||||
pip install -r requirements.txt
|
||||
make setup
|
||||
```
|
||||
|
||||
### Run the Server
|
||||
@@ -268,7 +264,10 @@ Interactive documentation available at:
|
||||
pytest
|
||||
|
||||
# Run unit tests only (no external services needed)
|
||||
pytest --ignore=tests/e2e --ignore=tests/integration
|
||||
pytest --ignore=tests/e2e --ignore=tests/integration --ignore=tests/contracts
|
||||
|
||||
# Wire-level contract tests against live service boundaries
|
||||
make test-contracts
|
||||
|
||||
# Run with coverage
|
||||
pytest --cov=src --cov-report=term-missing
|
||||
@@ -307,12 +306,17 @@ Create a `.env` file for custom configuration:
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8000
|
||||
|
||||
# Ollama Configuration
|
||||
# Ollama Configuration (primary backend)
|
||||
OLLAMA_HOST=http://localhost:11434
|
||||
OLLAMA_DEFAULT_MODEL=mistral-nemo:latest
|
||||
OLLAMA_DEFAULT_MODEL=gemma4:e2b
|
||||
OLLAMA_EMBEDDING_MODEL=nomic-embed-text
|
||||
OLLAMA_TIMEOUT=120
|
||||
|
||||
# Claude fallback (optional; used when Ollama is down or PREFER_CLOUD_BACKEND=true)
|
||||
# ANTHROPIC_API_KEY=sk-ant-api03-your-key-here
|
||||
ANTHROPIC_MODEL=claude-sonnet-5
|
||||
PREFER_CLOUD_BACKEND=false
|
||||
|
||||
# Redis Configuration
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
@@ -396,8 +400,7 @@ tatlock/
|
||||
│ │ └── multi_tenancy.py # User isolation utilities
|
||||
│ └── main.py # Application entry point
|
||||
├── tests/ # Comprehensive test suite
|
||||
├── PHILOSOPHY.md # System vision and architecture
|
||||
├── IMPLEMENTATION_ROADMAP.md # Development phases
|
||||
├── docs/ # Project documentation
|
||||
├── CHANGELOG.md # Version history
|
||||
└── README.md # This file
|
||||
```
|
||||
@@ -416,8 +419,8 @@ For LLM agent development guidelines and architectural decisions, see [AGENTS.md
|
||||
|
||||
## Documentation
|
||||
|
||||
- **System Philosophy**: [PHILOSOPHY.md](PHILOSOPHY.md) - Vision, goals, and architectural patterns
|
||||
- **User Guide**: This file - Installation, usage, and examples
|
||||
- **System Philosophy**: [docs/philosophy.md](docs/philosophy.md) - Vision, goals, and architectural patterns
|
||||
- **Development Roadmap**: [docs/roadmap.md](docs/roadmap.md) - Open work and planned phases
|
||||
- **Developer Guidelines**: [AGENTS.md](AGENTS.md) - LLM agent development patterns
|
||||
- **Version History**: [CHANGELOG.md](CHANGELOG.md) - Changes and releases
|
||||
|
||||
@@ -432,8 +435,8 @@ For LLM agent development guidelines and architectural decisions, see [AGENTS.md
|
||||
|
||||
## Version
|
||||
|
||||
Current version: **1.3.2** - Biographer tool type hints fix
|
||||
Current version: see [CHANGELOG.md](CHANGELOG.md)
|
||||
|
||||
---
|
||||
|
||||
**Note**: Tatlock is a production-ready homelab butler. All household staff use PydanticAI with Ollama for local LLM inference.
|
||||
**Note**: Tatlock is a production-ready homelab butler. All household staff use PydanticAI with local Ollama inference (gemma4), with an optional Claude cloud fallback.
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
# Claude Integration Plan
|
||||
|
||||
## Overview
|
||||
|
||||
Tatlock uses a bidirectional Claude architecture:
|
||||
- **Scenario A**: Tatlock powered by Claude backend (with Ollama fallback) — **COMPLETE**, then **rolled back to local-first**: Ollama/gemma4 is primary, Claude is retained as fallback (`PREFER_CLOUD_BACKEND=false`)
|
||||
- **Scenario B**: Tatlock exposed as MCP server for external Claude instances — **OPEN**
|
||||
- **Scenario C**: Offline operation via Ollama — **COMPLETE**
|
||||
|
||||
---
|
||||
|
||||
## MCP Server (Expose Tools to Claude) — NOT STARTED
|
||||
|
||||
Create an MCP server that exposes Tatlock's household tools to external Claude instances.
|
||||
|
||||
### New Files
|
||||
|
||||
```
|
||||
src/mcp/
|
||||
├── __init__.py
|
||||
├── server.py # MCP server using mcp Python SDK
|
||||
├── tool_adapters.py # Convert PydanticAI tools → MCP schemas
|
||||
├── auth.py # API key authentication
|
||||
└── transport.py # Streamable HTTP transport
|
||||
```
|
||||
|
||||
### Docker Stack Addition
|
||||
|
||||
```yaml
|
||||
tatlock-mcp:
|
||||
image: git.schweitz.internal/jpmschweitzer/tatlock:latest
|
||||
command: ["python", "-m", "src.mcp.server"]
|
||||
ports:
|
||||
- "8002:8002"
|
||||
environment:
|
||||
- MCP_AUTH_TOKEN=${MCP_AUTH_TOKEN}
|
||||
networks:
|
||||
- docker-dataplane
|
||||
```
|
||||
|
||||
### Claude Desktop Configuration
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"tatlock": {
|
||||
"command": "npx",
|
||||
"args": ["mcp-remote", "https://mcp.schweitz.net/sse", "--header", "Authorization: Bearer ${MCP_AUTH_TOKEN}"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Checklist
|
||||
|
||||
- [ ] Create `src/mcp/` module
|
||||
- [ ] Tool adapters (PydanticAI → MCP schema)
|
||||
- [ ] Authentication middleware
|
||||
- [ ] Streamable HTTP transport
|
||||
- [ ] Docker stack configuration
|
||||
|
||||
---
|
||||
|
||||
## Future Phases
|
||||
|
||||
- **LiteLLM Gateway** — Unified endpoint for all models, config-driven routing
|
||||
- **Multi-Provider** — Add OpenAI, Vertex AI, etc.
|
||||
- **Smart Routing** — Context-aware model selection, cost ceiling enforcement
|
||||
|
||||
---
|
||||
|
||||
## Offline Behavior
|
||||
|
||||
| Scenario | Behavior |
|
||||
|----------|----------|
|
||||
| No API key | Use Ollama exclusively |
|
||||
| API unreachable | Use Ollama, log warning |
|
||||
| API rate limited | Fallback to Ollama |
|
||||
|
||||
| Aspect | Claude | Ollama |
|
||||
|--------|--------|--------|
|
||||
| Context | 200k tokens | ~8k tokens |
|
||||
| Latency | 1-3s (network) | 0.5-1s (local) |
|
||||
| Personality | Preserved | Preserved |
|
||||
| Tools | All work | All work |
|
||||
| Cost | API charges | Free |
|
||||
|
||||
---
|
||||
|
||||
## Related Repo Handovers
|
||||
|
||||
Handover documents created in each repo: `PROJECT_CLAUDIFICATION_HANDOVER.md`
|
||||
|
||||
### Open Items
|
||||
|
||||
- **library-desk**: Review HybridRAG response size limits, smart_create endpoint, response formats
|
||||
- **core-api**: Review list_devices response format, error messages, rate limiting
|
||||
- **portainer-core**: Update stack with new env vars, configure secrets, update CONTAINERS.md
|
||||
- **webber**: Review content truncation limits, extraction quality
|
||||
- **tatlock-ui**: Test streaming with Claude backend, conversation history, tool call display
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
# Tatlock Implementation Roadmap
|
||||
|
||||
> **Reference**: See [philosophy.md](philosophy.md) for the target architecture and vision
|
||||
|
||||
This document tracks open/planned work. Completed phases have been removed.
|
||||
|
||||
## Current State (v2.0.5)
|
||||
|
||||
**What we have**:
|
||||
- OpenAI-compatible API (Responses API + Chat Completions)
|
||||
- Two-tier architecture (Steward → Tatlock)
|
||||
- Household staff: Tatlock (Butler), Steward, Librarian, Biographer
|
||||
- Core tools: Calculator, Date/Time, Web search (SearXNG)
|
||||
- Memory system: Qdrant (vector), Redis (session cache), multi-tenancy via ContextVar
|
||||
- Dual backend: Ollama/gemma4 (primary) + Claude (fallback)
|
||||
- 439 tests with good coverage
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: Expert Household Staff — Remaining Agents
|
||||
|
||||
**Goal**: Implement remaining domain-specific expert agents
|
||||
|
||||
### Planned Agents
|
||||
|
||||
1. **The Developer** (Software Development)
|
||||
- Code generation assistance
|
||||
- Debugging support
|
||||
- Documentation generation
|
||||
- Architecture guidance
|
||||
|
||||
2. **The Handyman** (System Maintenance)
|
||||
- System status queries
|
||||
- Log analysis
|
||||
- Basic troubleshooting
|
||||
- Infrastructure monitoring
|
||||
|
||||
3. **The Secretary** (Scheduling & Organization)
|
||||
- Calendar integration
|
||||
- Task management
|
||||
- Reminder system
|
||||
- Schedule conflict detection
|
||||
|
||||
4. **The Housekeeper** (Home Automation)
|
||||
- Home Assistant integration
|
||||
- Device control interface
|
||||
- Status queries
|
||||
- Automation triggers
|
||||
|
||||
### Each Agent Includes
|
||||
- Specialized prompt and personality
|
||||
- Domain-specific tools
|
||||
- MCP integration points (where applicable)
|
||||
- Integration with Butler orchestration
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Each agent implemented as separate module
|
||||
- [ ] Agents callable via tool framework
|
||||
- [ ] Can invoke specialized models (e.g., Codestral for Developer)
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: Persistence Layer — Database & Multi-Tenancy
|
||||
|
||||
**Goal**: Add persistent storage and multi-user support
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **PostgreSQL Integration**
|
||||
- Docker compose configuration
|
||||
- Database schema with tenant isolation
|
||||
- Alembic migrations
|
||||
- SQLAlchemy models
|
||||
|
||||
2. **Multi-Tenant Architecture**
|
||||
- Tenant identification middleware
|
||||
- Tenant-scoped database sessions
|
||||
- User authentication system
|
||||
- Per-tenant data isolation
|
||||
|
||||
3. **Core Data Models**
|
||||
- Users and tenants
|
||||
- Conversations and messages (migrate from in-memory)
|
||||
- Agent interactions log
|
||||
- System configuration and preferences
|
||||
|
||||
### Success Criteria
|
||||
- [ ] PostgreSQL container running
|
||||
- [ ] Multiple users authenticate separately
|
||||
- [ ] Each user sees only their own data
|
||||
- [ ] Conversations persist across restarts
|
||||
- [ ] Database migrations work correctly
|
||||
|
||||
---
|
||||
|
||||
## Phase 7: MCP (Model Context Protocol) Integration
|
||||
|
||||
**Goal**: Enable rich tool integrations via MCP
|
||||
|
||||
See also [claude-integration.md](claude-integration.md) for MCP server implementation details.
|
||||
|
||||
### Deliverables
|
||||
|
||||
1. **MCP Server Framework**
|
||||
- MCP server implementation
|
||||
- Tool registration via MCP
|
||||
- Schema validation
|
||||
- Error handling
|
||||
|
||||
2. **MCP Client in Agents**
|
||||
- PydanticAI MCP integration
|
||||
- Tool discovery from MCP servers
|
||||
- Dynamic tool loading
|
||||
|
||||
3. **Initial MCP Tools**
|
||||
- File system operations
|
||||
- Database queries
|
||||
- API integrations
|
||||
- System commands
|
||||
|
||||
### Success Criteria
|
||||
- [ ] MCP server running
|
||||
- [ ] Tools exposed via MCP protocol
|
||||
- [ ] Agents can discover and use MCP tools
|
||||
- [ ] New tools addable without code changes
|
||||
- [ ] MCP tools visible in Steward recommendations
|
||||
|
||||
---
|
||||
|
||||
## Phase 8: Advanced Memory & Context — Remaining Work
|
||||
|
||||
**Goal**: Implement sophisticated context management and personalization
|
||||
|
||||
### Open Deliverables
|
||||
|
||||
1. **Context Management**
|
||||
- Smart context window trimming
|
||||
- Conversation branching
|
||||
- Topic tracking
|
||||
|
||||
2. **Personalization**
|
||||
- User preference learning
|
||||
- Interaction pattern analysis
|
||||
- Adaptive responses
|
||||
- Custom agent personalities per user
|
||||
|
||||
### Success Criteria
|
||||
- [ ] Conversations automatically embedded to Qdrant
|
||||
- [ ] Memory improves over time (learning from interactions)
|
||||
|
||||
---
|
||||
|
||||
## Phase 9: Extended Household Staff
|
||||
|
||||
**Goal**: Add specialized agents for additional domains
|
||||
|
||||
### Future Agents
|
||||
- **The Accountant** — Expense tracking, budgets, financial reports
|
||||
- **The Chef** — Meal planning, recipes, nutrition tracking
|
||||
- Others as needs emerge
|
||||
|
||||
---
|
||||
|
||||
## Phase 10: User Experience Refinement
|
||||
|
||||
**Goal**: Polish the interaction experience
|
||||
|
||||
- Personality tuning and consistency
|
||||
- Better progress indicators
|
||||
- Response time improvements
|
||||
- Streaming smoothness
|
||||
|
||||
---
|
||||
|
||||
## Phase 11: Production Hardening
|
||||
|
||||
**Goal**: Make the system production-ready for homelab deployment
|
||||
|
||||
- Complete docker-compose stack
|
||||
- Health checks and monitoring
|
||||
- Authentication hardening and rate limiting
|
||||
- Installation and troubleshooting documentation
|
||||
|
||||
---
|
||||
|
||||
## Dependencies
|
||||
|
||||
```
|
||||
Phase 4 (Remaining Agents)
|
||||
↓
|
||||
Phase 5 (Database/Multi-Tenancy) ← Can be deferred
|
||||
↓
|
||||
Phase 7 (MCP) → Phase 8 (Advanced Memory)
|
||||
↓
|
||||
Phase 9 (Extended Staff) → Phase 10 (UX) → Phase 11 (Production)
|
||||
```
|
||||
|
||||
**Can Be Deferred**: Phase 5 until you need persistence
|
||||
**Parallel Opportunities**: Phases 7 and 8 can overlap; 9 and 10 ongoing
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Implement The Developer agent for code assistance
|
||||
2. Add Home Assistant integration for The Housekeeper
|
||||
3. Integrate scheduling service for The Secretary
|
||||
4. MCP server for external Claude access
|
||||
+60
-3
@@ -4,17 +4,69 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "tatlock"
|
||||
version = "2.0.1"
|
||||
version = "2.3.0"
|
||||
description = "OpenAI-compatible API with Ollama backend"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = []
|
||||
dependencies = [
|
||||
"fastapi>=0.123,<0.124",
|
||||
"uvicorn[standard]>=0.38,<0.39",
|
||||
"pydantic>=2.11,<2.13",
|
||||
"pydantic-settings>=2.12,<2.13",
|
||||
"pydantic-ai-slim[openai,anthropic]>=1.27,<1.28",
|
||||
"anthropic>=0.77,<1.0",
|
||||
"httpx>=0.28,<0.29",
|
||||
"sse-starlette>=3.0,<3.1",
|
||||
"python-dotenv>=1.2,<1.3",
|
||||
"starlette>=0.45,<0.46",
|
||||
"redis[hiredis]>=5.2,<6.0",
|
||||
"qdrant-client>=1.12,<2.0",
|
||||
"structlog>=24.1,<25.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.3,<8.4",
|
||||
"pytest-asyncio>=0.25,<0.26",
|
||||
"pytest-cov>=6.0,<6.1",
|
||||
"pytest-mock>=3.14,<3.15",
|
||||
"ruff>=0.8,<0.9",
|
||||
"mypy>=1.14,<1.15",
|
||||
"faker>=34.0,<35.0",
|
||||
"coverage[toml]>=7.7,<7.8",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
python_classes = ["Test*"]
|
||||
python_functions = ["test_*"]
|
||||
asyncio_mode = "auto"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
cache_dir = ".cache/pytest"
|
||||
markers = [
|
||||
"unit: Unit tests",
|
||||
"integration: Integration tests",
|
||||
"slow: Slow running tests",
|
||||
"contract: Wire-level contract tests against live service boundaries",
|
||||
]
|
||||
addopts = [
|
||||
"--verbose",
|
||||
"--strict-markers",
|
||||
"--tb=short",
|
||||
"--cov=src",
|
||||
"--cov-report=term-missing",
|
||||
"--cov-report=html:build/coverage/html",
|
||||
"--cov-report=xml:build/coverage/coverage.xml",
|
||||
"--cov-branch",
|
||||
]
|
||||
filterwarnings = [
|
||||
"ignore::DeprecationWarning",
|
||||
]
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["src"]
|
||||
branch = true
|
||||
data_file = "build/coverage/.coverage"
|
||||
omit = [
|
||||
"*/tests/*",
|
||||
"*/__pycache__/*",
|
||||
@@ -37,11 +89,15 @@ exclude_lines = [
|
||||
]
|
||||
|
||||
[tool.coverage.html]
|
||||
directory = "htmlcov"
|
||||
directory = "build/coverage/html"
|
||||
|
||||
[tool.coverage.xml]
|
||||
output = "build/coverage/coverage.xml"
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py312"
|
||||
cache-dir = ".cache/ruff"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
@@ -64,6 +120,7 @@ ignore = [
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.12"
|
||||
cache_dir = ".cache/mypy"
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
disallow_untyped_defs = true
|
||||
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
[pytest]
|
||||
testpaths = tests
|
||||
python_files = test_*.py
|
||||
python_classes = Test*
|
||||
python_functions = test_*
|
||||
asyncio_mode = auto
|
||||
asyncio_default_fixture_loop_scope = function
|
||||
|
||||
# Markers
|
||||
markers =
|
||||
unit: Unit tests
|
||||
integration: Integration tests
|
||||
slow: Slow running tests
|
||||
|
||||
# Coverage options (overridden by pyproject.toml)
|
||||
addopts =
|
||||
--verbose
|
||||
--strict-markers
|
||||
--tb=short
|
||||
--cov=src
|
||||
--cov-report=term-missing
|
||||
--cov-report=html
|
||||
--cov-report=xml
|
||||
--cov-branch
|
||||
|
||||
# Ignore warnings from dependencies
|
||||
filterwarnings =
|
||||
ignore::DeprecationWarning
|
||||
@@ -1,25 +0,0 @@
|
||||
# Development and Testing Dependencies
|
||||
# Install with: pip install -r requirements.txt -r requirements-dev.txt
|
||||
|
||||
# Testing Framework
|
||||
# Latest pytest with async support
|
||||
pytest>=8.3,<8.4
|
||||
pytest-asyncio>=0.25,<0.26
|
||||
pytest-cov>=6.0,<6.1
|
||||
|
||||
# Test client for FastAPI
|
||||
httpx>=0.28,<0.29 # Already in requirements.txt but needed for test client
|
||||
|
||||
# Code Quality
|
||||
# Linting and formatting
|
||||
ruff>=0.8,<0.9
|
||||
|
||||
# Type checking
|
||||
mypy>=1.14,<1.15
|
||||
|
||||
# Testing utilities
|
||||
pytest-mock>=3.14,<3.15
|
||||
faker>=34.0,<35.0
|
||||
|
||||
# Coverage reporting
|
||||
coverage[toml]>=7.7,<7.8
|
||||
@@ -1,60 +0,0 @@
|
||||
# Core FastAPI framework and server
|
||||
# FastAPI: Modern, fast web framework for building APIs
|
||||
# Latest: 0.123.9 (Dec 4, 2025) - No known CVEs
|
||||
fastapi>=0.123,<0.124
|
||||
|
||||
# ASGI server for running FastAPI
|
||||
# Latest: 0.38.0 (Oct 18, 2025) - No known CVEs
|
||||
# Note: Old versions had CVE-2020-7694/7695, but 0.38.0 is secure
|
||||
uvicorn[standard]>=0.38,<0.39
|
||||
|
||||
# Additional dependencies
|
||||
# Pydantic for data validation (comes with pydantic-ai but pinning explicitly)
|
||||
# Updated to >=2.11 due to ag-ui-protocol dependency requirement
|
||||
# Latest: 2.12.4 (Nov 5, 2025) - No known CVEs
|
||||
pydantic>=2.11,<2.13
|
||||
|
||||
# Pydantic settings for configuration management
|
||||
# Required explicitly since pydantic-ai-slim doesn't include it
|
||||
# Latest: 2.12.0 (Dec 2025) - No known CVEs
|
||||
pydantic-settings>=2.12,<2.13
|
||||
|
||||
# AI/LLM integration
|
||||
# PydanticAI: Agent framework for using Pydantic with LLMs
|
||||
# Using slim version with openai (Ollama) and anthropic (Claude) extras
|
||||
# See DEPENDENCY_SLIM.md for rollback instructions if this breaks
|
||||
pydantic-ai-slim[openai,anthropic]>=1.27,<1.28
|
||||
|
||||
# HTTP client for Ollama communication
|
||||
# Latest: 0.28.1 - No known CVEs
|
||||
httpx>=0.28,<0.29
|
||||
|
||||
# Server-Sent Events for streaming responses
|
||||
# Required for OpenAI-compatible streaming endpoints
|
||||
# Latest: 3.0.2 (Oct 30, 2025) - No known CVEs
|
||||
sse-starlette>=3.0,<3.1
|
||||
|
||||
# Configuration management
|
||||
# Latest: 1.2.1 (Oct 26, 2025) - No known CVEs
|
||||
python-dotenv>=1.2,<1.3
|
||||
|
||||
# ASGI toolkit (dependency of FastAPI, pinning for security)
|
||||
starlette>=0.45,<0.46
|
||||
|
||||
# Redis for performance benchmarking and caching
|
||||
# Latest: 5.2.1 (Dec 5, 2025) - No known CVEs
|
||||
# hiredis: C parser for better performance
|
||||
redis[hiredis]>=5.2,<6.0
|
||||
|
||||
# Qdrant vector database client for memory storage
|
||||
# Latest: 1.12.1 (Dec 2025) - No known CVEs
|
||||
qdrant-client>=1.12,<2.0
|
||||
|
||||
# Structured logging for observability
|
||||
# Latest: 24.4.0 (Aug 22, 2024) - No known CVEs
|
||||
structlog>=24.1,<25.0
|
||||
|
||||
# Note on version locking strategy:
|
||||
# Using >=X.Y,<X.(Y+1) format to lock to minor versions
|
||||
# This protects against supply chain attacks while allowing patch updates
|
||||
# Update regularly and review changelogs before upgrading minor versions
|
||||
@@ -0,0 +1,542 @@
|
||||
"""
|
||||
Benchmark tool calling across different Ollama models via Tatlock API.
|
||||
|
||||
Sends test prompts through the full Tatlock pipeline (Steward -> Orchestration
|
||||
-> Synthesis) and records tool selection accuracy, latency, and response quality.
|
||||
|
||||
Between models, swaps OLLAMA_DEFAULT_MODEL in .env and waits for uvicorn
|
||||
auto-reload. Requires the server to be running via ./wakeup.sh.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python scripts/benchmark_tool_calling.py
|
||||
.venv/bin/python scripts/benchmark_tool_calling.py --models "gemma4:e4b,gemma4:e2b"
|
||||
.venv/bin/python scripts/benchmark_tool_calling.py --iterations 3
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
API_BASE = "http://localhost:8777"
|
||||
CHAT_URL = f"{API_BASE}/v1/chat/completions"
|
||||
HEALTH_URL = f"{API_BASE}/health"
|
||||
OLLAMA_URL = "http://localhost:11434"
|
||||
ENV_PATH = Path(__file__).parent.parent / ".env"
|
||||
|
||||
DEFAULT_MODELS = ["mistral-nemo-large:latest", "gemma4:e4b", "gemma4:e2b"]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test scenarios
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Scenario:
|
||||
name: str
|
||||
prompt: str
|
||||
expected_tool: str | None # None = no tool expected
|
||||
# Patterns to check in the response text for indirect tool-use evidence
|
||||
success_patterns: list[str] = field(default_factory=list)
|
||||
category: str = "basic"
|
||||
|
||||
|
||||
SCENARIOS = [
|
||||
# --- Should call calculate_math ---
|
||||
Scenario(
|
||||
name="Simple arithmetic",
|
||||
prompt="What is 144 divided by 12?",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["12"],
|
||||
category="calculator",
|
||||
),
|
||||
Scenario(
|
||||
name="Square root",
|
||||
prompt="What's the square root of 256?",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["16"],
|
||||
category="calculator",
|
||||
),
|
||||
Scenario(
|
||||
name="Complex math",
|
||||
prompt="Calculate pi times the square of 5",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["78.5"], # pi * 25 ≈ 78.54
|
||||
category="calculator",
|
||||
),
|
||||
Scenario(
|
||||
name="Word problem",
|
||||
prompt="If I have 3 bags with 17 apples each and I eat 4, how many apples do I have?",
|
||||
expected_tool="calculate_math",
|
||||
success_patterns=["47"],
|
||||
category="calculator",
|
||||
),
|
||||
|
||||
# --- Should call get_current_time ---
|
||||
Scenario(
|
||||
name="Current date",
|
||||
prompt="What's today's date?",
|
||||
expected_tool="get_current_time",
|
||||
success_patterns=["2026"], # Should contain current year
|
||||
category="datetime",
|
||||
),
|
||||
Scenario(
|
||||
name="Current time",
|
||||
prompt="What time is it right now?",
|
||||
expected_tool="get_current_time",
|
||||
success_patterns=[":"], # Time format contains colons
|
||||
category="datetime",
|
||||
),
|
||||
|
||||
# --- Should call calculate_date_offset ---
|
||||
Scenario(
|
||||
name="Relative date past",
|
||||
prompt="What was the date 2 weeks ago?",
|
||||
expected_tool="calculate_date_offset",
|
||||
success_patterns=["2026"],
|
||||
category="datetime",
|
||||
),
|
||||
|
||||
# --- Should call calculate_time_difference ---
|
||||
Scenario(
|
||||
name="Date difference",
|
||||
prompt="How many days between January 1st 2025 and March 15th 2025?",
|
||||
expected_tool="calculate_time_difference",
|
||||
success_patterns=["73", "74"], # 73 or 74 days
|
||||
category="datetime",
|
||||
),
|
||||
|
||||
# --- Should NOT call any tool ---
|
||||
Scenario(
|
||||
name="Greeting",
|
||||
prompt="Hello! How are you?",
|
||||
expected_tool=None,
|
||||
success_patterns=["sir"], # Butler personality
|
||||
category="no_tool",
|
||||
),
|
||||
Scenario(
|
||||
name="Knowledge question",
|
||||
prompt="What is the capital of France?",
|
||||
expected_tool=None,
|
||||
success_patterns=["Paris"],
|
||||
category="no_tool",
|
||||
),
|
||||
Scenario(
|
||||
name="Opinion request",
|
||||
prompt="What do you think about rainy days?",
|
||||
expected_tool=None,
|
||||
category="no_tool",
|
||||
),
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Result tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunResult:
|
||||
scenario: str
|
||||
model: str
|
||||
iteration: int
|
||||
latency: float
|
||||
response_text: str
|
||||
has_correct_answer: bool
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelStats:
|
||||
model: str
|
||||
results: list[RunResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return len(self.results)
|
||||
|
||||
@property
|
||||
def errors(self) -> int:
|
||||
return sum(1 for r in self.results if r.error)
|
||||
|
||||
@property
|
||||
def accuracy(self) -> float:
|
||||
valid = [r for r in self.results if not r.error]
|
||||
if not valid:
|
||||
return 0
|
||||
return sum(1 for r in valid if r.has_correct_answer) / len(valid) * 100
|
||||
|
||||
@property
|
||||
def avg_latency(self) -> float:
|
||||
lats = [r.latency for r in self.results if not r.error]
|
||||
return statistics.mean(lats) if lats else 0
|
||||
|
||||
@property
|
||||
def p95_latency(self) -> float:
|
||||
lats = sorted(r.latency for r in self.results if not r.error)
|
||||
if not lats:
|
||||
return 0
|
||||
return lats[min(int(len(lats) * 0.95), len(lats) - 1)]
|
||||
|
||||
@property
|
||||
def max_latency(self) -> float:
|
||||
lats = [r.latency for r in self.results if not r.error]
|
||||
return max(lats) if lats else 0
|
||||
|
||||
def category_accuracy(self, category: str) -> float:
|
||||
cat_scenarios = {s.name for s in SCENARIOS if s.category == category}
|
||||
valid = [r for r in self.results if not r.error and r.scenario in cat_scenarios]
|
||||
if not valid:
|
||||
return 0
|
||||
return sum(1 for r in valid if r.has_correct_answer) / len(valid) * 100
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# .env manipulation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def swap_model_in_env(model_name: str):
|
||||
"""Swap OLLAMA_DEFAULT_MODEL in .env file."""
|
||||
content = ENV_PATH.read_text()
|
||||
content = re.sub(
|
||||
r'^OLLAMA_DEFAULT_MODEL=.*$',
|
||||
f'OLLAMA_DEFAULT_MODEL={model_name}',
|
||||
content,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
ENV_PATH.write_text(content)
|
||||
print(f" .env updated: OLLAMA_DEFAULT_MODEL={model_name}")
|
||||
|
||||
|
||||
async def wait_for_server_reload(client: httpx.AsyncClient, timeout: float = 30):
|
||||
"""Wait for uvicorn to auto-reload after .env change."""
|
||||
# Give uvicorn a moment to detect the file change
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Poll health endpoint
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
r = await client.get(HEALTH_URL, timeout=5)
|
||||
if r.status_code == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(1)
|
||||
|
||||
raise TimeoutError("Server did not come back after reload")
|
||||
|
||||
|
||||
async def warm_up_ollama_model(client: httpx.AsyncClient, model_name: str):
|
||||
"""Send a throwaway request to load the model into VRAM."""
|
||||
print(f" Warming up {model_name} in Ollama...", end=" ", flush=True)
|
||||
try:
|
||||
r = await client.post(
|
||||
f"{OLLAMA_URL}/api/generate",
|
||||
json={"model": model_name, "prompt": "hi", "stream": False},
|
||||
timeout=120,
|
||||
)
|
||||
r.raise_for_status()
|
||||
duration = r.json().get("total_duration", 0) / 1e9
|
||||
print(f"OK ({duration:.1f}s)")
|
||||
except Exception as e:
|
||||
print(f"WARN: {e}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core benchmark logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def run_scenario(
|
||||
client: httpx.AsyncClient,
|
||||
scenario: Scenario,
|
||||
model: str,
|
||||
iteration: int,
|
||||
) -> RunResult:
|
||||
"""Run a single scenario through the Tatlock API."""
|
||||
payload = {
|
||||
"model": "Tatlock",
|
||||
"messages": [{"role": "user", "content": scenario.prompt}],
|
||||
}
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r = await client.post(CHAT_URL, json=payload, timeout=120)
|
||||
latency = time.monotonic() - start
|
||||
|
||||
if r.status_code != 200:
|
||||
return RunResult(
|
||||
scenario=scenario.name,
|
||||
model=model,
|
||||
iteration=iteration,
|
||||
latency=latency,
|
||||
response_text="",
|
||||
has_correct_answer=False,
|
||||
error=f"HTTP {r.status_code}: {r.text[:100]}",
|
||||
)
|
||||
|
||||
data = r.json()
|
||||
response_text = data["choices"][0]["message"]["content"]
|
||||
|
||||
# Check if the response contains expected patterns
|
||||
has_correct = True
|
||||
if scenario.success_patterns:
|
||||
has_correct = any(
|
||||
p.lower() in response_text.lower()
|
||||
for p in scenario.success_patterns
|
||||
)
|
||||
|
||||
return RunResult(
|
||||
scenario=scenario.name,
|
||||
model=model,
|
||||
iteration=iteration,
|
||||
latency=latency,
|
||||
response_text=response_text,
|
||||
has_correct_answer=has_correct,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
latency = time.monotonic() - start
|
||||
return RunResult(
|
||||
scenario=scenario.name,
|
||||
model=model,
|
||||
iteration=iteration,
|
||||
latency=latency,
|
||||
response_text="",
|
||||
has_correct_answer=False,
|
||||
error=str(e)[:200],
|
||||
)
|
||||
|
||||
|
||||
async def benchmark_model(
|
||||
client: httpx.AsyncClient,
|
||||
model_name: str,
|
||||
iterations: int,
|
||||
) -> ModelStats:
|
||||
"""Run all scenarios for a single model."""
|
||||
stats = ModelStats(model=model_name)
|
||||
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f" Model: {model_name}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
# Swap model in .env
|
||||
swap_model_in_env(model_name)
|
||||
|
||||
# Warm up model in Ollama BEFORE server reload picks it up
|
||||
await warm_up_ollama_model(client, model_name)
|
||||
|
||||
# Wait for server to reload with new model
|
||||
print(" Waiting for server reload...", end=" ", flush=True)
|
||||
await wait_for_server_reload(client)
|
||||
print("OK")
|
||||
|
||||
# Run a throwaway request through the full pipeline to warm up
|
||||
print(" Warming up pipeline...", end=" ", flush=True)
|
||||
try:
|
||||
await client.post(
|
||||
CHAT_URL,
|
||||
json={"model": "Tatlock", "messages": [{"role": "user", "content": "hi"}]},
|
||||
timeout=120,
|
||||
)
|
||||
print("OK")
|
||||
except Exception as e:
|
||||
print(f"WARN: {e}")
|
||||
|
||||
for iteration in range(iterations):
|
||||
if iterations > 1:
|
||||
print(f"\n --- Iteration {iteration + 1}/{iterations} ---")
|
||||
|
||||
for scenario in SCENARIOS:
|
||||
result = await run_scenario(client, scenario, model_name, iteration)
|
||||
stats.results.append(result)
|
||||
|
||||
# Display
|
||||
if result.error:
|
||||
print(
|
||||
f" [ERR ] {scenario.name:30s} {result.latency:5.1f}s "
|
||||
f"{result.error[:60]}"
|
||||
)
|
||||
elif result.has_correct_answer:
|
||||
preview = result.response_text[:60].replace("\n", " ")
|
||||
print(f" [OK ] {scenario.name:30s} {result.latency:5.1f}s {preview}")
|
||||
else:
|
||||
preview = result.response_text[:60].replace("\n", " ")
|
||||
print(f" [MISS] {scenario.name:30s} {result.latency:5.1f}s {preview}")
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def print_comparison(all_stats: list[ModelStats]):
|
||||
"""Print side-by-side comparison table."""
|
||||
print("\n" + "=" * 80)
|
||||
print(" COMPARISON SUMMARY")
|
||||
print("=" * 80)
|
||||
|
||||
col_width = max(len(s.model) for s in all_stats) + 2
|
||||
label_width = 32
|
||||
|
||||
header = f"{'Metric':<{label_width}}"
|
||||
for s in all_stats:
|
||||
header += f" {s.model:>{col_width}}"
|
||||
print(f"\n{header}")
|
||||
print("-" * (label_width + (col_width + 2) * len(all_stats)))
|
||||
|
||||
# Answer accuracy
|
||||
row = f"{'Correct answer rate':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.accuracy:>{col_width - 1}.1f}%"
|
||||
print(row)
|
||||
|
||||
# Latency
|
||||
row = f"{'Avg latency':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.avg_latency:>{col_width - 1}.1f}s"
|
||||
print(row)
|
||||
|
||||
row = f"{'P95 latency':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.p95_latency:>{col_width - 1}.1f}s"
|
||||
print(row)
|
||||
|
||||
row = f"{'Max latency':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.max_latency:>{col_width - 1}.1f}s"
|
||||
print(row)
|
||||
|
||||
# Errors
|
||||
row = f"{'Errors':<{label_width}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.errors:>{col_width}}"
|
||||
print(row)
|
||||
|
||||
# Per-category
|
||||
categories = sorted(set(sc.category for sc in SCENARIOS))
|
||||
print(f"\n{'Per-category accuracy':<{label_width}}")
|
||||
print("-" * (label_width + (col_width + 2) * len(all_stats)))
|
||||
for cat in categories:
|
||||
row = f" {cat:<{label_width - 2}}"
|
||||
for s in all_stats:
|
||||
row += f" {s.category_accuracy(cat):>{col_width - 1}.1f}%"
|
||||
print(row)
|
||||
|
||||
# Mismatches
|
||||
print(f"\n{'Missed answers':<50}")
|
||||
print("-" * 80)
|
||||
any_miss = False
|
||||
for scenario in SCENARIOS:
|
||||
misses = []
|
||||
for s in all_stats:
|
||||
sc_results = [r for r in s.results if r.scenario == scenario.name]
|
||||
fails = [r for r in sc_results if not r.has_correct_answer and not r.error]
|
||||
if fails:
|
||||
preview = fails[0].response_text[:50].replace("\n", " ")
|
||||
misses.append(f"{s.model}: \"{preview}\"")
|
||||
if misses:
|
||||
any_miss = True
|
||||
print(f" {scenario.name}")
|
||||
for m in misses:
|
||||
print(f" {m}")
|
||||
|
||||
if not any_miss:
|
||||
print(" (none)")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
|
||||
|
||||
def save_results(all_stats: list[ModelStats], output_path: Path):
|
||||
"""Save detailed results to JSON."""
|
||||
data = {}
|
||||
for stats in all_stats:
|
||||
data[stats.model] = {
|
||||
"summary": {
|
||||
"accuracy": stats.accuracy,
|
||||
"avg_latency": round(stats.avg_latency, 2),
|
||||
"p95_latency": round(stats.p95_latency, 2),
|
||||
"max_latency": round(stats.max_latency, 2),
|
||||
"errors": stats.errors,
|
||||
"total_runs": stats.total,
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"scenario": r.scenario,
|
||||
"iteration": r.iteration,
|
||||
"latency": round(r.latency, 3),
|
||||
"has_correct_answer": r.has_correct_answer,
|
||||
"response_text": r.response_text,
|
||||
"error": r.error,
|
||||
}
|
||||
for r in stats.results
|
||||
],
|
||||
}
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(data, indent=2))
|
||||
print(f"\nDetailed results saved to: {output_path}")
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description="Benchmark tool calling across Ollama models via Tatlock API")
|
||||
parser.add_argument(
|
||||
"--iterations", type=int, default=1,
|
||||
help="Iterations per model (default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--models", type=str, default=",".join(DEFAULT_MODELS),
|
||||
help=f"Comma-separated models (default: {','.join(DEFAULT_MODELS)})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", type=str, default="logs/benchmark_results.json",
|
||||
help="JSON output path (default: logs/benchmark_results.json)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
models = [m.strip() for m in args.models.split(",")]
|
||||
|
||||
# Verify server is running
|
||||
async with httpx.AsyncClient() as client:
|
||||
try:
|
||||
r = await client.get(HEALTH_URL, timeout=5)
|
||||
r.raise_for_status()
|
||||
print("Server is running.")
|
||||
except Exception:
|
||||
print("ERROR: Server not running. Start it with ./wakeup.sh first.")
|
||||
return
|
||||
|
||||
print("=" * 70)
|
||||
print(" Tool Calling Benchmark (via Tatlock API)")
|
||||
print("=" * 70)
|
||||
print(f" Models: {', '.join(models)}")
|
||||
print(f" Scenarios: {len(SCENARIOS)}")
|
||||
print(f" Iterations: {args.iterations}")
|
||||
print(f" Total runs: {len(SCENARIOS) * args.iterations * len(models)}")
|
||||
|
||||
# Remember original model to restore after benchmark
|
||||
original_env = ENV_PATH.read_text()
|
||||
|
||||
all_stats = []
|
||||
async with httpx.AsyncClient() as client:
|
||||
for model in models:
|
||||
stats = await benchmark_model(client, model, args.iterations)
|
||||
all_stats.append(stats)
|
||||
|
||||
# Restore original .env
|
||||
ENV_PATH.write_text(original_env)
|
||||
print(f"\n .env restored to original")
|
||||
|
||||
print_comparison(all_stats)
|
||||
save_results(all_stats, Path(args.output))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -204,13 +204,13 @@ async def run_housekeeper(
|
||||
)
|
||||
|
||||
try:
|
||||
# Use temperature 0.1 for slight exploration
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
# Temperature 0.1 for slight exploration (skipped on Claude backend)
|
||||
from src.anthropic.model_selector import get_sampling_settings
|
||||
|
||||
result = await agent.run(
|
||||
prompt,
|
||||
message_history=message_history,
|
||||
model_settings=ModelSettings(temperature=0.1),
|
||||
model_settings=get_sampling_settings(0.1),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -266,13 +266,13 @@ async def run_housekeeper_stream(
|
||||
)
|
||||
|
||||
try:
|
||||
# Use temperature 0.1 for slight exploration
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
# Temperature 0.1 for slight exploration (skipped on Claude backend)
|
||||
from src.anthropic.model_selector import get_sampling_settings
|
||||
|
||||
async with agent.run_stream(
|
||||
prompt,
|
||||
message_history=message_history,
|
||||
model_settings=ModelSettings(temperature=0.1),
|
||||
model_settings=get_sampling_settings(0.1),
|
||||
) as response:
|
||||
async for delta in response.stream_text(delta=True):
|
||||
yield delta
|
||||
|
||||
+28
-14
@@ -11,7 +11,7 @@ Uses plain text output (not JSON) for reliability. Supports both Claude
|
||||
import httpx
|
||||
from typing import Optional
|
||||
|
||||
from src.anthropic.model_selector import is_claude_available, get_model_info
|
||||
from src.anthropic.model_selector import get_model_info, is_claude_available, resolve_backend
|
||||
from src.core.config import config
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
@@ -113,18 +113,19 @@ class StewardAgent:
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Steward with backend selection based on availability."""
|
||||
# Ollama config (fallback)
|
||||
# Ollama config (primary)
|
||||
self.ollama_host = str(config.OLLAMA_HOST).rstrip('/')
|
||||
self.ollama_model = config.OLLAMA_DEFAULT_MODEL
|
||||
|
||||
# Claude config (preferred)
|
||||
# Claude config (fallback)
|
||||
self.claude_model = config.ANTHROPIC_MODEL
|
||||
self._anthropic_client = None
|
||||
|
||||
# Determine which backend to use
|
||||
self._use_claude = config.PREFER_CLOUD_BACKEND and is_claude_available()
|
||||
# Determine which backend to use (Ollama-first, Claude when
|
||||
# preferred via config or when Ollama is down)
|
||||
self._use_claude = resolve_backend() == "claude"
|
||||
|
||||
self.timeout = 30.0 # 30 second timeout for analysis
|
||||
self.timeout = float(config.STEWARD_TIMEOUT)
|
||||
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
@@ -145,12 +146,12 @@ class StewardAgent:
|
||||
"""Call Claude API directly for plain text generation."""
|
||||
client = self._get_anthropic_client()
|
||||
|
||||
# No temperature: rejected by Claude Sonnet 5+ (sampling params deprecated)
|
||||
response = await client.messages.create(
|
||||
model=self.claude_model,
|
||||
max_tokens=1024,
|
||||
system=system_prompt,
|
||||
messages=[{"role": "user", "content": user_message}],
|
||||
temperature=0.3, # Lower = more consistent
|
||||
)
|
||||
|
||||
return response.content[0].text.strip()
|
||||
@@ -227,20 +228,33 @@ class StewardAgent:
|
||||
return analysis_text
|
||||
|
||||
except Exception as e:
|
||||
# If Claude fails, try Ollama as fallback
|
||||
# Mid-request fallback: retry on the other backend when possible
|
||||
if self._use_claude:
|
||||
logger.warning(
|
||||
"steward_claude_fallback",
|
||||
error=str(e),
|
||||
)
|
||||
analysis_text = await self._call_ollama(prompt)
|
||||
logger.debug(
|
||||
"steward_analysis_received",
|
||||
backend="ollama_fallback",
|
||||
text_preview=analysis_text[:150],
|
||||
fallback_backend = "ollama_fallback"
|
||||
elif is_claude_available():
|
||||
logger.warning(
|
||||
"steward_ollama_fallback",
|
||||
error=str(e),
|
||||
)
|
||||
return analysis_text
|
||||
raise
|
||||
analysis_text = await self._call_claude(
|
||||
system_prompt="You are the Steward of the household, advising the Butler (Tatlock) on which capabilities to use. Be concise and specific.",
|
||||
user_message=prompt,
|
||||
)
|
||||
fallback_backend = "claude_fallback"
|
||||
else:
|
||||
raise
|
||||
|
||||
logger.debug(
|
||||
"steward_analysis_received",
|
||||
backend=fallback_backend,
|
||||
text_preview=analysis_text[:150],
|
||||
)
|
||||
return analysis_text
|
||||
|
||||
|
||||
# Global Steward instance
|
||||
|
||||
+23
-7
@@ -129,6 +129,22 @@ or
|
||||
"""
|
||||
|
||||
|
||||
# Tool-phase prompt for orchestrate_tool_calls(). The butler personality prompt
|
||||
# suppresses tool calling on small local models (gemma4 reasons about the tool,
|
||||
# then answers from memory with wrong arithmetic), so the orchestration phase
|
||||
# uses a terse operator prompt; synthesize_from_results() applies the persona.
|
||||
TATLOCK_ORCHESTRATION_PROMPT = """You are the tool-execution phase of Tatlock, \
|
||||
a butler assistant. Your only job is to gather accurate results by calling the \
|
||||
provided tools.
|
||||
|
||||
- ALWAYS use tools for the task - never answer from memory and never do mental math.
|
||||
- Mathematics: call the calculate tool, even for trivial arithmetic.
|
||||
- Dates and times: call the date/time tools, never guess.
|
||||
- When the instructions say DELEGATE to an agent, call the matching delegate_to_* tool.
|
||||
- After the tool results arrive, reply with a one-line factual summary of the results. \
|
||||
A later step writes the polished reply, so do not add personality."""
|
||||
|
||||
|
||||
class TatlockAgent(AgentInterface):
|
||||
"""
|
||||
Tatlock - The Butler agent using PydanticAI with Ollama.
|
||||
@@ -494,13 +510,13 @@ class TatlockAgent(AgentInterface):
|
||||
)
|
||||
|
||||
# Run with scoped tools and tracker
|
||||
# Force tool_choice: required to make LLM actually call tools
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
# Force tool_choice to make LLM actually call tools
|
||||
from src.anthropic.model_selector import get_tool_choice_settings
|
||||
result = await scoped_agent.run(
|
||||
enriched_message,
|
||||
message_history=pydantic_history if pydantic_history else None,
|
||||
deps=tool_tracker,
|
||||
model_settings=ModelSettings(extra_body={"tool_choice": "required"})
|
||||
model_settings=get_tool_choice_settings(),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
@@ -624,7 +640,6 @@ class TatlockAgent(AgentInterface):
|
||||
- tool_outputs: Dict mapping tool names to their outputs
|
||||
- raw_output: The agent's raw text output
|
||||
"""
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
from pydantic_ai.messages import (
|
||||
ModelRequest,
|
||||
ModelResponse,
|
||||
@@ -655,10 +670,10 @@ class TatlockAgent(AgentInterface):
|
||||
# Create a fresh agent instance with scoped tools only
|
||||
model = get_model()
|
||||
|
||||
# Create agent with scoped tools
|
||||
# Create agent with scoped tools, using the tool-phase prompt
|
||||
scoped_agent = Agent(
|
||||
model,
|
||||
system_prompt=TATLOCK_SYSTEM_PROMPT,
|
||||
system_prompt=TATLOCK_ORCHESTRATION_PROMPT,
|
||||
tools=scoped_tools,
|
||||
)
|
||||
|
||||
@@ -684,11 +699,12 @@ class TatlockAgent(AgentInterface):
|
||||
)
|
||||
|
||||
# Run with scoped tools and tracker
|
||||
from src.anthropic.model_selector import get_tool_choice_settings
|
||||
result = await scoped_agent.run(
|
||||
enriched_message,
|
||||
message_history=pydantic_history if pydantic_history else None,
|
||||
deps=tool_tracker,
|
||||
model_settings=ModelSettings(extra_body={"tool_choice": "required"})
|
||||
model_settings=get_tool_choice_settings(),
|
||||
)
|
||||
|
||||
# Extract tool calls and results from the agent's messages
|
||||
|
||||
@@ -1,17 +1,26 @@
|
||||
"""
|
||||
Anthropic/Claude integration module.
|
||||
|
||||
Provides model selection with automatic fallback between Claude and Ollama.
|
||||
Provides model selection with Ollama as primary backend and Claude
|
||||
as the cloud fallback.
|
||||
"""
|
||||
|
||||
from src.anthropic.model_selector import (
|
||||
check_claude_health,
|
||||
check_ollama_health,
|
||||
get_model,
|
||||
get_tool_choice_settings,
|
||||
is_claude_available,
|
||||
is_ollama_available,
|
||||
resolve_backend,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"check_claude_health",
|
||||
"check_ollama_health",
|
||||
"get_model",
|
||||
"get_tool_choice_settings",
|
||||
"is_claude_available",
|
||||
"is_ollama_available",
|
||||
"resolve_backend",
|
||||
]
|
||||
|
||||
+170
-29
@@ -1,23 +1,84 @@
|
||||
"""
|
||||
Model selector for Claude/Ollama backend switching.
|
||||
Model selector for Ollama/Claude backend switching.
|
||||
|
||||
Provides automatic model selection with Claude as preferred backend
|
||||
and Ollama as offline fallback.
|
||||
Provides automatic model selection with Ollama as the primary local backend
|
||||
and Claude as the cloud fallback. Claude is used when PREFER_CLOUD_BACKEND
|
||||
is enabled, or automatically when Ollama is unavailable at startup.
|
||||
|
||||
The Anthropic SDK is imported lazily so a missing or broken `anthropic`
|
||||
package degrades to Ollama-only operation instead of crashing the app.
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from pydantic_ai.providers.anthropic import AnthropicProvider
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import httpx
|
||||
|
||||
from src.core.config import config
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Cached health check result (set once at startup)
|
||||
# Cached health check results (set once at startup)
|
||||
_claude_available: bool | None = None
|
||||
_ollama_available: bool | None = None
|
||||
|
||||
|
||||
async def check_ollama_health() -> bool:
|
||||
"""
|
||||
Check if the Ollama server is reachable and has the configured model.
|
||||
|
||||
This should be called once at application startup.
|
||||
The result is cached in `_ollama_available`.
|
||||
|
||||
Returns:
|
||||
True if Ollama is reachable and OLLAMA_DEFAULT_MODEL is pulled.
|
||||
"""
|
||||
global _ollama_available
|
||||
|
||||
host = str(config.OLLAMA_HOST).rstrip("/")
|
||||
model = config.OLLAMA_DEFAULT_MODEL
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
response = await client.get(f"{host}/api/tags")
|
||||
response.raise_for_status()
|
||||
names = [m.get("name", "") for m in response.json().get("models", [])]
|
||||
|
||||
if model in names or f"{model}:latest" in names:
|
||||
_ollama_available = True
|
||||
logger.info(
|
||||
"ollama_health_check_passed",
|
||||
host=host,
|
||||
model=model,
|
||||
)
|
||||
return True
|
||||
|
||||
_ollama_available = False
|
||||
logger.warning(
|
||||
"ollama_health_check_failed",
|
||||
reason="model_not_pulled",
|
||||
host=host,
|
||||
model=model,
|
||||
hint=f"run `ollama pull {model}`",
|
||||
)
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
_ollama_available = False
|
||||
logger.warning(
|
||||
"ollama_health_check_failed",
|
||||
reason="server_unreachable",
|
||||
host=host,
|
||||
error=str(e),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
async def check_claude_health() -> bool:
|
||||
@@ -85,46 +146,93 @@ def is_claude_available() -> bool:
|
||||
return _claude_available is True
|
||||
|
||||
|
||||
def get_model(prefer_cloud: bool | None = None) -> Union[AnthropicModel, OpenAIChatModel]:
|
||||
def is_ollama_available() -> bool:
|
||||
"""
|
||||
Check if Ollama is available (from cached health check result).
|
||||
|
||||
Returns:
|
||||
False only if the startup health check confirmed Ollama is down.
|
||||
Unknown (check not run yet) counts as available so that contexts
|
||||
without lifespan events keep the local-first behavior.
|
||||
"""
|
||||
return _ollama_available is not False
|
||||
|
||||
|
||||
def resolve_backend(prefer_cloud: bool | None = None) -> str:
|
||||
"""
|
||||
Resolve which backend should serve requests.
|
||||
|
||||
Ollama is the primary backend. Claude is used when explicitly
|
||||
preferred via PREFER_CLOUD_BACKEND, or as automatic fallback
|
||||
when the startup health check found Ollama down.
|
||||
|
||||
Args:
|
||||
prefer_cloud: Override config.PREFER_CLOUD_BACKEND for this call.
|
||||
|
||||
Returns:
|
||||
"claude" or "ollama".
|
||||
"""
|
||||
use_cloud = prefer_cloud if prefer_cloud is not None else config.PREFER_CLOUD_BACKEND
|
||||
|
||||
if use_cloud and is_claude_available():
|
||||
return "claude"
|
||||
|
||||
if not is_ollama_available() and is_claude_available():
|
||||
logger.warning(
|
||||
"backend_fallback_to_claude",
|
||||
reason="ollama_unavailable",
|
||||
)
|
||||
return "claude"
|
||||
|
||||
return "ollama"
|
||||
|
||||
|
||||
def get_model(prefer_cloud: bool | None = None) -> AnthropicModel | OpenAIChatModel:
|
||||
"""
|
||||
Get the best available model.
|
||||
|
||||
Returns Claude if available and preferred, otherwise Ollama.
|
||||
Returns Ollama unless Claude is preferred (or Ollama is down).
|
||||
|
||||
Args:
|
||||
prefer_cloud: Override config.PREFER_CLOUD_BACKEND for this call.
|
||||
If None, uses the config value.
|
||||
|
||||
Returns:
|
||||
PydanticAI model instance (AnthropicModel or OpenAIChatModel).
|
||||
PydanticAI model instance (OpenAIChatModel or AnthropicModel).
|
||||
|
||||
Example:
|
||||
>>> model = get_model()
|
||||
>>> agent = Agent(model, system_prompt="...")
|
||||
"""
|
||||
# Determine preference
|
||||
use_cloud = prefer_cloud if prefer_cloud is not None else config.PREFER_CLOUD_BACKEND
|
||||
if resolve_backend(prefer_cloud) == "claude":
|
||||
try:
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
from pydantic_ai.providers.anthropic import AnthropicProvider
|
||||
|
||||
# Use Claude if available and preferred
|
||||
if use_cloud and is_claude_available():
|
||||
logger.debug(
|
||||
"model_selected",
|
||||
backend="claude",
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return AnthropicModel(
|
||||
model_name=config.ANTHROPIC_MODEL,
|
||||
provider=AnthropicProvider(api_key=config.ANTHROPIC_API_KEY),
|
||||
)
|
||||
logger.debug(
|
||||
"model_selected",
|
||||
backend="claude",
|
||||
model=config.ANTHROPIC_MODEL,
|
||||
)
|
||||
return AnthropicModel(
|
||||
model_name=config.ANTHROPIC_MODEL,
|
||||
provider=AnthropicProvider(api_key=config.ANTHROPIC_API_KEY),
|
||||
)
|
||||
except ImportError as e:
|
||||
logger.error(
|
||||
"claude_backend_import_failed",
|
||||
error=str(e),
|
||||
hint="anthropic package missing or incompatible; using Ollama",
|
||||
)
|
||||
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
|
||||
# Fall back to Ollama
|
||||
from src.ollama.provider import get_ollama_provider
|
||||
|
||||
logger.debug(
|
||||
"model_selected",
|
||||
backend="ollama",
|
||||
model=config.OLLAMA_DEFAULT_MODEL,
|
||||
reason="fallback" if use_cloud else "preferred_local",
|
||||
)
|
||||
return OpenAIChatModel(
|
||||
model_name=config.OLLAMA_DEFAULT_MODEL,
|
||||
@@ -132,6 +240,37 @@ def get_model(prefer_cloud: bool | None = None) -> Union[AnthropicModel, OpenAIC
|
||||
)
|
||||
|
||||
|
||||
def get_tool_choice_settings() -> ModelSettings:
|
||||
"""
|
||||
Get model_settings for forcing tool calls on the first request.
|
||||
|
||||
For Claude: PydanticAI handles tool_choice natively, so no extra_body needed.
|
||||
For Ollama: Pass tool_choice="required" via extra_body to force tool calling.
|
||||
"""
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
if resolve_backend() == "claude":
|
||||
# PydanticAI's Anthropic model handles tool_choice internally
|
||||
return ModelSettings()
|
||||
else:
|
||||
# Ollama needs explicit tool_choice via extra_body
|
||||
return ModelSettings(extra_body={"tool_choice": "required"})
|
||||
|
||||
|
||||
def get_sampling_settings(temperature: float) -> ModelSettings:
|
||||
"""
|
||||
Get model_settings with a sampling temperature where the backend allows it.
|
||||
|
||||
Ollama accepts a temperature; Claude Sonnet 5+ rejects sampling
|
||||
parameters, so the Claude backend gets empty settings.
|
||||
"""
|
||||
from pydantic_ai.settings import ModelSettings
|
||||
|
||||
if resolve_backend() == "claude":
|
||||
return ModelSettings()
|
||||
return ModelSettings(temperature=temperature)
|
||||
|
||||
|
||||
def get_model_info() -> dict:
|
||||
"""
|
||||
Get information about the current model configuration.
|
||||
@@ -141,12 +280,14 @@ def get_model_info() -> dict:
|
||||
Returns:
|
||||
Dict with backend, model name, and availability info.
|
||||
"""
|
||||
use_cloud = config.PREFER_CLOUD_BACKEND and is_claude_available()
|
||||
backend = resolve_backend()
|
||||
|
||||
return {
|
||||
"backend": "claude" if use_cloud else "ollama",
|
||||
"model": config.ANTHROPIC_MODEL if use_cloud else config.OLLAMA_DEFAULT_MODEL,
|
||||
"backend": backend,
|
||||
"model": config.ANTHROPIC_MODEL if backend == "claude" else config.OLLAMA_DEFAULT_MODEL,
|
||||
"claude_available": is_claude_available(),
|
||||
"claude_configured": bool(config.ANTHROPIC_API_KEY),
|
||||
"ollama_available": is_ollama_available(),
|
||||
"ollama_model": config.OLLAMA_DEFAULT_MODEL,
|
||||
"prefer_cloud": config.PREFER_CLOUD_BACKEND,
|
||||
}
|
||||
|
||||
+22
-18
@@ -7,7 +7,7 @@ import logging
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from fastapi import APIRouter
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from src.chat import service
|
||||
from src.chat.schemas import (
|
||||
@@ -22,47 +22,51 @@ router = APIRouter(prefix="/chat", tags=["chat"])
|
||||
|
||||
async def _stream_response(
|
||||
request: ChatCompletionRequest,
|
||||
) -> AsyncGenerator[dict, None]:
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Generate SSE stream for chat completion.
|
||||
|
||||
EventSourceResponse adds "data: " prefix automatically.
|
||||
We just yield the dict/string content.
|
||||
Yields raw SSE-formatted strings matching OpenAI's format exactly:
|
||||
data: {json}\n\n
|
||||
"""
|
||||
try:
|
||||
async for chunk in service.create_chat_completion_stream(request):
|
||||
# Yield dict - EventSourceResponse will format as SSE
|
||||
yield {"data": chunk.model_dump_json()}
|
||||
yield f"data: {chunk.model_dump_json(exclude_unset=True)}\n\n"
|
||||
|
||||
# Send [DONE] message
|
||||
yield {"data": "[DONE]"}
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in streaming response: {e}")
|
||||
error_data = {"error": {"message": str(e), "type": "internal_error"}}
|
||||
yield {"data": json.dumps(error_data)}
|
||||
error_data = json.dumps({"error": {"message": str(e), "type": "internal_error"}})
|
||||
yield f"data: {error_data}\n\n"
|
||||
|
||||
|
||||
@router.post("/completions", response_model=ChatCompletionResponse)
|
||||
async def create_chat_completion(
|
||||
request: ChatCompletionRequest,
|
||||
) -> ChatCompletionResponse | EventSourceResponse:
|
||||
) -> ChatCompletionResponse | StreamingResponse:
|
||||
"""
|
||||
Create chat completion (OpenAI-compatible).
|
||||
|
||||
|
||||
Supports both regular and streaming responses.
|
||||
Currently returns mock lorem ipsum responses.
|
||||
|
||||
|
||||
Args:
|
||||
request: Chat completion request
|
||||
|
||||
|
||||
Returns:
|
||||
Chat completion response or SSE stream
|
||||
"""
|
||||
logger.info(f"Chat completion request for model: {request.model}")
|
||||
|
||||
|
||||
if request.stream:
|
||||
logger.info("Streaming response requested")
|
||||
return EventSourceResponse(_stream_response(request))
|
||||
|
||||
return StreamingResponse(
|
||||
_stream_response(request),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-store",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
return await service.create_chat_completion(request)
|
||||
|
||||
+12
-8
@@ -64,33 +64,37 @@ class Config(BaseSettings):
|
||||
API_PORT: int = Field(default=8000, description="API port")
|
||||
API_PREFIX: str = Field(default="/v1", description="API route prefix")
|
||||
|
||||
# Anthropic Configuration (Claude - preferred backend)
|
||||
# Anthropic Configuration (Claude - cloud fallback)
|
||||
ANTHROPIC_API_KEY: str | None = Field(
|
||||
default=None,
|
||||
description="Anthropic API key for Claude access"
|
||||
description="Anthropic API key for the Claude fallback backend"
|
||||
)
|
||||
ANTHROPIC_MODEL: str = Field(
|
||||
default="claude-sonnet-4-20250514",
|
||||
description="Claude model to use"
|
||||
default="claude-sonnet-5",
|
||||
description="Claude model for the fallback backend"
|
||||
)
|
||||
PREFER_CLOUD_BACKEND: bool = Field(
|
||||
default=True,
|
||||
description="Prefer Claude over Ollama when available"
|
||||
default=False,
|
||||
description="Prefer Claude over Ollama (default: local-first)"
|
||||
)
|
||||
|
||||
# Ollama Configuration (local fallback)
|
||||
# Ollama Configuration (local - primary backend)
|
||||
OLLAMA_HOST: HttpUrl = Field(
|
||||
default="http://localhost:11434",
|
||||
description="Ollama server URL"
|
||||
)
|
||||
OLLAMA_DEFAULT_MODEL: str = Field(
|
||||
default="mistral-nemo:latest",
|
||||
default="gemma4:e2b",
|
||||
description="Default Ollama model"
|
||||
)
|
||||
OLLAMA_TIMEOUT: int = Field(
|
||||
default=120,
|
||||
description="Ollama request timeout in seconds"
|
||||
)
|
||||
STEWARD_TIMEOUT: int = Field(
|
||||
default=60,
|
||||
description="Steward analysis timeout in seconds (gemma4 needs ~35s warm)"
|
||||
)
|
||||
STREAM_TIMEOUT: int = Field(
|
||||
default=20,
|
||||
description="Timeout for each streaming turn in seconds"
|
||||
|
||||
+9
-3
@@ -9,7 +9,11 @@ from src.agents.biographer import register_biographer
|
||||
from src.agents.housekeeper import register_housekeeper
|
||||
from src.agents.librarian import register_librarian
|
||||
from src.agents.tatlock_core import TATLOCK_CORE_CAPABILITY, tatlock_core_tools
|
||||
from src.anthropic.model_selector import check_claude_health, get_model_info
|
||||
from src.anthropic.model_selector import (
|
||||
check_claude_health,
|
||||
check_ollama_health,
|
||||
get_model_info,
|
||||
)
|
||||
from src.core.household_registry import get_household_registry
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
@@ -87,7 +91,7 @@ async def initialize_application():
|
||||
Initialize the application.
|
||||
|
||||
Performs all startup tasks:
|
||||
1. Check Claude API health (for backend selection)
|
||||
1. Check Ollama (primary) and Claude (fallback) health for backend selection
|
||||
2. Register household members
|
||||
3. (Future) Initialize connections
|
||||
|
||||
@@ -95,13 +99,15 @@ async def initialize_application():
|
||||
"""
|
||||
logger.info("application_initialization_starting")
|
||||
|
||||
# Check Claude API health for backend selection
|
||||
# Check backend health: Ollama is primary, Claude is the fallback
|
||||
await check_ollama_health()
|
||||
await check_claude_health()
|
||||
model_info = get_model_info()
|
||||
logger.info(
|
||||
"model_backend_configured",
|
||||
backend=model_info["backend"],
|
||||
model=model_info["model"],
|
||||
ollama_available=model_info["ollama_available"],
|
||||
claude_available=model_info["claude_available"],
|
||||
)
|
||||
|
||||
|
||||
@@ -651,15 +651,13 @@ async def create_response_with_steward(request: ResponseRequest) -> Response:
|
||||
# Build response output items
|
||||
output_items = []
|
||||
|
||||
# Add Steward reasoning as a reasoning output item
|
||||
output_items.append(ReasoningOutputItem(
|
||||
id=f"reasoning_{generate_id()}",
|
||||
summary=[
|
||||
"🎩 Steward's Analysis:",
|
||||
enriched.steward_reasoning,
|
||||
],
|
||||
status="completed"
|
||||
))
|
||||
# Add Steward reasoning as reasoning output
|
||||
if enriched.steward_reasoning:
|
||||
output_items.append(ReasoningOutputItem(
|
||||
id=f"rs_{generate_id()}",
|
||||
summary=[enriched.steward_reasoning],
|
||||
status="completed"
|
||||
))
|
||||
|
||||
# Add Tatlock's message
|
||||
output_items.append(MessageOutputItem(
|
||||
|
||||
@@ -166,26 +166,6 @@ class StreamingCoordinator:
|
||||
conversation_id=conversation_id,
|
||||
)
|
||||
|
||||
# Stream Steward's analysis as reasoning summary
|
||||
steward_lines = enriched.steward_reasoning.split('\n')
|
||||
for line in steward_lines:
|
||||
if line.strip():
|
||||
yield ReasoningSummaryDelta(delta=line + "\n")
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
yield ReasoningSummaryDone()
|
||||
|
||||
# Add Steward reasoning to output items
|
||||
reasoning_item = ReasoningOutputItem(
|
||||
id=f"reasoning_{generate_id()}",
|
||||
summary=[
|
||||
"🎩 Steward's Analysis:",
|
||||
enriched.steward_reasoning,
|
||||
],
|
||||
status="completed"
|
||||
)
|
||||
output_items.append(reasoning_item)
|
||||
|
||||
# Initialize tool tracker
|
||||
tracker = ToolCallTracker(
|
||||
recommended_capabilities=enriched.recommendation.recommended_capabilities,
|
||||
|
||||
@@ -9,13 +9,13 @@ import pytest
|
||||
|
||||
from src.agents.steward.schemas import ConversationContext, StewardRecommendation
|
||||
from src.agents.steward.service import analyze_request, format_steward_note, _build_enriched_query
|
||||
from src.core.startup import initialize_application
|
||||
from src.core.startup import register_household_members
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def setup_household_registry():
|
||||
"""Initialize household registry before running tests."""
|
||||
initialize_application()
|
||||
register_household_members()
|
||||
|
||||
|
||||
class TestAnalyzeRequest:
|
||||
@@ -29,17 +29,14 @@ class TestAnalyzeRequest:
|
||||
mock_agent.analyze = AsyncMock(return_value="Simple greeting requires no tools. This is a simple request.")
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"Hello!",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"Hello!",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert result.recommended_capabilities == []
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert mock_agent.analyze.called
|
||||
assert result.recommended_capabilities == []
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert mock_agent.analyze.called
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_math_request(self):
|
||||
@@ -50,16 +47,13 @@ class TestAnalyzeRequest:
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"What's sqrt(144)?",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"What's sqrt(144)?",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert "tatlock_core" in result.recommended_capabilities
|
||||
assert result.estimated_complexity == "simple"
|
||||
assert "tatlock_core" in result.recommended_capabilities
|
||||
assert result.estimated_complexity == "simple"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_conversation_history(self):
|
||||
@@ -75,21 +69,18 @@ class TestAnalyzeRequest:
|
||||
]
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"And what's that times 5?",
|
||||
conversation_history=conversation_history,
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"And what's that times 5?",
|
||||
conversation_history=conversation_history,
|
||||
)
|
||||
assert result.conversation_context.has_previous_context is True
|
||||
assert 0 in result.conversation_context.relevant_turns
|
||||
|
||||
assert result.conversation_context.has_previous_context is True
|
||||
assert 0 in result.conversation_context.relevant_turns
|
||||
|
||||
# Verify conversation history was passed
|
||||
call_kwargs = mock_agent.analyze.call_args.kwargs
|
||||
assert "conversation_history" in call_kwargs
|
||||
assert len(call_kwargs["conversation_history"]) == 2
|
||||
# Verify conversation history was passed
|
||||
call_kwargs = mock_agent.analyze.call_args.kwargs
|
||||
assert "conversation_history" in call_kwargs
|
||||
assert len(call_kwargs["conversation_history"]) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_missing_capabilities(self):
|
||||
@@ -100,16 +91,13 @@ class TestAnalyzeRequest:
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"Generate an image of a sunset",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"Generate an image of a sunset",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
assert result.missing_capabilities is not None
|
||||
assert "not available" in result.missing_capabilities
|
||||
assert result.missing_capabilities is not None
|
||||
assert "not available" in result.missing_capabilities
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_conversation_id(self):
|
||||
@@ -120,18 +108,15 @@ class TestAnalyzeRequest:
|
||||
)
|
||||
|
||||
with patch("src.agents.steward.service.get_steward_agent", return_value=mock_agent):
|
||||
with patch("src.agents.steward.service.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
result = await analyze_request(
|
||||
"Test request",
|
||||
conversation_history=[],
|
||||
conversation_id="test_conv_123",
|
||||
)
|
||||
|
||||
result = await analyze_request(
|
||||
"Test request",
|
||||
conversation_history=[],
|
||||
conversation_id="test_conv_123",
|
||||
)
|
||||
|
||||
# Verify analysis completed successfully
|
||||
assert result.recommended_capabilities == ["tatlock_core"]
|
||||
assert result.estimated_complexity == "simple"
|
||||
# Verify analysis completed successfully
|
||||
assert result.recommended_capabilities == ["tatlock_core"]
|
||||
assert result.estimated_complexity == "simple"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_handles_errors(self):
|
||||
|
||||
@@ -34,7 +34,7 @@ async def test_tatlock_conversation_history_memory(async_client: AsyncClient):
|
||||
response_1 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data_1,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_1.status_code == 200
|
||||
@@ -56,7 +56,7 @@ async def test_tatlock_conversation_history_memory(async_client: AsyncClient):
|
||||
response_2 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data_2,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_2.status_code == 200
|
||||
@@ -95,7 +95,7 @@ async def test_tatlock_multi_turn_context(async_client: AsyncClient):
|
||||
response_1 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_1,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_1.status_code == 200
|
||||
@@ -117,7 +117,7 @@ async def test_tatlock_multi_turn_context(async_client: AsyncClient):
|
||||
response_2 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_2,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_2.status_code == 200
|
||||
@@ -150,7 +150,7 @@ async def test_tatlock_tool_call_logging_search(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=60.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -192,7 +192,7 @@ async def test_tatlock_tool_call_logging_calculator(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -243,7 +243,7 @@ async def test_tatlock_tool_call_logging_datetime(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -293,7 +293,7 @@ async def test_tatlock_no_tool_calls_no_logging(async_client: AsyncClient):
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
@@ -336,7 +336,7 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
|
||||
response_1 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_1,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_1.status_code == 200
|
||||
@@ -362,7 +362,7 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
|
||||
response_2 = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_2,
|
||||
timeout=30.0
|
||||
timeout=120.0
|
||||
)
|
||||
|
||||
assert response_2.status_code == 200
|
||||
@@ -378,3 +378,57 @@ async def test_tatlock_conversation_history_with_tools(async_client: AsyncClient
|
||||
)
|
||||
if not has_calculation:
|
||||
pytest.xfail(f"LLM did not remember calculation (non-deterministic): {second_response[:200]}")
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_tatlock_ollama_fallback(async_client: AsyncClient):
|
||||
"""
|
||||
Test that Tatlock falls back to Ollama when Claude is unavailable.
|
||||
|
||||
Patches _claude_available to False to force the Ollama path,
|
||||
then verifies the system still produces a valid response.
|
||||
"""
|
||||
import src.anthropic.model_selector as model_selector
|
||||
|
||||
# Save original value
|
||||
original = model_selector._claude_available
|
||||
|
||||
try:
|
||||
# Force Ollama fallback
|
||||
model_selector._claude_available = False
|
||||
|
||||
# Verify we're actually using Ollama
|
||||
info = model_selector.get_model_info()
|
||||
assert info["backend"] == "ollama", f"Expected ollama backend, got {info['backend']}"
|
||||
|
||||
request_data = {
|
||||
"model": "Tatlock",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Say hello to me."}
|
||||
],
|
||||
"stream": False
|
||||
}
|
||||
|
||||
# 300s: this test forbids the Claude rescue, and the full local
|
||||
# Steward -> orchestrate -> synthesize flow on gemma4 exceeds 120s
|
||||
response = await async_client.post(
|
||||
"/v1/chat/completions",
|
||||
json=request_data,
|
||||
timeout=300.0
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Verify response structure is valid
|
||||
assert "choices" in data
|
||||
assert len(data["choices"]) == 1
|
||||
full_response = data["choices"][0]["message"]["content"]
|
||||
assert len(full_response) > 0, "Ollama should produce a non-empty response"
|
||||
|
||||
print(f"\nOllama fallback response: {full_response[:200]}")
|
||||
|
||||
finally:
|
||||
# Restore original value
|
||||
model_selector._claude_available = original
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Unit tests for backend selection (Ollama primary, Claude fallback).
|
||||
|
||||
These tests set the cached health-check globals directly so they are
|
||||
deterministic regardless of which services are reachable.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from src.anthropic import model_selector
|
||||
from src.core.config import config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_first(monkeypatch):
|
||||
"""Baseline: local-first config, both backends healthy."""
|
||||
monkeypatch.setattr(config, "PREFER_CLOUD_BACKEND", False)
|
||||
monkeypatch.setattr(config, "ANTHROPIC_API_KEY", "sk-test-fake")
|
||||
monkeypatch.setattr(model_selector, "_claude_available", True)
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", True)
|
||||
|
||||
|
||||
class TestResolveBackend:
|
||||
def test_default_is_ollama(self, local_first):
|
||||
assert model_selector.resolve_backend() == "ollama"
|
||||
|
||||
def test_prefer_cloud_config_selects_claude(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(config, "PREFER_CLOUD_BACKEND", True)
|
||||
assert model_selector.resolve_backend() == "claude"
|
||||
|
||||
def test_prefer_cloud_override_selects_claude(self, local_first):
|
||||
assert model_selector.resolve_backend(prefer_cloud=True) == "claude"
|
||||
|
||||
def test_prefer_cloud_without_claude_falls_back_to_ollama(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(config, "PREFER_CLOUD_BACKEND", True)
|
||||
monkeypatch.setattr(model_selector, "_claude_available", False)
|
||||
assert model_selector.resolve_backend() == "ollama"
|
||||
|
||||
def test_ollama_down_falls_back_to_claude(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", False)
|
||||
assert model_selector.resolve_backend() == "claude"
|
||||
|
||||
def test_ollama_down_without_claude_stays_ollama(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", False)
|
||||
monkeypatch.setattr(model_selector, "_claude_available", False)
|
||||
assert model_selector.resolve_backend() == "ollama"
|
||||
|
||||
def test_unknown_ollama_state_counts_as_available(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", None)
|
||||
assert model_selector.resolve_backend() == "ollama"
|
||||
|
||||
|
||||
class TestGetModel:
|
||||
def test_ollama_backend_returns_openai_chat_model(self, local_first):
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
|
||||
model = model_selector.get_model()
|
||||
assert isinstance(model, OpenAIChatModel)
|
||||
assert model.model_name == config.OLLAMA_DEFAULT_MODEL
|
||||
|
||||
def test_claude_backend_returns_anthropic_model(self, local_first):
|
||||
from pydantic_ai.models.anthropic import AnthropicModel
|
||||
|
||||
model = model_selector.get_model(prefer_cloud=True)
|
||||
assert isinstance(model, AnthropicModel)
|
||||
assert model.model_name == config.ANTHROPIC_MODEL
|
||||
|
||||
|
||||
class TestToolChoiceSettings:
|
||||
def test_ollama_forces_tool_choice(self, local_first):
|
||||
settings = model_selector.get_tool_choice_settings()
|
||||
assert settings.get("extra_body") == {"tool_choice": "required"}
|
||||
|
||||
def test_claude_uses_native_tool_choice(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(config, "PREFER_CLOUD_BACKEND", True)
|
||||
settings = model_selector.get_tool_choice_settings()
|
||||
assert not settings.get("extra_body")
|
||||
|
||||
|
||||
class TestGetModelInfo:
|
||||
def test_reports_ollama_primary(self, local_first):
|
||||
info = model_selector.get_model_info()
|
||||
assert info["backend"] == "ollama"
|
||||
assert info["model"] == config.OLLAMA_DEFAULT_MODEL
|
||||
assert info["ollama_available"] is True
|
||||
assert info["claude_available"] is True
|
||||
assert info["prefer_cloud"] is False
|
||||
|
||||
def test_reports_claude_when_ollama_down(self, local_first, monkeypatch):
|
||||
monkeypatch.setattr(model_selector, "_ollama_available", False)
|
||||
info = model_selector.get_model_info()
|
||||
assert info["backend"] == "claude"
|
||||
assert info["model"] == config.ANTHROPIC_MODEL
|
||||
+16
-3
@@ -2,6 +2,8 @@
|
||||
Shared test fixtures for all tests.
|
||||
Following FastAPI testing best practices.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
@@ -9,11 +11,22 @@ from httpx import AsyncClient, ASGITransport
|
||||
from src.main import app
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def _initialize_app():
|
||||
"""
|
||||
Run application lifespan (Claude health check, household registration, etc.)
|
||||
once per test session. ASGITransport doesn't trigger lifespan events,
|
||||
so we call it explicitly.
|
||||
"""
|
||||
from src.core.startup import initialize_application
|
||||
asyncio.run(initialize_application())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
"""
|
||||
Synchronous test client for FastAPI.
|
||||
|
||||
|
||||
Use for simple tests that don't require async.
|
||||
"""
|
||||
return TestClient(app)
|
||||
@@ -23,7 +36,7 @@ def client() -> TestClient:
|
||||
async def async_client() -> AsyncClient:
|
||||
"""
|
||||
Async test client for FastAPI.
|
||||
|
||||
|
||||
Use for testing async endpoints and streaming.
|
||||
"""
|
||||
async with AsyncClient(
|
||||
@@ -37,7 +50,7 @@ async def async_client() -> AsyncClient:
|
||||
def mock_chat_request() -> dict:
|
||||
"""Standard chat completion request fixture."""
|
||||
return {
|
||||
"model": "Tatlock",
|
||||
"model": "lorem-tester",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello, world!"}
|
||||
],
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
"""
|
||||
Wire-level contract tests for external service boundaries.
|
||||
|
||||
Each test sends the raw request the application code sends (no client
|
||||
wrappers, no mocks) and asserts on the response shape, so boundary
|
||||
breakage is caught directly instead of surfacing as agent misbehavior.
|
||||
|
||||
Semantics:
|
||||
- Service unreachable -> skip (an outage is not a contract violation)
|
||||
- Service reachable but wrong response shape -> fail
|
||||
|
||||
Run with: make test-contracts
|
||||
"""
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from src.core.config import config
|
||||
|
||||
OLLAMA = str(config.OLLAMA_HOST).rstrip("/")
|
||||
QDRANT = f"http://{config.QDRANT_HOST}:{config.QDRANT_PORT}"
|
||||
SEARXNG = str(config.SEARXNG_HOST).rstrip("/")
|
||||
|
||||
CALCULATOR_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calculator",
|
||||
"description": "Evaluate a math expression",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"expression": {"type": "string"}},
|
||||
"required": ["expression"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def _get_or_skip(url: str, service: str, timeout: float = 5.0) -> httpx.Response:
|
||||
"""GET a URL, skipping the test if the service is unreachable."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
return await client.get(url)
|
||||
except httpx.TransportError as e:
|
||||
pytest.skip(f"{service} unreachable at {url}: {e}")
|
||||
|
||||
|
||||
async def _post_or_skip(
|
||||
url: str, service: str, payload: dict, timeout: float, headers: dict | None = None
|
||||
) -> httpx.Response:
|
||||
"""POST a payload, skipping the test if the service is unreachable."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
return await client.post(url, json=payload, headers=headers)
|
||||
except httpx.TransportError as e:
|
||||
pytest.skip(f"{service} unreachable at {url}: {e}")
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestOllamaContract:
|
||||
"""Boundary: Ollama native API and its OpenAI-compat layer."""
|
||||
|
||||
async def test_tags_lists_configured_model(self):
|
||||
# Mirrors check_ollama_health()
|
||||
response = await _get_or_skip(f"{OLLAMA}/api/tags", "ollama")
|
||||
assert response.status_code == 200
|
||||
names = [m["name"] for m in response.json()["models"]]
|
||||
model = config.OLLAMA_DEFAULT_MODEL
|
||||
assert model in names or f"{model}:latest" in names, (
|
||||
f"{model} not pulled; available: {names}"
|
||||
)
|
||||
|
||||
async def test_generate_returns_plain_text(self):
|
||||
# Mirrors StewardAgent._call_ollama()
|
||||
response = await _post_or_skip(
|
||||
f"{OLLAMA}/api/generate",
|
||||
"ollama",
|
||||
{
|
||||
"model": config.OLLAMA_DEFAULT_MODEL,
|
||||
"prompt": "Reply with the single word: pong",
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.3, "top_p": 0.9},
|
||||
},
|
||||
timeout=config.OLLAMA_TIMEOUT,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["response"].strip()
|
||||
|
||||
async def test_openai_compat_tool_calling(self):
|
||||
# Mirrors the request PydanticAI's OpenAIChatModel sends for the
|
||||
# orchestration phase, including the extra_body tool_choice.
|
||||
response = await _post_or_skip(
|
||||
f"{OLLAMA}/v1/chat/completions",
|
||||
"ollama",
|
||||
{
|
||||
"model": config.OLLAMA_DEFAULT_MODEL,
|
||||
"messages": [
|
||||
{"role": "user", "content": "What is 6 * 7? Use the calculator."}
|
||||
],
|
||||
"tools": [CALCULATOR_TOOL],
|
||||
"tool_choice": "required",
|
||||
"stream": False,
|
||||
},
|
||||
timeout=config.OLLAMA_TIMEOUT,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
message = response.json()["choices"][0]["message"]
|
||||
tool_calls = message.get("tool_calls")
|
||||
assert tool_calls, f"model answered in text instead of calling the tool: {message}"
|
||||
assert tool_calls[0]["function"]["name"] == "calculator"
|
||||
arguments = json.loads(tool_calls[0]["function"]["arguments"])
|
||||
assert "expression" in arguments
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestAnthropicContract:
|
||||
"""Boundary: Anthropic Messages API (the Claude fallback backend)."""
|
||||
|
||||
HEADERS_KEY = "anthropic-version"
|
||||
|
||||
def _headers(self) -> dict:
|
||||
if not config.ANTHROPIC_API_KEY:
|
||||
pytest.skip("ANTHROPIC_API_KEY not configured")
|
||||
return {
|
||||
"x-api-key": config.ANTHROPIC_API_KEY,
|
||||
"anthropic-version": "2023-06-01",
|
||||
}
|
||||
|
||||
async def test_minimal_message_accepted(self):
|
||||
# Mirrors check_claude_health(): tiny request, no sampling params
|
||||
response = await _post_or_skip(
|
||||
"https://api.anthropic.com/v1/messages",
|
||||
"anthropic",
|
||||
{
|
||||
"model": config.ANTHROPIC_MODEL,
|
||||
"max_tokens": 1,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
},
|
||||
timeout=30.0,
|
||||
headers=self._headers(),
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
async def test_temperature_rejected(self):
|
||||
# Pins the Claude Sonnet 5+ contract that broke the Steward:
|
||||
# sampling parameters are rejected with a 400 (and not billed).
|
||||
response = await _post_or_skip(
|
||||
"https://api.anthropic.com/v1/messages",
|
||||
"anthropic",
|
||||
{
|
||||
"model": config.ANTHROPIC_MODEL,
|
||||
"max_tokens": 1,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"temperature": 0.3,
|
||||
},
|
||||
timeout=30.0,
|
||||
headers=self._headers(),
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "temperature" in response.text
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestQdrantContract:
|
||||
"""Boundary: Qdrant REST API (Biographer's vector memory)."""
|
||||
|
||||
async def test_collections_endpoint(self):
|
||||
response = await _get_or_skip(f"{QDRANT}/collections", "qdrant")
|
||||
assert response.status_code == 200
|
||||
assert "collections" in response.json()["result"]
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestSearxngContract:
|
||||
"""Boundary: SearXNG JSON search API (web search tool)."""
|
||||
|
||||
async def test_json_search(self):
|
||||
response = await _get_or_skip(
|
||||
f"{SEARXNG}/search?q=test&format=json", "searxng", timeout=config.SEARXNG_TIMEOUT
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert "results" in response.json()
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestLibraryDeskContract:
|
||||
"""Boundary: library-desk research API (the Librarian's backend)."""
|
||||
|
||||
async def test_health(self):
|
||||
host = getattr(config, "LIBRARY_DESK_HOST", None)
|
||||
if not host:
|
||||
pytest.skip("LIBRARY_DESK_HOST not configured")
|
||||
response = await _get_or_skip(f"{str(host).rstrip('/')}/health", "library-desk")
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.contract
|
||||
class TestRedisContract:
|
||||
"""Boundary: Redis on the configured memory DB."""
|
||||
|
||||
async def test_roundtrip(self):
|
||||
import redis.asyncio as redis
|
||||
|
||||
client = redis.Redis(
|
||||
host=config.REDIS_HOST,
|
||||
port=config.REDIS_PORT,
|
||||
db=config.REDIS_MEMORY_DB,
|
||||
socket_connect_timeout=3,
|
||||
)
|
||||
try:
|
||||
await client.ping()
|
||||
except Exception as e:
|
||||
pytest.skip(f"redis unreachable: {e}")
|
||||
try:
|
||||
await client.set("contract-test-key", "ok", ex=30)
|
||||
assert await client.get("contract-test-key") == b"ok"
|
||||
await client.delete("contract-test-key")
|
||||
finally:
|
||||
await client.aclose()
|
||||
@@ -3,8 +3,6 @@ Tests for tool call tracking.
|
||||
|
||||
Tests capability extraction and recommendation matching.
|
||||
"""
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.core.tool_tracking import ToolCallTracker
|
||||
@@ -35,15 +33,11 @@ class TestToolCallTracker:
|
||||
recommended_capabilities=["librarian", "biographer"]
|
||||
)
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
await tracker.track_call("delegate_to_librarian", 1.0)
|
||||
|
||||
await tracker.track_call("delegate_to_librarian", 1.0)
|
||||
|
||||
# Should NOT log warning since librarian was recommended
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.was_recommended is True
|
||||
# Should record the call
|
||||
assert "delegate_to_librarian" in tracker.actual_calls
|
||||
assert tracker.actual_calls["delegate_to_librarian"] == [1.0]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_track_call_detects_not_recommended(self):
|
||||
@@ -52,14 +46,12 @@ class TestToolCallTracker:
|
||||
recommended_capabilities=["librarian"]
|
||||
)
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
await tracker.track_call("delegate_to_housekeeper", 1.0)
|
||||
|
||||
await tracker.track_call("delegate_to_housekeeper", 1.0)
|
||||
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.was_recommended is False
|
||||
# Should record the call even though not recommended
|
||||
assert "delegate_to_housekeeper" in tracker.actual_calls
|
||||
summary = tracker.get_summary()
|
||||
assert summary["accuracy"]["not_recommended_but_used"] == 1
|
||||
|
||||
def test_get_summary_with_delegation_tools(self):
|
||||
"""Test summary correctly maps delegation tools to capabilities."""
|
||||
@@ -87,15 +79,9 @@ class TestToolCallTracker:
|
||||
"delegate_to_librarian": [1.0],
|
||||
}
|
||||
|
||||
with patch("src.core.tool_tracking.get_benchmark_store") as mock_store:
|
||||
mock_store.return_value.record = AsyncMock()
|
||||
await tracker.finalize()
|
||||
|
||||
await tracker.finalize()
|
||||
|
||||
# Should record benchmark for unused biographer
|
||||
assert mock_store.return_value.record.called
|
||||
call_args = mock_store.return_value.record.call_args
|
||||
benchmark = call_args[0][0]
|
||||
assert benchmark.tool_name == "biographer"
|
||||
assert benchmark.was_recommended is True
|
||||
assert benchmark.was_actually_used is False
|
||||
# Summary should show biographer as recommended but unused
|
||||
summary = tracker.get_summary()
|
||||
assert summary["accuracy"]["recommended_and_used"] == 1 # librarian
|
||||
assert summary["accuracy"]["recommended_but_unused"] == 1 # biographer
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
|
||||
|
||||
#!/bin/bash
|
||||
# Tatlock Server Startup Script
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${GREEN}Starting Tatlock server...${NC}"
|
||||
|
||||
# Check if port 8777 is already in use
|
||||
if lsof -Pi :8777 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
|
||||
echo -e "${RED}Error: Port 8777 is already in use${NC}"
|
||||
echo "Run: lsof -i :8777 to see what's using it"
|
||||
echo "Or run: kill \$(lsof -t -i:8777) to stop it"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Activate virtual environment if not already activated
|
||||
if [ -z "$VIRTUAL_ENV" ]; then
|
||||
if [ -d ".venv" ]; then
|
||||
echo -e "${YELLOW}Activating virtual environment...${NC}"
|
||||
source .venv/bin/activate
|
||||
else
|
||||
echo -e "${RED}Error: Virtual environment not found${NC}"
|
||||
echo "Run: python -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
LOGS_DIR="logs"
|
||||
mkdir -p "$LOGS_DIR"
|
||||
|
||||
# Clear/create log file
|
||||
LOG_FILE="$LOGS_DIR/server.log"
|
||||
> "$LOG_FILE"
|
||||
echo -e "${YELLOW}Logs will be written to: ${LOG_FILE}${NC}"
|
||||
|
||||
# Start the server
|
||||
echo -e "${GREEN}Starting uvicorn server on http://tower-of-joy:8777${NC}"
|
||||
echo -e "${YELLOW}Press Ctrl+C to stop the server${NC}"
|
||||
echo ""
|
||||
|
||||
uvicorn src.main:app --reload --host 0.0.0.0 --port 8777 2>&1 | tee "$LOG_FILE"
|
||||
Reference in New Issue
Block a user