feat: implement Phase 2 two-tier architecture with Steward
Add comprehensive two-tier architecture where Steward analyzes requests and Tatlock executes with scoped tools. Includes full infrastructure for request preprocessing, tool tracking, benchmarking, and streaming. **Added:** - Steward agent for request analysis and capability recommendation - Household Registry for centralized capability management - Request preprocessing pipeline (Steward → Tatlock flow) - Tool usage tracking and benchmarking system - Streaming transparency (Steward reasoning visible in streams) - Structured logging with operation timing - Redis benchmark storage with 30-day expiry - Benchmark analysis CLI tools **Infrastructure:** - src/agents/steward/ - Steward agent implementation - src/agents/tatlock_core/ - Tatlock capability domain - src/core/preprocessing.py - Request preprocessing pipeline - src/core/tool_tracking.py - Tool call tracking - src/core/benchmarks.py - Benchmark recording system - src/core/household_registry.py - Capability registry - src/core/startup.py - Application startup coordination - src/core/logging_config.py - Structured logging setup **Integration:** - Responses API uses Steward for Tatlock requests - Chat Completions wraps Responses API for OpenAI compatibility - Streaming coordinator supports Steward + Tatlock flow - Tool scoping per request based on Steward recommendations **Testing:** - Integration tests for Steward-Tatlock flow - Benchmark and registry unit tests - Steward streaming tests See PHASE2_PLAN.md and PHASE2_COMPLETE.md for detailed documentation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.5
Executable
+296
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Benchmark analysis tool for Steward performance and tool recommendation accuracy.
|
||||
|
||||
Usage:
|
||||
# View Steward performance over last 24 hours
|
||||
python scripts/benchmark_analysis.py --operation steward_analysis --hours 24
|
||||
|
||||
# Analyze tool recommendation accuracy over last 7 days
|
||||
python scripts/benchmark_analysis.py --tool-accuracy --days 7
|
||||
|
||||
# Get summary of all operations in last hour
|
||||
python scripts/benchmark_analysis.py --summary --hours 1
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
project_root = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List
|
||||
from collections import defaultdict
|
||||
|
||||
from src.core.benchmarks import get_benchmark_store, PerformanceBenchmark
|
||||
|
||||
|
||||
async def analyze_steward_performance(hours: int = 24):
|
||||
"""
|
||||
Analyze Steward analysis performance over time.
|
||||
|
||||
Args:
|
||||
hours: Number of hours to look back
|
||||
"""
|
||||
store = get_benchmark_store()
|
||||
|
||||
# Query benchmarks from last N hours
|
||||
since = datetime.now() - timedelta(hours=hours)
|
||||
benchmarks = await store.query(
|
||||
operation="steward_analysis",
|
||||
since=since
|
||||
)
|
||||
|
||||
if not benchmarks:
|
||||
print(f"No Steward analysis benchmarks found in the last {hours} hours.")
|
||||
return
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Steward Analysis Performance (Last {hours} hours)")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Calculate statistics
|
||||
durations = [b.duration_seconds for b in benchmarks]
|
||||
recommendation_counts = [b.recommendation_count for b in benchmarks if b.recommendation_count is not None]
|
||||
|
||||
avg_duration = sum(durations) / len(durations)
|
||||
min_duration = min(durations)
|
||||
max_duration = max(durations)
|
||||
|
||||
print(f"Total Analyses: {len(benchmarks)}")
|
||||
print(f"Success Rate: {sum(1 for b in benchmarks if b.success) / len(benchmarks) * 100:.1f}%")
|
||||
print(f"\nLatency Statistics:")
|
||||
print(f" Average: {avg_duration:.3f}s")
|
||||
print(f" Min: {min_duration:.3f}s")
|
||||
print(f" Max: {max_duration:.3f}s")
|
||||
|
||||
if recommendation_counts:
|
||||
avg_recommendations = sum(recommendation_counts) / len(recommendation_counts)
|
||||
print(f"\nRecommendation Statistics:")
|
||||
print(f" Average recommendations per request: {avg_recommendations:.1f}")
|
||||
print(f" Min recommendations: {min(recommendation_counts)}")
|
||||
print(f" Max recommendations: {max(recommendation_counts)}")
|
||||
|
||||
# Distribution
|
||||
print(f"\nRecommendation Count Distribution:")
|
||||
distribution = defaultdict(int)
|
||||
for count in recommendation_counts:
|
||||
distribution[count] += 1
|
||||
for count in sorted(distribution.keys()):
|
||||
percentage = distribution[count] / len(recommendation_counts) * 100
|
||||
print(f" {count} capabilities: {distribution[count]} ({percentage:.1f}%)")
|
||||
|
||||
# Complexity distribution
|
||||
complexities = defaultdict(int)
|
||||
for b in benchmarks:
|
||||
if b.metadata and "complexity" in b.metadata:
|
||||
complexities[b.metadata["complexity"]] += 1
|
||||
|
||||
if complexities:
|
||||
print(f"\nComplexity Distribution:")
|
||||
for complexity in sorted(complexities.keys()):
|
||||
percentage = complexities[complexity] / len(benchmarks) * 100
|
||||
print(f" {complexity}: {complexities[complexity]} ({percentage:.1f}%)")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
async def analyze_tool_accuracy(days: int = 7):
|
||||
"""
|
||||
Analyze tool recommendation accuracy.
|
||||
|
||||
Args:
|
||||
days: Number of days to look back
|
||||
"""
|
||||
store = get_benchmark_store()
|
||||
|
||||
# Query tool call benchmarks from last N days
|
||||
since = datetime.now() - timedelta(days=days)
|
||||
benchmarks = await store.query(
|
||||
operation="tool_call",
|
||||
since=since
|
||||
)
|
||||
|
||||
if not benchmarks:
|
||||
print(f"No tool call benchmarks found in the last {days} days.")
|
||||
return
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Tool Recommendation Accuracy (Last {days} days)")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Categorize tool calls
|
||||
recommended_and_used = [] # True positives
|
||||
recommended_not_used = [] # False positives (recommended but not used)
|
||||
not_recommended_but_used = [] # False negatives (used but not recommended)
|
||||
|
||||
for b in benchmarks:
|
||||
if b.was_recommended and b.was_actually_used:
|
||||
recommended_and_used.append(b)
|
||||
elif b.was_recommended and not b.was_actually_used:
|
||||
recommended_not_used.append(b)
|
||||
elif not b.was_recommended and b.was_actually_used:
|
||||
not_recommended_but_used.append(b)
|
||||
|
||||
total_recommendations = len(recommended_and_used) + len(recommended_not_used)
|
||||
total_tool_calls = len(recommended_and_used) + len(not_recommended_but_used)
|
||||
|
||||
print(f"Total Tool Calls: {total_tool_calls}")
|
||||
print(f"Total Recommendations: {total_recommendations}")
|
||||
|
||||
if total_recommendations > 0:
|
||||
precision = len(recommended_and_used) / total_recommendations * 100
|
||||
print(f"\nPrecision: {precision:.1f}%")
|
||||
print(f" (recommended and actually used / all recommendations)")
|
||||
|
||||
if total_tool_calls > 0:
|
||||
recall = len(recommended_and_used) / total_tool_calls * 100
|
||||
print(f"\nRecall: {recall:.1f}%")
|
||||
print(f" (recommended and actually used / all tool calls)")
|
||||
|
||||
if total_recommendations > 0 and total_tool_calls > 0:
|
||||
f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0
|
||||
print(f"\nF1 Score: {f1:.1f}%")
|
||||
|
||||
print(f"\nBreakdown:")
|
||||
print(f" ✅ Recommended & Used: {len(recommended_and_used)}")
|
||||
print(f" ⚠️ Recommended but Not Used: {len(recommended_not_used)}")
|
||||
print(f" ❌ Not Recommended but Used: {len(not_recommended_but_used)}")
|
||||
|
||||
# Tool-specific accuracy
|
||||
tool_usage = defaultdict(lambda: {"recommended_used": 0, "not_recommended_used": 0})
|
||||
|
||||
for b in recommended_and_used:
|
||||
if b.tool_name:
|
||||
tool_usage[b.tool_name]["recommended_used"] += 1
|
||||
|
||||
for b in not_recommended_but_used:
|
||||
if b.tool_name:
|
||||
tool_usage[b.tool_name]["not_recommended_used"] += 1
|
||||
|
||||
if tool_usage:
|
||||
print(f"\nPer-Tool Accuracy:")
|
||||
for tool_name in sorted(tool_usage.keys()):
|
||||
stats = tool_usage[tool_name]
|
||||
total = stats["recommended_used"] + stats["not_recommended_used"]
|
||||
accuracy = stats["recommended_used"] / total * 100 if total > 0 else 0
|
||||
print(f" {tool_name}: {accuracy:.1f}% ({stats['recommended_used']}/{total})")
|
||||
|
||||
# Duration statistics for tool calls
|
||||
durations = [b.duration_seconds for b in benchmarks if b.duration_seconds]
|
||||
if durations:
|
||||
avg_duration = sum(durations) / len(durations)
|
||||
print(f"\nTool Call Duration:")
|
||||
print(f" Average: {avg_duration:.3f}s")
|
||||
print(f" Min: {min(durations):.3f}s")
|
||||
print(f" Max: {max(durations):.3f}s")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
async def show_summary(hours: int = 1):
|
||||
"""
|
||||
Show summary of all operations in the specified time window.
|
||||
|
||||
Args:
|
||||
hours: Number of hours to look back
|
||||
"""
|
||||
store = get_benchmark_store()
|
||||
|
||||
since = datetime.now() - timedelta(hours=hours)
|
||||
|
||||
# Query all operations
|
||||
all_benchmarks = await store.query(since=since)
|
||||
|
||||
if not all_benchmarks:
|
||||
print(f"No benchmarks found in the last {hours} hours.")
|
||||
return
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Benchmark Summary (Last {hours} hours)")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Group by operation
|
||||
by_operation = defaultdict(list)
|
||||
for b in all_benchmarks:
|
||||
by_operation[b.operation].append(b)
|
||||
|
||||
print(f"Total Operations: {len(all_benchmarks)}\n")
|
||||
|
||||
for operation in sorted(by_operation.keys()):
|
||||
benchmarks = by_operation[operation]
|
||||
durations = [b.duration_seconds for b in benchmarks if b.duration_seconds]
|
||||
avg_duration = sum(durations) / len(durations) if durations else 0
|
||||
success_rate = sum(1 for b in benchmarks if b.success) / len(benchmarks) * 100
|
||||
|
||||
print(f"{operation}:")
|
||||
print(f" Count: {len(benchmarks)}")
|
||||
print(f" Success Rate: {success_rate:.1f}%")
|
||||
if durations:
|
||||
print(f" Avg Duration: {avg_duration:.3f}s")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Analyze Tatlock benchmark data",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog=__doc__
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--operation",
|
||||
choices=["steward_analysis", "tool_call"],
|
||||
help="Analyze specific operation type"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--hours",
|
||||
type=int,
|
||||
default=24,
|
||||
help="Number of hours to look back (default: 24)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--days",
|
||||
type=int,
|
||||
default=7,
|
||||
help="Number of days to look back (default: 7)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--tool-accuracy",
|
||||
action="store_true",
|
||||
help="Analyze tool recommendation accuracy"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--summary",
|
||||
action="store_true",
|
||||
help="Show summary of all operations"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Run analysis
|
||||
if args.tool_accuracy:
|
||||
asyncio.run(analyze_tool_accuracy(args.days))
|
||||
elif args.summary:
|
||||
asyncio.run(show_summary(args.hours))
|
||||
elif args.operation == "steward_analysis":
|
||||
asyncio.run(analyze_steward_performance(args.hours))
|
||||
elif args.operation == "tool_call":
|
||||
# Show tool-specific analysis within the hours window
|
||||
asyncio.run(analyze_tool_accuracy(days=args.hours // 24 or 1))
|
||||
else:
|
||||
# Default: show summary
|
||||
asyncio.run(show_summary(args.hours))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+300
@@ -0,0 +1,300 @@
|
||||
"""
|
||||
Benchmark script for the Steward agent.
|
||||
|
||||
Tests Steward's request analysis performance with various scenarios
|
||||
to ensure it meets latency targets:
|
||||
- Target max: 5 seconds
|
||||
- Target average: ~1.67 seconds
|
||||
|
||||
Usage:
|
||||
python scripts/benchmark_steward.py [--iterations N] [--verbose]
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import statistics
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
|
||||
from src.agents.steward import analyze_request
|
||||
from src.core.startup import initialize_application
|
||||
|
||||
|
||||
class BenchmarkResult:
|
||||
"""Results from a single benchmark run."""
|
||||
|
||||
def __init__(self, scenario: str, duration: float, success: bool, error: str = None):
|
||||
self.scenario = scenario
|
||||
self.duration = duration
|
||||
self.success = success
|
||||
self.error = error
|
||||
|
||||
|
||||
async def benchmark_scenario(
|
||||
name: str,
|
||||
request: str,
|
||||
history: list[dict],
|
||||
iterations: int = 10
|
||||
) -> List[BenchmarkResult]:
|
||||
"""
|
||||
Benchmark a specific scenario.
|
||||
|
||||
Args:
|
||||
name: Scenario name
|
||||
request: User request to analyze
|
||||
history: Conversation history
|
||||
iterations: Number of times to run
|
||||
|
||||
Returns:
|
||||
List of benchmark results
|
||||
"""
|
||||
results = []
|
||||
|
||||
print(f"\n📊 Benchmarking: {name}")
|
||||
print(f" Request: {request[:50]}{'...' if len(request) > 50 else ''}")
|
||||
print(f" History length: {len(history)} turns")
|
||||
print(f" Iterations: {iterations}")
|
||||
|
||||
for i in range(iterations):
|
||||
try:
|
||||
start = datetime.now()
|
||||
await analyze_request(request, history)
|
||||
duration = (datetime.now() - start).total_seconds()
|
||||
|
||||
results.append(BenchmarkResult(name, duration, True))
|
||||
|
||||
# Progress indicator
|
||||
print(".", end="", flush=True)
|
||||
|
||||
except Exception as e:
|
||||
duration = (datetime.now() - start).total_seconds()
|
||||
results.append(BenchmarkResult(name, duration, False, str(e)))
|
||||
print("E", end="", flush=True)
|
||||
|
||||
print() # New line after progress
|
||||
return results
|
||||
|
||||
|
||||
def analyze_results(results: List[BenchmarkResult], scenario_name: str):
|
||||
"""
|
||||
Analyze and display benchmark results.
|
||||
|
||||
Args:
|
||||
results: List of benchmark results
|
||||
scenario_name: Name of the scenario
|
||||
"""
|
||||
successful = [r for r in results if r.success]
|
||||
failed = [r for r in results if not r.success]
|
||||
|
||||
if not successful:
|
||||
print(f"\n❌ {scenario_name}: All runs failed!")
|
||||
for r in failed[:3]: # Show first 3 errors
|
||||
print(f" Error: {r.error}")
|
||||
return
|
||||
|
||||
durations = [r.duration for r in successful]
|
||||
|
||||
min_duration = min(durations)
|
||||
max_duration = max(durations)
|
||||
avg_duration = statistics.mean(durations)
|
||||
median_duration = statistics.median(durations)
|
||||
|
||||
# Calculate percentiles
|
||||
sorted_durations = sorted(durations)
|
||||
p95_idx = int(len(sorted_durations) * 0.95)
|
||||
p99_idx = int(len(sorted_durations) * 0.99)
|
||||
p95 = sorted_durations[p95_idx] if p95_idx < len(sorted_durations) else max_duration
|
||||
p99 = sorted_durations[p99_idx] if p99_idx < len(sorted_durations) else max_duration
|
||||
|
||||
# Targets
|
||||
target_max = 5.0
|
||||
target_avg = 1.67
|
||||
|
||||
# Status emojis
|
||||
max_status = "✅" if max_duration <= target_max else "⚠️"
|
||||
avg_status = "✅" if avg_duration <= target_avg else "⚠️"
|
||||
|
||||
print(f"\n Results ({len(successful)}/{len(results)} successful):")
|
||||
print(f" Min: {min_duration:6.3f}s")
|
||||
print(f" Avg: {avg_duration:6.3f}s {avg_status} (target: ≤{target_avg}s)")
|
||||
print(f" Median: {median_duration:6.3f}s")
|
||||
print(f" P95: {p95:6.3f}s")
|
||||
print(f" P99: {p99:6.3f}s")
|
||||
print(f" Max: {max_duration:6.3f}s {max_status} (target: ≤{target_max}s)")
|
||||
|
||||
if failed:
|
||||
print(f" Failed: {len(failed)} runs")
|
||||
|
||||
return {
|
||||
"min": min_duration,
|
||||
"avg": avg_duration,
|
||||
"median": median_duration,
|
||||
"p95": p95,
|
||||
"p99": p99,
|
||||
"max": max_duration,
|
||||
"success_rate": len(successful) / len(results) * 100,
|
||||
}
|
||||
|
||||
|
||||
async def run_benchmarks(iterations: int = 10, verbose: bool = False):
|
||||
"""
|
||||
Run comprehensive Steward benchmarks.
|
||||
|
||||
Args:
|
||||
iterations: Number of iterations per scenario
|
||||
verbose: Enable verbose output
|
||||
"""
|
||||
print("=" * 60)
|
||||
print("🔬 Steward Performance Benchmark")
|
||||
print("=" * 60)
|
||||
print(f"\nTargets:")
|
||||
print(f" - Maximum response time: ≤5.0s")
|
||||
print(f" - Average response time: ≤1.67s")
|
||||
print(f"\nIterations per scenario: {iterations}")
|
||||
|
||||
# Initialize application
|
||||
print("\n🚀 Initializing application...")
|
||||
initialize_application()
|
||||
|
||||
all_stats = {}
|
||||
|
||||
# Scenario 1: Simple greeting (no capabilities needed)
|
||||
results = await benchmark_scenario(
|
||||
"Simple Greeting",
|
||||
"Hello!",
|
||||
[],
|
||||
iterations
|
||||
)
|
||||
all_stats["simple_greeting"] = analyze_results(results, "Simple Greeting")
|
||||
|
||||
# Scenario 2: Single tool request (calculator)
|
||||
results = await benchmark_scenario(
|
||||
"Calculator Request",
|
||||
"What's sqrt(144) + 25?",
|
||||
[],
|
||||
iterations
|
||||
)
|
||||
all_stats["calculator"] = analyze_results(results, "Calculator Request")
|
||||
|
||||
# Scenario 3: Web search request
|
||||
results = await benchmark_scenario(
|
||||
"Web Search Request",
|
||||
"Search for the latest Python 3.12 features",
|
||||
[],
|
||||
iterations
|
||||
)
|
||||
all_stats["web_search"] = analyze_results(results, "Web Search Request")
|
||||
|
||||
# Scenario 4: Request with conversation history (short)
|
||||
short_history = [
|
||||
{"role": "user", "content": "What's 15 times 7?"},
|
||||
{"role": "assistant", "content": "105"},
|
||||
]
|
||||
results = await benchmark_scenario(
|
||||
"With Short History",
|
||||
"And what's that divided by 3?",
|
||||
short_history,
|
||||
iterations
|
||||
)
|
||||
all_stats["short_history"] = analyze_results(results, "With Short History")
|
||||
|
||||
# Scenario 5: Request with longer conversation history
|
||||
long_history = [
|
||||
{"role": "user", "content": f"Question {i}"} if i % 2 == 0
|
||||
else {"role": "assistant", "content": f"Answer {i}"}
|
||||
for i in range(20)
|
||||
]
|
||||
results = await benchmark_scenario(
|
||||
"With Long History",
|
||||
"What was the first question I asked?",
|
||||
long_history,
|
||||
iterations
|
||||
)
|
||||
all_stats["long_history"] = analyze_results(results, "With Long History")
|
||||
|
||||
# Scenario 6: Complex request
|
||||
results = await benchmark_scenario(
|
||||
"Complex Request",
|
||||
"Calculate the compound interest on $5000 at 4.5% over 10 years, "
|
||||
"then search for current savings account rates to compare",
|
||||
[],
|
||||
iterations
|
||||
)
|
||||
all_stats["complex"] = analyze_results(results, "Complex Request")
|
||||
|
||||
# Scenario 7: Missing capabilities
|
||||
results = await benchmark_scenario(
|
||||
"Missing Capabilities",
|
||||
"Generate an image of a sunset over mountains",
|
||||
[],
|
||||
iterations
|
||||
)
|
||||
all_stats["missing_caps"] = analyze_results(results, "Missing Capabilities")
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 60)
|
||||
print("📈 SUMMARY")
|
||||
print("=" * 60)
|
||||
|
||||
# Calculate overall stats
|
||||
all_avgs = [stats["avg"] for stats in all_stats.values() if stats]
|
||||
all_maxs = [stats["max"] for stats in all_stats.values() if stats]
|
||||
|
||||
if all_avgs:
|
||||
overall_avg = statistics.mean(all_avgs)
|
||||
overall_max = max(all_maxs)
|
||||
|
||||
avg_status = "✅" if overall_avg <= 1.67 else "⚠️"
|
||||
max_status = "✅" if overall_max <= 5.0 else "⚠️"
|
||||
|
||||
print(f"\nOverall Performance:")
|
||||
print(f" Average of averages: {overall_avg:.3f}s {avg_status}")
|
||||
print(f" Maximum observed: {overall_max:.3f}s {max_status}")
|
||||
|
||||
# Performance verdict
|
||||
print(f"\n{'=' * 60}")
|
||||
if overall_avg <= 1.67 and overall_max <= 5.0:
|
||||
print("✅ PERFORMANCE TARGETS MET!")
|
||||
print(f" The Steward is operating within target parameters.")
|
||||
elif overall_max <= 5.0:
|
||||
print("⚠️ PARTIAL SUCCESS")
|
||||
print(f" Max response time is good, but average is above target.")
|
||||
print(f" Average: {overall_avg:.3f}s (target: ≤1.67s)")
|
||||
print(f"\n Recommendations:")
|
||||
print(f" - Consider using a faster model")
|
||||
print(f" - Optimize system prompt length")
|
||||
print(f" - Review tool call limits")
|
||||
else:
|
||||
print("❌ PERFORMANCE TARGETS NOT MET")
|
||||
print(f" Max: {overall_max:.3f}s (target: ≤5.0s)")
|
||||
print(f" Avg: {overall_avg:.3f}s (target: ≤1.67s)")
|
||||
print(f"\n Recommendations:")
|
||||
print(f" - Switch to a faster model (current: mistral-nemo)")
|
||||
print(f" - Reduce system prompt complexity")
|
||||
print(f" - Limit tool calls (currently limited to 3)")
|
||||
print(f" - Consider caching household registry responses")
|
||||
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(description="Benchmark Steward agent performance")
|
||||
parser.add_argument(
|
||||
"--iterations",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Number of iterations per scenario (default: 10)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose output"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
await run_benchmarks(iterations=args.iterations, verbose=args.verbose)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple test to verify Steward agent works correctly.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from src.agents.steward import analyze_request
|
||||
from src.core.startup import initialize_application
|
||||
|
||||
|
||||
async def main():
|
||||
"""Test a simple request."""
|
||||
print("Initializing application...")
|
||||
initialize_application()
|
||||
|
||||
print("\nTesting simple greeting...")
|
||||
result = await analyze_request(
|
||||
"Hello!",
|
||||
conversation_history=[],
|
||||
)
|
||||
|
||||
print(f"\nResult type: {type(result)}")
|
||||
print(f"Result: {result}")
|
||||
|
||||
if hasattr(result, 'recommended_capabilities'):
|
||||
print(f"\nRecommended capabilities: {result.recommended_capabilities}")
|
||||
print(f"Complexity: {result.estimated_complexity}")
|
||||
print(f"Reasoning: {result.reasoning}")
|
||||
else:
|
||||
print("\nERROR: Result doesn't have expected attributes!")
|
||||
print(f"Result attributes: {dir(result)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user