feat: integrate tracing throughout request pipeline

Instrument the full request flow with trace spans for debugging:

- Wrap expert delegations (librarian/biographer/housekeeper) in spans
- Add orchestrate and synthesize spans to TatlockAgent
- Trace Steward analysis in preprocessing
- Start/end traces in response service with context management
- Simplify router by moving context handling to service layer
- Include tracing router in debug mode
- Remove benchmark recording from tool_tracking and steward service

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2025-12-22 10:26:37 +01:00
co-authored by Claude Opus 4.5
parent 2a9449bc81
commit 87f2926db2
8 changed files with 502 additions and 369 deletions
+7 -40
View File
@@ -1,13 +1,11 @@
"""
Tool call tracking and benchmarking.
Tool call tracking.
Tracks which tools are recommended by the Steward versus which tools
are actually used by Tatlock, recording benchmarks for analysis.
are actually used by Tatlock for debugging and analysis.
"""
from datetime import datetime, timezone
from typing import Optional
from src.core.benchmarks import PerformanceBenchmark, get_benchmark_store
from src.core.logging_config import get_logger
logger = get_logger(__name__)
@@ -15,7 +13,7 @@ logger = get_logger(__name__)
class ToolCallTracker:
"""
Tracks tool calls for benchmarking and accuracy analysis.
Tracks tool calls for accuracy analysis.
Compares Steward's recommendations with Tatlock's actual tool usage
to measure recommendation accuracy.
@@ -53,6 +51,10 @@ class ToolCallTracker:
return tool_name.replace("delegate_to_", "")
return tool_name
def log_call(self, message: str):
"""Log a tool call message (for UI display)."""
logger.debug("tool_call_message", message=message)
async def track_call(self, tool_name: str, duration: float):
"""
Record a tool call with timing.
@@ -78,23 +80,6 @@ class ToolCallTracker:
recommended=list(self.recommended_capabilities),
)
# Record benchmark to Redis
benchmark = PerformanceBenchmark(
timestamp=datetime.now(timezone.utc),
operation="tool_call",
duration_seconds=duration,
success=True, # If we got here, the call succeeded
tool_name=tool_name,
was_recommended=was_recommended,
was_actually_used=True,
conversation_id=self.conversation_id,
metadata={
"recommended_capabilities": list(self.recommended_capabilities),
},
)
await get_benchmark_store().record(benchmark)
logger.debug(
"tool_call_tracked",
tool_name=tool_name,
@@ -124,24 +109,6 @@ class ToolCallTracker:
conversation_id=self.conversation_id,
)
# Record benchmarks for unused recommendations
for tool_name in unused_tools:
benchmark = PerformanceBenchmark(
timestamp=datetime.now(timezone.utc),
operation="tool_call",
duration_seconds=0.0, # Not used
success=True,
tool_name=tool_name,
was_recommended=True,
was_actually_used=False,
conversation_id=self.conversation_id,
metadata={
"recommended_capabilities": list(self.recommended_capabilities),
"reason": "recommended_but_unused",
},
)
await get_benchmark_store().record(benchmark)
# Log summary
total_calls = sum(len(durations) for durations in self.actual_calls.values())
logger.info(