feat: integrate tracing throughout request pipeline
Instrument the full request flow with trace spans for debugging: - Wrap expert delegations (librarian/biographer/housekeeper) in spans - Add orchestrate and synthesize spans to TatlockAgent - Trace Steward analysis in preprocessing - Start/end traces in response service with context management - Simplify router by moving context handling to service layer - Include tracing router in debug mode - Remove benchmark recording from tool_tracking and steward service 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,11 @@
|
||||
"""
|
||||
Tool call tracking and benchmarking.
|
||||
Tool call tracking.
|
||||
|
||||
Tracks which tools are recommended by the Steward versus which tools
|
||||
are actually used by Tatlock, recording benchmarks for analysis.
|
||||
are actually used by Tatlock for debugging and analysis.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from src.core.benchmarks import PerformanceBenchmark, get_benchmark_store
|
||||
from src.core.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -15,7 +13,7 @@ logger = get_logger(__name__)
|
||||
|
||||
class ToolCallTracker:
|
||||
"""
|
||||
Tracks tool calls for benchmarking and accuracy analysis.
|
||||
Tracks tool calls for accuracy analysis.
|
||||
|
||||
Compares Steward's recommendations with Tatlock's actual tool usage
|
||||
to measure recommendation accuracy.
|
||||
@@ -53,6 +51,10 @@ class ToolCallTracker:
|
||||
return tool_name.replace("delegate_to_", "")
|
||||
return tool_name
|
||||
|
||||
def log_call(self, message: str):
|
||||
"""Log a tool call message (for UI display)."""
|
||||
logger.debug("tool_call_message", message=message)
|
||||
|
||||
async def track_call(self, tool_name: str, duration: float):
|
||||
"""
|
||||
Record a tool call with timing.
|
||||
@@ -78,23 +80,6 @@ class ToolCallTracker:
|
||||
recommended=list(self.recommended_capabilities),
|
||||
)
|
||||
|
||||
# Record benchmark to Redis
|
||||
benchmark = PerformanceBenchmark(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
operation="tool_call",
|
||||
duration_seconds=duration,
|
||||
success=True, # If we got here, the call succeeded
|
||||
tool_name=tool_name,
|
||||
was_recommended=was_recommended,
|
||||
was_actually_used=True,
|
||||
conversation_id=self.conversation_id,
|
||||
metadata={
|
||||
"recommended_capabilities": list(self.recommended_capabilities),
|
||||
},
|
||||
)
|
||||
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
logger.debug(
|
||||
"tool_call_tracked",
|
||||
tool_name=tool_name,
|
||||
@@ -124,24 +109,6 @@ class ToolCallTracker:
|
||||
conversation_id=self.conversation_id,
|
||||
)
|
||||
|
||||
# Record benchmarks for unused recommendations
|
||||
for tool_name in unused_tools:
|
||||
benchmark = PerformanceBenchmark(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
operation="tool_call",
|
||||
duration_seconds=0.0, # Not used
|
||||
success=True,
|
||||
tool_name=tool_name,
|
||||
was_recommended=True,
|
||||
was_actually_used=False,
|
||||
conversation_id=self.conversation_id,
|
||||
metadata={
|
||||
"recommended_capabilities": list(self.recommended_capabilities),
|
||||
"reason": "recommended_but_unused",
|
||||
},
|
||||
)
|
||||
await get_benchmark_store().record(benchmark)
|
||||
|
||||
# Log summary
|
||||
total_calls = sum(len(durations) for durations in self.actual_calls.values())
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user