feat: integrate tracing throughout request pipeline

Instrument the full request flow with trace spans for debugging:

- Wrap expert delegations (librarian/biographer/housekeeper) in spans
- Add orchestrate and synthesize spans to TatlockAgent
- Trace Steward analysis in preprocessing
- Start/end traces in response service with context management
- Simplify router by moving context handling to service layer
- Include tracing router in debug mode
- Remove benchmark recording from tool_tracking and steward service

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

src/agents/delegation.py

@@ -13,6 +13,7 @@ from enum import Enum
 from typing import AsyncGenerator, Callable, Optional, Any
 
 from src.core.logging_config import get_logger
+from src.core.tracing import trace_span, SpanType
 
 logger = get_logger(__name__)
 
@@ -239,38 +240,58 @@ async def delegate_to_librarian(
         has_context=bool(context),
     )
 
-    try:
-        # Use run() not run_stream() - avoids Ollama bug
-        output = await run_librarian(task=task, context=context)
+    async with trace_span(
+        "delegate_to_librarian",
+        SpanType.EXPERT,
+        metadata={
+            "expert": "librarian",
+            "task_preview": task[:100],
+            "has_context": bool(context),
+        },
+    ) as span:
+        try:
+            # Use run() not run_stream() - avoids Ollama bug
+            output = await run_librarian(task=task, context=context)
 
-        logger.info(
-            "delegation_to_librarian_completed",
-            task=task[:50],
-            output_length=len(output),
-        )
+            logger.info(
+                "delegation_to_librarian_completed",
+                task=task[:50],
+                output_length=len(output),
+            )
 
-        return DelegationResult(
-            expert_name="librarian",
-            task=task,
-            success=True,
-            output=output,
-        )
+            if span:
+                span.metadata["success"] = True
+                span.metadata["output_length"] = len(output)
+                span.details["task"] = task
+                span.details["context"] = context[:500] if context else None
+                span.details["result_preview"] = output[:1000]
 
-    except Exception as e:
-        logger.error(
-            "delegation_to_librarian_error",
-            task=task[:50],
-            error=str(e),
-            exc_info=True,
-        )
+            return DelegationResult(
+                expert_name="librarian",
+                task=task,
+                success=True,
+                output=output,
+            )
 
-        return DelegationResult(
-            expert_name="librarian",
-            task=task,
-            success=False,
-            output="",
-            error=str(e),
-        )
+        except Exception as e:
+            logger.error(
+                "delegation_to_librarian_error",
+                task=task[:50],
+                error=str(e),
+                exc_info=True,
+            )
+
+            if span:
+                span.metadata["success"] = False
+                span.details["error"] = str(e)
+
+            return DelegationResult(
+                expert_name="librarian",
+                task=task,
+                success=False,
+                output="",
+                error=str(e),
+            )
 
 
 async def delegate_to_biographer(
@@ -317,38 +338,58 @@ async def delegate_to_biographer(
         has_context=bool(context),
     )
 
-    try:
-        # Use run() not run_stream() - avoids Ollama bug
-        output = await run_biographer(task=task, context=context)
+    async with trace_span(
+        "delegate_to_biographer",
+        SpanType.EXPERT,
+        metadata={
+            "expert": "biographer",
+            "task_preview": task[:100],
+            "has_context": bool(context),
+        },
+    ) as span:
+        try:
+            # Use run() not run_stream() - avoids Ollama bug
+            output = await run_biographer(task=task, context=context)
 
-        logger.info(
-            "delegation_to_biographer_completed",
-            task=task[:50],
-            output_length=len(output),
-        )
+            logger.info(
+                "delegation_to_biographer_completed",
+                task=task[:50],
+                output_length=len(output),
+            )
 
-        return DelegationResult(
-            expert_name="biographer",
-            task=task,
-            success=True,
-            output=output,
-        )
+            if span:
+                span.metadata["success"] = True
+                span.metadata["output_length"] = len(output)
+                span.details["task"] = task
+                span.details["context"] = context[:500] if context else None
+                span.details["result_preview"] = output[:1000]
 
-    except Exception as e:
-        logger.error(
-            "delegation_to_biographer_error",
-            task=task[:50],
-            error=str(e),
-            exc_info=True,
-        )
+            return DelegationResult(
+                expert_name="biographer",
+                task=task,
+                success=True,
+                output=output,
+            )
 
-        return DelegationResult(
-            expert_name="biographer",
-            task=task,
-            success=False,
-            output="",
-            error=str(e),
-        )
+        except Exception as e:
+            logger.error(
+                "delegation_to_biographer_error",
+                task=task[:50],
+                error=str(e),
+                exc_info=True,
+            )
+
+            if span:
+                span.metadata["success"] = False
+                span.details["error"] = str(e)
+
+            return DelegationResult(
+                expert_name="biographer",
+                task=task,
+                success=False,
+                output="",
+                error=str(e),
+            )
 
 
 async def delegate_to_housekeeper(
@@ -394,38 +435,58 @@ async def delegate_to_housekeeper(
         has_context=bool(context),
     )
 
-    try:
-        # Use run() not run_stream() - avoids Ollama bug
-        output = await run_housekeeper(task=task, context=context)
+    async with trace_span(
+        "delegate_to_housekeeper",
+        SpanType.EXPERT,
+        metadata={
+            "expert": "housekeeper",
+            "task_preview": task[:100],
+            "has_context": bool(context),
+        },
+    ) as span:
+        try:
+            # Use run() not run_stream() - avoids Ollama bug
+            output = await run_housekeeper(task=task, context=context)
 
-        logger.info(
-            "delegation_to_housekeeper_completed",
-            task=task[:50],
-            output_length=len(output),
-        )
+            logger.info(
+                "delegation_to_housekeeper_completed",
+                task=task[:50],
+                output_length=len(output),
+            )
 
-        return DelegationResult(
-            expert_name="housekeeper",
-            task=task,
-            success=True,
-            output=output,
-        )
+            if span:
+                span.metadata["success"] = True
+                span.metadata["output_length"] = len(output)
+                span.details["task"] = task
+                span.details["context"] = context[:500] if context else None
+                span.details["result_preview"] = output[:1000]
 
-    except Exception as e:
-        logger.error(
-            "delegation_to_housekeeper_error",
-            task=task[:50],
-            error=str(e),
-            exc_info=True,
-        )
+            return DelegationResult(
+                expert_name="housekeeper",
+                task=task,
+                success=True,
+                output=output,
+            )
 
-        return DelegationResult(
-            expert_name="housekeeper",
-            task=task,
-            success=False,
-            output="",
-            error=str(e),
-        )
+        except Exception as e:
+            logger.error(
+                "delegation_to_housekeeper_error",
+                task=task[:50],
+                error=str(e),
+                exc_info=True,
+            )
+
+            if span:
+                span.metadata["success"] = False
+                span.details["error"] = str(e)
+
+            return DelegationResult(
+                expert_name="housekeeper",
+                task=task,
+                success=False,
+                output="",
+                error=str(e),
+            )
 
 
 # =============================================================================

src/agents/steward/service.py

@@ -1,8 +1,8 @@
 """
 Steward service layer.
 
-Provides high-level interface for request analysis with logging,
-benchmarking, and error handling.
+Provides high-level interface for request analysis with logging
+and error handling.
 
 Parses plain text recommendations into structured data.
 Includes memory pre-fetch for user context injection.
@@ -10,7 +10,6 @@ Includes memory pre-fetch for user context injection.
 import re
 from typing import Any, Optional
 
-from src.core.benchmarks import PerformanceBenchmark, get_benchmark_store
 from src.core.household_registry import get_household_registry
 from src.core.logging_config import get_logger, log_operation
 from src.core.memory_service import memory_service
@@ -285,8 +284,7 @@ async def analyze_request(
     This is the main entry point for Steward analysis. It:
     1. Calls the Steward agent with full conversation history
     2. Logs the operation with timing
-    3. Records performance benchmarks to Redis
-    4. Returns structured recommendations
+    3. Returns structured recommendations
 
     Args:
         user_request: The current user message to analyze
@@ -365,23 +363,6 @@ async def analyze_request(
                 reasoning=analysis_text[:200],  # First 200 chars
             )
 
-            # Record performance benchmark
-            if log_ctx.get("duration_seconds"):
-                benchmark = PerformanceBenchmark(
-                    operation="steward_analysis",
-                    duration_seconds=log_ctx["duration_seconds"],
-                    success=True,
-                    recommendation_count=len(recommendation.recommended_capabilities),
-                    confidence=None,  # Could add confidence scoring in future
-                    conversation_id=conversation_id,
-                    metadata={
-                        "complexity": recommendation.estimated_complexity,
-                        "has_context": recommendation.conversation_context.has_previous_context,
-                        "missing_capabilities": recommendation.missing_capabilities is not None,
-                    },
-                )
-                await get_benchmark_store().record(benchmark)
-
             return recommendation
 
         except Exception as e:

src/agents/tatlock.py

@@ -20,6 +20,11 @@ from src.agents.tatlock_core.tools import (
 )
 from src.core.config import config
 from src.core.logging_config import get_logger
+from src.core.tracing import (
+    start_span, end_span, get_current_span,
+    add_tool_spans_from_messages,
+    SpanType, SpanStatus,
+)
 
 logger = get_logger(__name__)
 
@@ -433,7 +438,7 @@ class TatlockAgent(AgentInterface):
             steward_note: Note from Steward (prepended to request, invisible to user)
             scoped_tools: List of tool definitions from household registry
             message_history: Conversation history in PydanticAI format
-            tool_tracker: Optional tool call tracker for benchmarking
+            tool_tracker: Optional tool call tracker for analysis
 
         Returns:
             str: Tatlock's response text
@@ -627,7 +632,7 @@ class TatlockAgent(AgentInterface):
             steward_note: Note from Steward (invisible to user)
             scoped_tools: List of tool definitions from household registry
             message_history: Conversation history
-            tool_tracker: Optional tool call tracker for benchmarking
+            tool_tracker: Optional tool call tracker for analysis
 
         Returns:
             dict with:
@@ -655,6 +660,16 @@ class TatlockAgent(AgentInterface):
             history_length=len(message_history),
         )
 
+        # Start tracing span for orchestration phase
+        orchestrate_span = start_span(
+            "tatlock_orchestrate",
+            SpanType.TATLOCK,
+            metadata={
+                "scoped_tool_count": len(scoped_tools),
+                "tool_names": [getattr(t, '__name__', str(t)) for t in scoped_tools[:5]],
+            },
+        )
+
         # Create a fresh agent instance with scoped tools only
         clean_host = self.ollama_host.rstrip('/')
         base_url = f"{clean_host}/v1"
@@ -731,6 +746,23 @@ class TatlockAgent(AgentInterface):
             tool_output_count=len(tool_outputs),
         )
 
+        # Add tool-level spans from result messages
+        if orchestrate_span:
+            add_tool_spans_from_messages(result.new_messages(), orchestrate_span)
+
+        # End orchestration span with results
+        end_span(
+            orchestrate_span,
+            metadata_update={
+                "tools_called": tools_called,
+                "expert_count": len(expert_results),
+                "tool_output_count": len(tool_outputs),
+            },
+            details_update={
+                "steward_note_preview": steward_note[:500] if steward_note else None,
+            },
+        )
+
         return {
             "tools_called": tools_called,
             "expert_results": expert_results,
@@ -769,6 +801,16 @@ class TatlockAgent(AgentInterface):
             tool_count=len(orchestration_results.get("tool_outputs", {})),
         )
 
+        # Start tracing span for synthesis phase
+        synthesize_span = start_span(
+            "tatlock_synthesize",
+            SpanType.TATLOCK,
+            metadata={
+                "expert_count": len(orchestration_results.get("expert_results", {})),
+                "tool_output_count": len(orchestration_results.get("tool_outputs", {})),
+            },
+        )
+
         # Build synthesis prompt with all available information
         synthesis_parts = []
         synthesis_parts.append(f"The user asked: {user_message}")
@@ -841,6 +883,18 @@ class TatlockAgent(AgentInterface):
             response_preview=result.output[:100],
         )
 
+        # End synthesis span with result
+        end_span(
+            synthesize_span,
+            metadata_update={
+                "response_length": len(result.output),
+            },
+            details_update={
+                "synthesis_prompt": synthesis_prompt[:1000],
+                "response_preview": result.output[:500],
+            },
+        )
+
         return result.output
 
     async def get_capabilities(self) -> dict: